From c638191a47ae5767de82b383ecb04496ce18877b Mon Sep 17 00:00:00 2001 From: Andy LoPresto Date: Mon, 15 Aug 2016 20:18:47 -0700 Subject: [PATCH] NIFI-1831 Added internal logic and command-line tool to allow AES-encrypted sensitive configuration values in nifi.properties. This closes #834. --- LICENSE | 22 + NOTICE | 2 + .../ProcessorInitializationContext.java | 1 - nifi-assembly/src/main/assembly/common.xml | 2 + .../org/apache/nifi/bootstrap/RunNiFi.java | 109 +- .../org/apache/nifi/util/NiFiProperties.java | 24 +- .../org/apache/nifi/util/StringUtils.java | 36 + .../apache/nifi/util/NiFiPropertiesTest.java | 6 +- .../nifi/controller/queue/SortColumn.java | 12 +- .../nifi/amqp/processors/AMQPUtils.java | 83 +- .../ClusterProtocolHeartbeatMonitor.java | 2 - .../StatusHistoryEndpointMerger.java | 1 - .../node/CuratorNodeProtocolSender.java | 1 - .../node/NodeClusterCoordinator.java | 1 - .../TestThreadPoolRequestReplicator.java | 17 +- .../node/TestNodeClusterCoordinator.java | 1 - .../apache/nifi/cluster/integration/Node.java | 4 +- .../nifi-framework-core/pom.xml | 4 + .../nifi/connectable/StandardConnection.java | 23 +- .../controller/FileSystemSwapManager.java | 7 +- .../cluster/ClusterProtocolHeartbeater.java | 2 - .../CuratorLeaderElectionManager.java | 1 - .../apache/nifi/encrypt/StringEncryptor.java | 2 +- .../remote/StandardRemoteProcessGroup.java | 55 +- .../src/main/resources/nifi-context.xml | 12 +- .../nifi/controller/TestFlowController.java | 33 +- .../TestStandardProcessScheduler.java | 1 - ...TestStandardControllerServiceProvider.java | 9 +- .../nifi/nar/NarThreadContextClassLoader.java | 16 +- .../nifi-properties-loader/pom.xml | 53 + .../AESSensitivePropertyProvider.java | 251 +++ .../AESSensitivePropertyProviderFactory.java | 53 + ...eSensitivePropertyProtectionException.java | 128 ++ .../nifi/properties/NiFiPropertiesLoader.java | 254 +++ .../properties/ProtectedNiFiProperties.java | 521 ++++++ .../SensitivePropertyProtectionException.java | 91 + .../properties/SensitivePropertyProvider.java | 52 + .../SensitivePropertyProviderFactory.java | 23 + .../properties/StandardNiFiProperties.java | 81 + ...ensitivePropertyProviderFactoryTest.groovy | 97 ++ .../AESSensitivePropertyProviderTest.groovy | 463 ++++++ .../NiFiPropertiesLoaderGroovyTest.groovy | 393 +++++ .../ProtectedNiFiPropertiesGroovyTest.groovy | 860 ++++++++++ .../StandardNiFiPropertiesGroovyTest.groovy | 150 ++ .../bootstrap_tests/conf/bootstrap.conf | 74 + ...sitive_properties_protected_aes.properties | 129 ++ .../missing_bootstrap/nifi.properties | 14 + .../missing_key/bootstrap.conf | 74 + .../missing_key/nifi.properties | 14 + .../missing_key_line/bootstrap.conf | 71 + .../missing_key_line/nifi.properties | 14 + .../unreadable_bootstrap/bootstrap.conf | 74 + .../unreadable_bootstrap/nifi.properties | 14 + .../unreadable_conf/bootstrap.conf | 74 + .../unreadable_conf/nifi.properties | 14 + .../test/resources/conf/nifi.blank.properties | 124 ++ .../resources/conf/nifi.missing.properties | 122 ++ .../src/test/resources/conf/nifi.properties | 124 ++ .../conf/nifi_no_permissions.properties | 14 + ..._with_additional_sensitive_keys.properties | 125 ++ ...sitive_properties_protected_aes.properties | 129 ++ ...rsive_additional_sensitive_keys.properties | 125 ++ ...sitive_properties_protected_aes.properties | 128 ++ ...rotected_aes_multiple_malformed.properties | 128 ++ ..._protected_aes_single_malformed.properties | 128 ++ ...ve_properties_protected_unknown.properties | 128 ++ ...ensitive_properties_unprotected.properties | 125 ++ ...operties_unprotected_extra_line.properties | 126 ++ .../nifi-framework/nifi-resources/pom.xml | 3 + .../src/main/resources/conf/bootstrap.conf | 2 + .../src/main/resources/conf/nifi.properties | 2 + .../nifi-framework/nifi-runtime/pom.xml | 10 + .../src/main/java/org/apache/nifi/NiFi.java | 104 +- .../org/apache/nifi/NiFiGroovyTest.groovy | 277 ++++ .../NiFiProperties/conf/nifi.properties | 188 +++ ...sitive_properties_protected_aes.properties | 189 +++ ...ies_protected_aes_different_key.properties | 192 +++ .../src/test/resources/logback-test.xml | 34 + .../StandardHttpFlowFileServerProtocol.java | 14 +- .../socket/SocketFlowFileServerProtocol.java | 27 +- .../nifi/web/api/DataTransferResource.java | 198 +-- .../nifi/web/api/SiteToSiteResource.java | 7 +- .../nifi-framework/pom.xml | 1 + .../nifi-framework-bundle/pom.xml | 5 + .../TestVolatileProvenanceRepository.java | 1 - nifi-toolkit/nifi-toolkit-assembly/NOTICE | 14 +- nifi-toolkit/nifi-toolkit-assembly/pom.xml | 6 +- .../src/main/assembly/dependencies.xml | 9 + .../src/main/resources/bin/encrypt-config.bat | 40 + .../src/main/resources/bin/encrypt-config.sh | 120 ++ .../nifi-toolkit-encrypt-config/LICENSE | 225 +++ .../nifi-toolkit-encrypt-config/pom.xml | 162 ++ .../properties/ConfigEncryptionTool.groovy | 578 +++++++ .../org/apache/nifi/properties/JavaMain.java | 23 + .../nifi/util/console/CharacterDevice.java | 73 + .../nifi/util/console/ConsoleDevice.java | 66 + .../nifi/util/console/ConsoleException.java | 35 + .../apache/nifi/util/console/TextDevice.java | 43 + .../apache/nifi/util/console/TextDevices.java | 80 + .../src/main/resources/log4j.properties | 22 + .../src/main/resources/logback.xml | 34 + .../ConfigEncryptionToolTest.groovy | 1458 +++++++++++++++++ .../src/test/resources/bootstrap.conf | 72 + .../bootstrap_with_empty_master_key.conf | 74 + .../resources/bootstrap_with_master_key.conf | 74 + .../src/test/resources/log4j.properties | 22 + .../src/test/resources/logback-test.xml | 34 + ...sitive_properties_protected_aes.properties | 34 + ...ensitive_properties_unprotected.properties | 31 + ...sitive_properties_protected_aes.properties | 128 ++ ...ensitive_properties_unprotected.properties | 125 ++ ...ed_and_empty_protection_schemes.properties | 127 ++ nifi-toolkit/nifi-toolkit-tls/pom.xml | 4 - nifi-toolkit/pom.xml | 21 +- pom.xml | 12 +- 115 files changed, 10687 insertions(+), 364 deletions(-) create mode 100644 nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-properties-loader/pom.xml create mode 100644 nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-properties-loader/src/main/java/org/apache/nifi/properties/AESSensitivePropertyProvider.java create mode 100644 nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-properties-loader/src/main/java/org/apache/nifi/properties/AESSensitivePropertyProviderFactory.java create mode 100644 nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-properties-loader/src/main/java/org/apache/nifi/properties/MultipleSensitivePropertyProtectionException.java create mode 100644 nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-properties-loader/src/main/java/org/apache/nifi/properties/NiFiPropertiesLoader.java create mode 100644 nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-properties-loader/src/main/java/org/apache/nifi/properties/ProtectedNiFiProperties.java create mode 100644 nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-properties-loader/src/main/java/org/apache/nifi/properties/SensitivePropertyProtectionException.java create mode 100644 nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-properties-loader/src/main/java/org/apache/nifi/properties/SensitivePropertyProvider.java create mode 100644 nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-properties-loader/src/main/java/org/apache/nifi/properties/SensitivePropertyProviderFactory.java create mode 100644 nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-properties-loader/src/main/java/org/apache/nifi/properties/StandardNiFiProperties.java create mode 100644 nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-properties-loader/src/test/groovy/org/apache/nifi/properties/AESSensitivePropertyProviderFactoryTest.groovy create mode 100644 nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-properties-loader/src/test/groovy/org/apache/nifi/properties/AESSensitivePropertyProviderTest.groovy create mode 100644 nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-properties-loader/src/test/groovy/org/apache/nifi/properties/NiFiPropertiesLoaderGroovyTest.groovy create mode 100644 nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-properties-loader/src/test/groovy/org/apache/nifi/properties/ProtectedNiFiPropertiesGroovyTest.groovy create mode 100644 nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-properties-loader/src/test/groovy/org/apache/nifi/properties/StandardNiFiPropertiesGroovyTest.groovy create mode 100644 nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-properties-loader/src/test/resources/bootstrap_tests/conf/bootstrap.conf create mode 100644 nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-properties-loader/src/test/resources/bootstrap_tests/conf/nifi_with_sensitive_properties_protected_aes.properties create mode 100644 nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-properties-loader/src/test/resources/bootstrap_tests/missing_bootstrap/nifi.properties create mode 100644 nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-properties-loader/src/test/resources/bootstrap_tests/missing_key/bootstrap.conf create mode 100644 nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-properties-loader/src/test/resources/bootstrap_tests/missing_key/nifi.properties create mode 100644 nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-properties-loader/src/test/resources/bootstrap_tests/missing_key_line/bootstrap.conf create mode 100644 nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-properties-loader/src/test/resources/bootstrap_tests/missing_key_line/nifi.properties create mode 100755 nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-properties-loader/src/test/resources/bootstrap_tests/unreadable_bootstrap/bootstrap.conf create mode 100644 nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-properties-loader/src/test/resources/bootstrap_tests/unreadable_bootstrap/nifi.properties create mode 100755 nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-properties-loader/src/test/resources/bootstrap_tests/unreadable_conf/bootstrap.conf create mode 100644 nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-properties-loader/src/test/resources/bootstrap_tests/unreadable_conf/nifi.properties create mode 100644 nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-properties-loader/src/test/resources/conf/nifi.blank.properties create mode 100644 nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-properties-loader/src/test/resources/conf/nifi.missing.properties create mode 100644 nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-properties-loader/src/test/resources/conf/nifi.properties create mode 100644 nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-properties-loader/src/test/resources/conf/nifi_no_permissions.properties create mode 100644 nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-properties-loader/src/test/resources/conf/nifi_with_additional_sensitive_keys.properties create mode 100644 nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-properties-loader/src/test/resources/conf/nifi_with_all_sensitive_properties_protected_aes.properties create mode 100644 nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-properties-loader/src/test/resources/conf/nifi_with_recursive_additional_sensitive_keys.properties create mode 100644 nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-properties-loader/src/test/resources/conf/nifi_with_sensitive_properties_protected_aes.properties create mode 100644 nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-properties-loader/src/test/resources/conf/nifi_with_sensitive_properties_protected_aes_multiple_malformed.properties create mode 100644 nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-properties-loader/src/test/resources/conf/nifi_with_sensitive_properties_protected_aes_single_malformed.properties create mode 100644 nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-properties-loader/src/test/resources/conf/nifi_with_sensitive_properties_protected_unknown.properties create mode 100644 nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-properties-loader/src/test/resources/conf/nifi_with_sensitive_properties_unprotected.properties create mode 100644 nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-properties-loader/src/test/resources/conf/nifi_with_sensitive_properties_unprotected_extra_line.properties create mode 100644 nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-runtime/src/test/groovy/org/apache/nifi/NiFiGroovyTest.groovy create mode 100644 nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-runtime/src/test/resources/NiFiProperties/conf/nifi.properties create mode 100644 nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-runtime/src/test/resources/NiFiProperties/conf/nifi_with_sensitive_properties_protected_aes.properties create mode 100644 nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-runtime/src/test/resources/NiFiProperties/conf/nifi_with_sensitive_properties_protected_aes_different_key.properties create mode 100644 nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-runtime/src/test/resources/logback-test.xml create mode 100644 nifi-toolkit/nifi-toolkit-assembly/src/main/resources/bin/encrypt-config.bat create mode 100644 nifi-toolkit/nifi-toolkit-assembly/src/main/resources/bin/encrypt-config.sh create mode 100644 nifi-toolkit/nifi-toolkit-encrypt-config/LICENSE create mode 100644 nifi-toolkit/nifi-toolkit-encrypt-config/pom.xml create mode 100644 nifi-toolkit/nifi-toolkit-encrypt-config/src/main/groovy/org/apache/nifi/properties/ConfigEncryptionTool.groovy create mode 100644 nifi-toolkit/nifi-toolkit-encrypt-config/src/main/java/org/apache/nifi/properties/JavaMain.java create mode 100644 nifi-toolkit/nifi-toolkit-encrypt-config/src/main/java/org/apache/nifi/util/console/CharacterDevice.java create mode 100644 nifi-toolkit/nifi-toolkit-encrypt-config/src/main/java/org/apache/nifi/util/console/ConsoleDevice.java create mode 100644 nifi-toolkit/nifi-toolkit-encrypt-config/src/main/java/org/apache/nifi/util/console/ConsoleException.java create mode 100644 nifi-toolkit/nifi-toolkit-encrypt-config/src/main/java/org/apache/nifi/util/console/TextDevice.java create mode 100644 nifi-toolkit/nifi-toolkit-encrypt-config/src/main/java/org/apache/nifi/util/console/TextDevices.java create mode 100644 nifi-toolkit/nifi-toolkit-encrypt-config/src/main/resources/log4j.properties create mode 100644 nifi-toolkit/nifi-toolkit-encrypt-config/src/main/resources/logback.xml create mode 100644 nifi-toolkit/nifi-toolkit-encrypt-config/src/test/groovy/org/apache/nifi/properties/ConfigEncryptionToolTest.groovy create mode 100644 nifi-toolkit/nifi-toolkit-encrypt-config/src/test/resources/bootstrap.conf create mode 100644 nifi-toolkit/nifi-toolkit-encrypt-config/src/test/resources/bootstrap_with_empty_master_key.conf create mode 100644 nifi-toolkit/nifi-toolkit-encrypt-config/src/test/resources/bootstrap_with_master_key.conf create mode 100644 nifi-toolkit/nifi-toolkit-encrypt-config/src/test/resources/log4j.properties create mode 100644 nifi-toolkit/nifi-toolkit-encrypt-config/src/test/resources/logback-test.xml create mode 100644 nifi-toolkit/nifi-toolkit-encrypt-config/src/test/resources/nifi_with_few_sensitive_properties_protected_aes.properties create mode 100644 nifi-toolkit/nifi-toolkit-encrypt-config/src/test/resources/nifi_with_few_sensitive_properties_unprotected.properties create mode 100644 nifi-toolkit/nifi-toolkit-encrypt-config/src/test/resources/nifi_with_sensitive_properties_protected_aes.properties create mode 100644 nifi-toolkit/nifi-toolkit-encrypt-config/src/test/resources/nifi_with_sensitive_properties_unprotected.properties create mode 100644 nifi-toolkit/nifi-toolkit-encrypt-config/src/test/resources/nifi_with_sensitive_properties_unprotected_and_empty_protection_schemes.properties diff --git a/LICENSE b/LICENSE index 74473f9c6e..3db82f6b34 100644 --- a/LICENSE +++ b/LICENSE @@ -583,3 +583,25 @@ This product bundles 'jsonlint' which is available under an MIT license. LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +This product bundles source from 'AbstractingTheJavaConsole'. The source is available under an MIT LICENSE. + + Copyright (C) 2010 McDowell + + Permission is hereby granted, free of charge, to any person obtaining a copy + of this software and associated documentation files (the "Software"), to deal + in the Software without restriction, including without limitation the rights + to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + copies of the Software, and to permit persons to whom the Software is + furnished to do so, subject to the following conditions: + + The above copyright notice and this permission notice shall be included in + all copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + THE SOFTWARE. diff --git a/NOTICE b/NOTICE index f6ac34908e..c2026ba3d1 100644 --- a/NOTICE +++ b/NOTICE @@ -13,6 +13,8 @@ Copyright 2012, 2013 Willi Ballenthin william.ballenthin@mandiant.com while at Mandiant http://www.mandiant.com The derived work is adapted from Evtx/Evtx.py, Evtx/BinaryParser.py, Evtx/Nodes.py, Evtx/Views.py and can be found in the org.apache.nifi.processors.evtx.parser package. + + This includes derived works from the Apache Storm (ASLv2 licensed) project (https://github.com/apache/storm): Copyright 2015 The Apache Software Foundation The derived work is adapted from diff --git a/nifi-api/src/main/java/org/apache/nifi/processor/ProcessorInitializationContext.java b/nifi-api/src/main/java/org/apache/nifi/processor/ProcessorInitializationContext.java index df1819376a..b4c76435b0 100644 --- a/nifi-api/src/main/java/org/apache/nifi/processor/ProcessorInitializationContext.java +++ b/nifi-api/src/main/java/org/apache/nifi/processor/ProcessorInitializationContext.java @@ -52,5 +52,4 @@ public interface ProcessorInitializationContext extends KerberosContext { * type of this NiFi instance. */ NodeTypeProvider getNodeTypeProvider(); - } diff --git a/nifi-assembly/src/main/assembly/common.xml b/nifi-assembly/src/main/assembly/common.xml index ec26548370..80af55205d 100644 --- a/nifi-assembly/src/main/assembly/common.xml +++ b/nifi-assembly/src/main/assembly/common.xml @@ -28,6 +28,8 @@ slf4j-api logback-classic nifi-api + commons-lang3 + bcprov-jdk15on diff --git a/nifi-bootstrap/src/main/java/org/apache/nifi/bootstrap/RunNiFi.java b/nifi-bootstrap/src/main/java/org/apache/nifi/bootstrap/RunNiFi.java index 173a19919b..eb3d8eceaf 100644 --- a/nifi-bootstrap/src/main/java/org/apache/nifi/bootstrap/RunNiFi.java +++ b/nifi-bootstrap/src/main/java/org/apache/nifi/bootstrap/RunNiFi.java @@ -53,7 +53,6 @@ import java.util.concurrent.TimeUnit; import java.util.concurrent.locks.Condition; import java.util.concurrent.locks.Lock; import java.util.concurrent.locks.ReentrantLock; - import org.apache.commons.lang3.StringUtils; import org.apache.nifi.bootstrap.notification.NotificationType; import org.apache.nifi.util.file.FileUtils; @@ -61,7 +60,6 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; /** - * *

* The class which bootstraps Apache NiFi. This class looks for the * bootstrap.conf file by looking in the following places (in order):

@@ -73,7 +71,7 @@ import org.slf4j.LoggerFactory; *
  • ./conf/bootstrap.conf, where {@code ./} represents the working * directory.
    • * - * + *

    * If the {@code bootstrap.conf} file cannot be found, throws a {@code FileNotFoundException}. */ public class RunNiFi { @@ -98,6 +96,7 @@ public class RunNiFi { public static final String NIFI_PID_FILE_NAME = "nifi.pid"; public static final String NIFI_STATUS_FILE_NAME = "nifi.status"; public static final String NIFI_LOCK_FILE_NAME = "nifi.lock"; + public static final String NIFI_BOOTSTRAP_SENSITIVE_KEY = "nifi.bootstrap.sensitive.key"; public static final String PID_KEY = "pid"; @@ -332,7 +331,7 @@ public class RunNiFi { } - private File getBootstrapFile(final Logger logger, String directory, String defaultDirectory, String fileName) throws IOException{ + private File getBootstrapFile(final Logger logger, String directory, String defaultDirectory, String fileName) throws IOException { final File confDir = bootstrapConfigFile.getParentFile(); final File nifiHome = confDir.getParentFile(); @@ -341,9 +340,9 @@ public class RunNiFi { final File fileDir; - if(confFileDir != null){ + if (confFileDir != null) { fileDir = new File(confFileDir.trim()); - } else{ + } else { fileDir = new File(nifiHome, defaultDirectory); } @@ -353,19 +352,19 @@ public class RunNiFi { return statusFile; } - File getPidFile(final Logger logger) throws IOException{ - return getBootstrapFile(logger,NIFI_PID_DIR_PROP,DEFAULT_PID_DIR,NIFI_PID_FILE_NAME); + File getPidFile(final Logger logger) throws IOException { + return getBootstrapFile(logger, NIFI_PID_DIR_PROP, DEFAULT_PID_DIR, NIFI_PID_FILE_NAME); } - File getStatusFile(final Logger logger) throws IOException{ - return getBootstrapFile(logger,NIFI_PID_DIR_PROP,DEFAULT_PID_DIR,NIFI_STATUS_FILE_NAME); + File getStatusFile(final Logger logger) throws IOException { + return getBootstrapFile(logger, NIFI_PID_DIR_PROP, DEFAULT_PID_DIR, NIFI_STATUS_FILE_NAME); } - File getLockFile(final Logger logger) throws IOException{ - return getBootstrapFile(logger,NIFI_PID_DIR_PROP,DEFAULT_PID_DIR,NIFI_LOCK_FILE_NAME); + File getLockFile(final Logger logger) throws IOException { + return getBootstrapFile(logger, NIFI_PID_DIR_PROP, DEFAULT_PID_DIR, NIFI_LOCK_FILE_NAME); } - File getStatusFile() throws IOException{ + File getStatusFile() throws IOException { return getStatusFile(defaultLogger); } @@ -388,8 +387,8 @@ public class RunNiFi { return props; } - private synchronized void saveProperties(final Properties nifiProps, final Logger logger) throws IOException { - final String pid = nifiProps.getProperty(PID_KEY); + private synchronized void savePidProperties(final Properties pidProperties, final Logger logger) throws IOException { + final String pid = pidProperties.getProperty(PID_KEY); if (!StringUtils.isBlank(pid)) { writePidFile(pid, logger); } @@ -410,16 +409,16 @@ public class RunNiFi { Files.setPosixFilePermissions(statusFile.toPath(), perms); } catch (final Exception e) { logger.warn("Failed to set permissions so that only the owner can read status file {}; " - + "this may allows others to have access to the key needed to communicate with NiFi. " - + "Permissions should be changed so that only the owner can read this file", statusFile); + + "this may allows others to have access to the key needed to communicate with NiFi. " + + "Permissions should be changed so that only the owner can read this file", statusFile); } try (final FileOutputStream fos = new FileOutputStream(statusFile)) { - nifiProps.store(fos, null); + pidProperties.store(fos, null); fos.getFD().sync(); } - logger.debug("Saved Properties {} to {}", new Object[]{nifiProps, statusFile}); + logger.debug("Saved Properties {} to {}", new Object[]{pidProperties, statusFile}); } private synchronized void writePidFile(final String pid, final Logger logger) throws IOException { @@ -519,8 +518,8 @@ public class RunNiFi { boolean running = false; String line; try (final InputStream in = proc.getInputStream(); - final Reader streamReader = new InputStreamReader(in); - final BufferedReader reader = new BufferedReader(streamReader)) { + final Reader streamReader = new InputStreamReader(in); + final BufferedReader reader = new BufferedReader(streamReader)) { while ((line = reader.readLine()) != null) { if (line.trim().startsWith(pid)) { @@ -578,7 +577,7 @@ public class RunNiFi { return new Status(port, pid, true, true); } - final boolean alive = (pid == null) ? false : isProcessRunning(pid, logger); + final boolean alive = pid != null && isProcessRunning(pid, logger); return new Status(port, pid, pingSuccess, alive); } @@ -587,7 +586,7 @@ public class RunNiFi { final Status status = getStatus(logger); if (status.isRespondingToPing()) { logger.info("Apache NiFi is currently running, listening to Bootstrap on port {}, PID={}", - new Object[]{status.getPort(), status.getPid() == null ? "unknown" : status.getPid()}); + new Object[]{status.getPort(), status.getPid() == null ? "unknown" : status.getPid()}); return; } @@ -608,7 +607,7 @@ public class RunNiFi { } } - public void env(){ + public void env() { final Logger logger = cmdLogger; final Status status = getStatus(logger); if (status.getPid() == null) { @@ -641,19 +640,19 @@ public class RunNiFi { return; } - try{ + try { final Method getSystemPropertiesMethod = virtualMachine.getClass().getMethod("getSystemProperties"); - final Properties sysProps = (Properties)getSystemPropertiesMethod.invoke(virtualMachine); + final Properties sysProps = (Properties) getSystemPropertiesMethod.invoke(virtualMachine); for (Entry syspropEntry : sysProps.entrySet()) { - logger.info(syspropEntry.getKey().toString() + " = " +syspropEntry.getValue().toString()); + logger.info(syspropEntry.getKey().toString() + " = " + syspropEntry.getValue().toString()); } } catch (Throwable t) { throw new RuntimeException(t); } finally { try { detachMethod.invoke(virtualMachine); - } catch (final Exception e){ + } catch (final Exception e) { logger.warn("Caught exception detaching from process", e); } } @@ -721,7 +720,7 @@ public class RunNiFi { } serviceManager.notify(NotificationType.NIFI_STOPPED, "NiFi Stopped on Host " + hostname, - "Hello,\n\nApache NiFi has been told to initiate a shutdown on host " + hostname + " at " + now + " by user " + user); + "Hello,\n\nApache NiFi has been told to initiate a shutdown on host " + hostname + " at " + now + " by user " + user); } public void stop() throws IOException { @@ -824,9 +823,9 @@ public class RunNiFi { } catch (final IOException ioe) { if (pid == null) { logger.error("Failed to send shutdown command to port {} due to {}. No PID found for the NiFi process, so unable to kill process; " - + "the process should be killed manually.", new Object[] {port, ioe.toString()}); + + "the process should be killed manually.", new Object[]{port, ioe.toString()}); } else { - logger.error("Failed to send shutdown command to port {} due to {}. Will kill the NiFi Process with PID {}.", new Object[] {port, ioe.toString(), pid}); + logger.error("Failed to send shutdown command to port {} due to {}. Will kill the NiFi Process with PID {}.", port, ioe.toString(), pid); notifyStop(); killProcessTree(pid, logger); if (statusFile.exists() && !statusFile.delete()) { @@ -844,7 +843,7 @@ public class RunNiFi { final Process proc = Runtime.getRuntime().exec(new String[]{"ps", "-o", "pid", "--no-headers", "--ppid", ppid}); final List childPids = new ArrayList<>(); try (final InputStream in = proc.getInputStream(); - final BufferedReader reader = new BufferedReader(new InputStreamReader(in))) { + final BufferedReader reader = new BufferedReader(new InputStreamReader(in))) { String line; while ((line = reader.readLine()) != null) { @@ -900,7 +899,7 @@ public class RunNiFi { } final File prevLockFile = getLockFile(cmdLogger); - if (prevLockFile.exists() && !prevLockFile.delete()){ + if (prevLockFile.exists() && !prevLockFile.delete()) { cmdLogger.warn("Failed to delete previous lock file {}; this file should be cleaned up manually", prevLockFile); } @@ -931,7 +930,7 @@ public class RunNiFi { builder.directory(workingDir); } - final String nifiLogDir = replaceNull(System.getProperty("org.apache.nifi.bootstrap.config.log.dir"),DEFAULT_LOG_DIR).trim(); + final String nifiLogDir = replaceNull(System.getProperty("org.apache.nifi.bootstrap.config.log.dir"), DEFAULT_LOG_DIR).trim(); final String libFilename = replaceNull(props.get("lib.dir"), "./lib").trim(); File libDir = getFile(libFilename, workingDir); @@ -1001,7 +1000,7 @@ public class RunNiFi { if (javaHome != null) { String fileExtension = isWindows() ? ".exe" : ""; File javaFile = new File(javaHome + File.separatorChar + "bin" - + File.separatorChar + "java" + fileExtension); + + File.separatorChar + "java" + fileExtension); if (javaFile.exists() && javaFile.canExecute()) { javaCmd = javaFile.getAbsolutePath(); } @@ -1020,14 +1019,22 @@ public class RunNiFi { cmd.add("-Dnifi.properties.file.path=" + nifiPropsFilename); cmd.add("-Dnifi.bootstrap.listen.port=" + listenPort); cmd.add("-Dapp=NiFi"); - cmd.add("-Dorg.apache.nifi.bootstrap.config.log.dir="+nifiLogDir); + cmd.add("-Dorg.apache.nifi.bootstrap.config.log.dir=" + nifiLogDir); cmd.add("org.apache.nifi.NiFi"); + if (props.containsKey(NIFI_BOOTSTRAP_SENSITIVE_KEY) && props.get(NIFI_BOOTSTRAP_SENSITIVE_KEY) != null) { + cmd.add("-k " + props.get(NIFI_BOOTSTRAP_SENSITIVE_KEY)); + } builder.command(cmd); final StringBuilder cmdBuilder = new StringBuilder(); for (final String s : cmd) { - cmdBuilder.append(s).append(" "); + // Mask the key + if (s.startsWith("-k ")) { + cmdBuilder.append("-k ****"); + } else { + cmdBuilder.append(s).append(" "); + } } cmdLogger.info("Starting Apache NiFi..."); @@ -1044,12 +1051,12 @@ public class RunNiFi { gracefulShutdownSeconds = Integer.parseInt(gracefulShutdown); } catch (final NumberFormatException nfe) { throw new NumberFormatException("The '" + GRACEFUL_SHUTDOWN_PROP + "' property in Bootstrap Config File " - + bootstrapConfigAbsoluteFile.getAbsolutePath() + " has an invalid value. Must be a non-negative integer"); + + bootstrapConfigAbsoluteFile.getAbsolutePath() + " has an invalid value. Must be a non-negative integer"); } if (gracefulShutdownSeconds < 0) { throw new NumberFormatException("The '" + GRACEFUL_SHUTDOWN_PROP + "' property in Bootstrap Config File " - + bootstrapConfigAbsoluteFile.getAbsolutePath() + " has an invalid value. Must be a non-negative integer"); + + bootstrapConfigAbsoluteFile.getAbsolutePath() + " has an invalid value. Must be a non-negative integer"); } Process process = builder.start(); @@ -1057,9 +1064,9 @@ public class RunNiFi { Long pid = getPid(process, cmdLogger); if (pid != null) { nifiPid = pid; - final Properties nifiProps = new Properties(); - nifiProps.setProperty(PID_KEY, String.valueOf(nifiPid)); - saveProperties(nifiProps, cmdLogger); + final Properties pidProperties = new Properties(); + pidProperties.setProperty(PID_KEY, String.valueOf(nifiPid)); + savePidProperties(pidProperties, cmdLogger); } shutdownHook = new ShutdownHook(process, this, secretKey, gracefulShutdownSeconds, loggingExecutor); @@ -1098,7 +1105,7 @@ public class RunNiFi { return; } - final File lockFile = getLockFile(defaultLogger); + final File lockFile = getLockFile(defaultLogger); if (lockFile.exists()) { defaultLogger.info("A shutdown was initiated. Will not restart NiFi"); return; @@ -1119,9 +1126,9 @@ public class RunNiFi { pid = getPid(process, defaultLogger); if (pid != null) { nifiPid = pid; - final Properties nifiProps = new Properties(); - nifiProps.setProperty(PID_KEY, String.valueOf(nifiPid)); - saveProperties(nifiProps, defaultLogger); + final Properties pidProperties = new Properties(); + pidProperties.setProperty(PID_KEY, String.valueOf(nifiPid)); + savePidProperties(pidProperties, defaultLogger); } shutdownHook = new ShutdownHook(process, this, secretKey, gracefulShutdownSeconds, loggingExecutor); @@ -1134,14 +1141,14 @@ public class RunNiFi { // We are expected to restart nifi, so send a notification that it died. If we are not restarting nifi, // then this means that we are intentionally stopping the service. serviceManager.notify(NotificationType.NIFI_DIED, "NiFi Died on Host " + hostname, - "Hello,\n\nIt appears that Apache NiFi has died on host " + hostname + " at " + now + "; automatically restarting NiFi"); + "Hello,\n\nIt appears that Apache NiFi has died on host " + hostname + " at " + now + "; automatically restarting NiFi"); } else { defaultLogger.error("Apache NiFi does not appear to have started"); // We are expected to restart nifi, so send a notification that it died. If we are not restarting nifi, // then this means that we are intentionally stopping the service. serviceManager.notify(NotificationType.NIFI_DIED, "NiFi Died on Host " + hostname, - "Hello,\n\nIt appears that Apache NiFi has died on host " + hostname + " at " + now + - ". Attempted to restart NiFi but the services does not appear to have restarted!"); + "Hello,\n\nIt appears that Apache NiFi has died on host " + hostname + " at " + now + + ". Attempted to restart NiFi but the services does not appear to have restarted!"); } } else { return; @@ -1261,7 +1268,7 @@ public class RunNiFi { this.autoRestartNiFi = restart; } - void setNiFiCommandControlPort(final int port, final String secretKey) throws IOException{ + void setNiFiCommandControlPort(final int port, final String secretKey) throws IOException { this.ccPort = port; this.secretKey = secretKey; @@ -1279,7 +1286,7 @@ public class RunNiFi { nifiProps.setProperty("secret.key", secretKey); try { - saveProperties(nifiProps, defaultLogger); + savePidProperties(nifiProps, defaultLogger); } catch (final IOException ioe) { defaultLogger.warn("Apache NiFi has started but failed to persist NiFi Port information to {} due to {}", new Object[]{statusFile.getAbsolutePath(), ioe}); } diff --git a/nifi-commons/nifi-properties/src/main/java/org/apache/nifi/util/NiFiProperties.java b/nifi-commons/nifi-properties/src/main/java/org/apache/nifi/util/NiFiProperties.java index 4a7e5d8c57..619c104a22 100644 --- a/nifi-commons/nifi-properties/src/main/java/org/apache/nifi/util/NiFiProperties.java +++ b/nifi-commons/nifi-properties/src/main/java/org/apache/nifi/util/NiFiProperties.java @@ -38,7 +38,7 @@ import java.util.Set; * values to be available at runtime. It is strongly tied to the startup * properties needed and is often refer to as the 'nifi.properties' file. The * properties contains keys and values. Great care should be taken in leveraging - * this class or passing it along. It's use should be refactored and minimized + * this class or passing it along. Its use should be refactored and minimized * over time. */ public abstract class NiFiProperties { @@ -247,9 +247,9 @@ public abstract class NiFiProperties { public static final String DEFAULT_KERBEROS_AUTHENTICATION_EXPIRATION = "12 hours"; /** - * Retrieves the property value for the given property key + * Retrieves the property value for the given property key. * - * @param key the key of property value to lookup. + * @param key the key of property value to lookup * @return value of property at given key or null if not found */ public abstract String getProperty(String key); @@ -257,7 +257,7 @@ public abstract class NiFiProperties { /** * Retrieves all known property keys. * - * @return all known property keys. + * @return all known property keys */ public abstract Set getPropertyKeys(); @@ -375,11 +375,7 @@ public abstract class NiFiProperties { public Boolean isSiteToSiteSecure() { final String secureVal = getProperty(SITE_TO_SITE_SECURE, "true"); - if ("false".equalsIgnoreCase(secureVal)) { - return false; - } else { - return true; - } + return !"false".equalsIgnoreCase(secureVal); } @@ -389,11 +385,7 @@ public abstract class NiFiProperties { public Boolean isSiteToSiteHttpEnabled() { final String remoteInputHttpEnabled = getProperty(SITE_TO_SITE_HTTP_ENABLED, "false"); - if ("true".equalsIgnoreCase(remoteInputHttpEnabled)) { - return true; - } else { - return false; - } + return "true".equalsIgnoreCase(remoteInputHttpEnabled); } @@ -769,8 +761,8 @@ public abstract class NiFiProperties { * Returns true if client certificates are required for REST API. Determined * if the following conditions are all true: * - * - login identity provider is not populated - Kerberos service support is - * not enabled + * - login identity provider is not populated + * - Kerberos service support is not enabled * * @return true if client certificates are required for access to the REST * API diff --git a/nifi-commons/nifi-properties/src/main/java/org/apache/nifi/util/StringUtils.java b/nifi-commons/nifi-properties/src/main/java/org/apache/nifi/util/StringUtils.java index aa6f8f305f..37408f7a36 100644 --- a/nifi-commons/nifi-properties/src/main/java/org/apache/nifi/util/StringUtils.java +++ b/nifi-commons/nifi-properties/src/main/java/org/apache/nifi/util/StringUtils.java @@ -16,6 +16,8 @@ */ package org.apache.nifi.util; +import java.util.Collection; + /** * String Utils based on the Apache Commons Lang String Utils. * These simple util methods here allow us to avoid a dependency in the core @@ -63,4 +65,38 @@ public class StringUtils { } return str.substring(pos + separator.length()); } + + public static String join(final Collection collection, String delimiter) { + if (collection == null || collection.size() == 0) { + return EMPTY; + } + final StringBuilder sb = new StringBuilder(collection.size() * 16); + for (Object element : collection) { + sb.append((String) element); + sb.append(delimiter); + } + return sb.toString().substring(0, sb.lastIndexOf(delimiter)); + } + + public static String padLeft(final String source, int length, char padding) { + if (source != null) { + StringBuilder sb = new StringBuilder(source).reverse(); + while (sb.length() < length) { + sb.append(padding); + } + return sb.reverse().toString(); + } + return null; + } + + public static String padRight(final String source, int length, char padding) { + if (source != null) { + StringBuilder sb = new StringBuilder(source); + while (sb.length() < length) { + sb.append(padding); + } + return sb.toString(); + } + return null; + } } diff --git a/nifi-commons/nifi-properties/src/test/java/org/apache/nifi/util/NiFiPropertiesTest.java b/nifi-commons/nifi-properties/src/test/java/org/apache/nifi/util/NiFiPropertiesTest.java index 3547b92822..e174d140e6 100644 --- a/nifi-commons/nifi-properties/src/test/java/org/apache/nifi/util/NiFiPropertiesTest.java +++ b/nifi-commons/nifi-properties/src/test/java/org/apache/nifi/util/NiFiPropertiesTest.java @@ -80,11 +80,11 @@ public class NiFiPropertiesTest { } - private NiFiProperties loadNiFiProperties(final String propsPath) { + private NiFiProperties loadNiFiProperties(final String propsPath){ String realPath = null; - try { + try{ realPath = NiFiPropertiesTest.class.getResource(propsPath).toURI().getPath(); - } catch (final URISyntaxException ex) { + }catch(final URISyntaxException ex){ throw new RuntimeException(ex); } return NiFiProperties.createBasicNiFiProperties(realPath, null); diff --git a/nifi-framework-api/src/main/java/org/apache/nifi/controller/queue/SortColumn.java b/nifi-framework-api/src/main/java/org/apache/nifi/controller/queue/SortColumn.java index 30d285cf79..9f03769726 100644 --- a/nifi-framework-api/src/main/java/org/apache/nifi/controller/queue/SortColumn.java +++ b/nifi-framework-api/src/main/java/org/apache/nifi/controller/queue/SortColumn.java @@ -27,7 +27,7 @@ public enum SortColumn implements Comparator { /** * Sort based on the current position in the queue */ - QUEUE_POSITION (new Comparator() { + QUEUE_POSITION(new Comparator() { @Override public int compare(final FlowFileSummary o1, final FlowFileSummary o2) { return Integer.compare(o1.getPosition(), o2.getPosition()); @@ -37,7 +37,7 @@ public enum SortColumn implements Comparator { /** * Sort based on the UUID of the FlowFile */ - FLOWFILE_UUID (new Comparator() { + FLOWFILE_UUID(new Comparator() { @Override public int compare(final FlowFileSummary o1, final FlowFileSummary o2) { return o1.getUuid().compareTo(o2.getUuid()); @@ -47,7 +47,7 @@ public enum SortColumn implements Comparator { /** * Sort based on the 'filename' attribute of the FlowFile */ - FILENAME (new Comparator() { + FILENAME(new Comparator() { @Override public int compare(final FlowFileSummary o1, final FlowFileSummary o2) { return o1.getFilename().compareTo(o2.getFilename()); @@ -67,7 +67,7 @@ public enum SortColumn implements Comparator { /** * Sort based on how long the FlowFile has been sitting in the queue */ - QUEUED_DURATION (new Comparator() { + QUEUED_DURATION(new Comparator() { @Override public int compare(final FlowFileSummary o1, final FlowFileSummary o2) { return -Long.compare(o1.getLastQueuedTime(), o2.getLastQueuedTime()); @@ -78,7 +78,7 @@ public enum SortColumn implements Comparator { * Sort based on the age of the FlowFile. I.e., the time at which the FlowFile's * "greatest ancestor" entered the flow */ - FLOWFILE_AGE (new Comparator() { + FLOWFILE_AGE(new Comparator() { @Override public int compare(final FlowFileSummary o1, final FlowFileSummary o2) { return Long.compare(o1.getLineageStartDate(), o2.getLineageStartDate()); @@ -88,7 +88,7 @@ public enum SortColumn implements Comparator { /** * Sort based on when the FlowFile's penalization ends */ - PENALIZATION (new Comparator() { + PENALIZATION(new Comparator() { @Override public int compare(final FlowFileSummary o1, final FlowFileSummary o2) { return Boolean.compare(o1.isPenalized(), o2.isPenalized()); diff --git a/nifi-nar-bundles/nifi-amqp-bundle/nifi-amqp-processors/src/main/java/org/apache/nifi/amqp/processors/AMQPUtils.java b/nifi-nar-bundles/nifi-amqp-bundle/nifi-amqp-processors/src/main/java/org/apache/nifi/amqp/processors/AMQPUtils.java index 6cfa2c7176..68302a2ed3 100644 --- a/nifi-nar-bundles/nifi-amqp-bundle/nifi-amqp-processors/src/main/java/org/apache/nifi/amqp/processors/AMQPUtils.java +++ b/nifi-nar-bundles/nifi-amqp-bundle/nifi-amqp-processors/src/main/java/org/apache/nifi/amqp/processors/AMQPUtils.java @@ -32,7 +32,6 @@ import com.rabbitmq.client.AMQP.BasicProperties; /** * Utility helper class simplify interactions with target AMQP API and NIFI API. - * */ abstract class AMQPUtils { @@ -43,20 +42,20 @@ abstract class AMQPUtils { private final static Logger logger = LoggerFactory.getLogger(AMQPUtils.class); public enum PropertyNames { - CONTENT_TYPE (AMQP_PROP_PREFIX + "contentType"), - CONTENT_ENCODING (AMQP_PROP_PREFIX + "contentEncoding"), - HEADERS (AMQP_PROP_PREFIX + "headers"), - DELIVERY_MODE (AMQP_PROP_PREFIX + "deliveryMode"), - PRIORITY (AMQP_PROP_PREFIX + "priority"), - CORRELATION_ID (AMQP_PROP_PREFIX + "correlationId"), - REPLY_TO (AMQP_PROP_PREFIX + "replyTo"), - EXPIRATION (AMQP_PROP_PREFIX + "expiration"), - MESSAGE_ID (AMQP_PROP_PREFIX + "messageId"), - TIMESTAMP (AMQP_PROP_PREFIX + "timestamp"), - TYPE (AMQP_PROP_PREFIX + "type"), - USER_ID (AMQP_PROP_PREFIX + "userId"), - APP_ID (AMQP_PROP_PREFIX + "appId"), - CLUSTER_ID (AMQP_PROP_PREFIX + "clusterId"); + CONTENT_TYPE(AMQP_PROP_PREFIX + "contentType"), + CONTENT_ENCODING(AMQP_PROP_PREFIX + "contentEncoding"), + HEADERS(AMQP_PROP_PREFIX + "headers"), + DELIVERY_MODE(AMQP_PROP_PREFIX + "deliveryMode"), + PRIORITY(AMQP_PROP_PREFIX + "priority"), + CORRELATION_ID(AMQP_PROP_PREFIX + "correlationId"), + REPLY_TO(AMQP_PROP_PREFIX + "replyTo"), + EXPIRATION(AMQP_PROP_PREFIX + "expiration"), + MESSAGE_ID(AMQP_PROP_PREFIX + "messageId"), + TIMESTAMP(AMQP_PROP_PREFIX + "timestamp"), + TYPE(AMQP_PROP_PREFIX + "type"), + USER_ID(AMQP_PROP_PREFIX + "userId"), + APP_ID(AMQP_PROP_PREFIX + "appId"), + CLUSTER_ID(AMQP_PROP_PREFIX + "clusterId"); PropertyNames(String value) { this.value = value; @@ -71,7 +70,7 @@ abstract class AMQPUtils { } static { - for(PropertyNames propertyNames : PropertyNames.values()) { + for (PropertyNames propertyNames : PropertyNames.values()) { lookup.put(propertyNames.getValue(), propertyNames); } } @@ -89,15 +88,12 @@ abstract class AMQPUtils { /** * Updates {@link FlowFile} with attributes representing AMQP properties * - * @param amqpProperties - * instance of {@link BasicProperties} - * @param flowFile - * instance of target {@link FlowFile} - * @param processSession - * instance of {@link ProcessSession} + * @param amqpProperties instance of {@link BasicProperties} + * @param flowFile instance of target {@link FlowFile} + * @param processSession instance of {@link ProcessSession} */ public static FlowFile updateFlowFileAttributesWithAmqpProperties(BasicProperties amqpProperties, FlowFile flowFile, ProcessSession processSession) { - if (amqpProperties != null){ + if (amqpProperties != null) { try { Method[] methods = BasicProperties.class.getDeclaredMethods(); Map attributes = new HashMap<>(); @@ -126,8 +122,7 @@ abstract class AMQPUtils { /** * Will validate if provided name corresponds to valid AMQP property. * - * @param name - * the name of the property + * @param name the name of the property * @return 'true' if valid otherwise 'false' */ public static boolean isValidAmqpPropertyName(String name) { @@ -147,11 +142,10 @@ abstract class AMQPUtils { * Will validate if provided amqpPropValue can be converted to a {@link Map}. * Should be passed in the format: amqp$headers=key=value,key=value etc. * - * @param amqpPropValue - * the value of the property + * @param amqpPropValue the value of the property * @return {@link Map} if valid otherwise null */ - public static Map validateAMQPHeaderProperty(String amqpPropValue){ + public static Map validateAMQPHeaderProperty(String amqpPropValue) { String[] strEntries = amqpPropValue.split(","); Map headers = new HashMap<>(); for (String strEntry : strEntries) { @@ -170,11 +164,10 @@ abstract class AMQPUtils { * Will validate if provided amqpPropValue can be converted to an {@link Integer}, and that its * value is 1 or 2. * - * @param amqpPropValue - * the value of the property + * @param amqpPropValue the value of the property * @return {@link Integer} if valid otherwise null */ - public static Integer validateAMQPDeliveryModeProperty(String amqpPropValue){ + public static Integer validateAMQPDeliveryModeProperty(String amqpPropValue) { Integer deliveryMode = toInt(amqpPropValue); if (deliveryMode == null || !(deliveryMode == 1 || deliveryMode == 2)) { @@ -187,14 +180,13 @@ abstract class AMQPUtils { * Will validate if provided amqpPropValue can be converted to an {@link Integer} and that its * value is between 0 and 9 (inclusive). * - * @param amqpPropValue - * the value of the property + * @param amqpPropValue the value of the property * @return {@link Integer} if valid otherwise null */ - public static Integer validateAMQPPriorityProperty(String amqpPropValue){ + public static Integer validateAMQPPriorityProperty(String amqpPropValue) { Integer priority = toInt(amqpPropValue); - if (priority == null || !(priority >= 0 && priority <= 9)){ + if (priority == null || !(priority >= 0 && priority <= 9)) { logger.warn("Invalid value for AMQP priority property: " + amqpPropValue); } return priority; @@ -203,14 +195,13 @@ abstract class AMQPUtils { /** * Will validate if provided amqpPropValue can be converted to a {@link Date}. * - * @param amqpPropValue - * the value of the property + * @param amqpPropValue the value of the property * @return {@link Date} if valid otherwise null */ - public static Date validateAMQPTimestampProperty(String amqpPropValue){ + public static Date validateAMQPTimestampProperty(String amqpPropValue) { Long timestamp = toLong(amqpPropValue); - if (timestamp == null){ + if (timestamp == null) { logger.warn("Invalid value for AMQP timestamp property: " + amqpPropValue); return null; } @@ -222,14 +213,13 @@ abstract class AMQPUtils { /** * Takes a {@link String} and tries to convert to an {@link Integer}. * - * @param strVal - * the value to be converted + * @param strVal the value to be converted * @return {@link Integer} if valid otherwise null */ - private static Integer toInt(String strVal){ + private static Integer toInt(String strVal) { try { return Integer.parseInt(strVal); - } catch (NumberFormatException aE){ + } catch (NumberFormatException aE) { return null; } } @@ -237,14 +227,13 @@ abstract class AMQPUtils { /** * Takes a {@link String} and tries to convert to a {@link Long}. * - * @param strVal - * the value to be converted + * @param strVal the value to be converted * @return {@link Long} if valid otherwise null */ - private static Long toLong(String strVal){ + private static Long toLong(String strVal) { try { return Long.parseLong(strVal); - } catch (NumberFormatException aE){ + } catch (NumberFormatException aE) { return null; } } diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-cluster/src/main/java/org/apache/nifi/cluster/coordination/heartbeat/ClusterProtocolHeartbeatMonitor.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-cluster/src/main/java/org/apache/nifi/cluster/coordination/heartbeat/ClusterProtocolHeartbeatMonitor.java index 20fbfd211f..716610c93c 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-cluster/src/main/java/org/apache/nifi/cluster/coordination/heartbeat/ClusterProtocolHeartbeatMonitor.java +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-cluster/src/main/java/org/apache/nifi/cluster/coordination/heartbeat/ClusterProtocolHeartbeatMonitor.java @@ -25,10 +25,8 @@ import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; import java.util.function.Function; import java.util.stream.Collectors; - import javax.xml.bind.JAXBContext; import javax.xml.bind.Unmarshaller; - import org.apache.nifi.cluster.coordination.ClusterCoordinator; import org.apache.nifi.cluster.coordination.node.NodeConnectionState; import org.apache.nifi.cluster.coordination.node.NodeConnectionStatus; diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-cluster/src/main/java/org/apache/nifi/cluster/coordination/http/endpoints/StatusHistoryEndpointMerger.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-cluster/src/main/java/org/apache/nifi/cluster/coordination/http/endpoints/StatusHistoryEndpointMerger.java index ddd8759fa1..60b40f7ac7 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-cluster/src/main/java/org/apache/nifi/cluster/coordination/http/endpoints/StatusHistoryEndpointMerger.java +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-cluster/src/main/java/org/apache/nifi/cluster/coordination/http/endpoints/StatusHistoryEndpointMerger.java @@ -26,7 +26,6 @@ import java.util.Map; import java.util.Set; import java.util.TreeMap; import java.util.regex.Pattern; - import org.apache.nifi.cluster.coordination.http.EndpointResponseMerger; import org.apache.nifi.cluster.manager.NodeResponse; import org.apache.nifi.cluster.protocol.NodeIdentifier; diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-cluster/src/main/java/org/apache/nifi/cluster/coordination/node/CuratorNodeProtocolSender.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-cluster/src/main/java/org/apache/nifi/cluster/coordination/node/CuratorNodeProtocolSender.java index 18474615a5..08069597f5 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-cluster/src/main/java/org/apache/nifi/cluster/coordination/node/CuratorNodeProtocolSender.java +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-cluster/src/main/java/org/apache/nifi/cluster/coordination/node/CuratorNodeProtocolSender.java @@ -19,7 +19,6 @@ package org.apache.nifi.cluster.coordination.node; import java.io.IOException; import java.net.InetSocketAddress; import java.nio.charset.StandardCharsets; - import org.apache.curator.RetryPolicy; import org.apache.curator.framework.CuratorFramework; import org.apache.curator.framework.CuratorFrameworkFactory; diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-cluster/src/main/java/org/apache/nifi/cluster/coordination/node/NodeClusterCoordinator.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-cluster/src/main/java/org/apache/nifi/cluster/coordination/node/NodeClusterCoordinator.java index 2d25c8ef7b..b1a088edba 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-cluster/src/main/java/org/apache/nifi/cluster/coordination/node/NodeClusterCoordinator.java +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-cluster/src/main/java/org/apache/nifi/cluster/coordination/node/NodeClusterCoordinator.java @@ -31,7 +31,6 @@ import java.util.concurrent.atomic.AtomicLong; import java.util.function.Supplier; import java.util.regex.Pattern; import java.util.stream.Collectors; - import org.apache.commons.collections4.queue.CircularFifoQueue; import org.apache.commons.lang3.StringUtils; import org.apache.nifi.cluster.coordination.ClusterCoordinator; diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-cluster/src/test/java/org/apache/nifi/cluster/coordination/http/replication/TestThreadPoolRequestReplicator.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-cluster/src/test/java/org/apache/nifi/cluster/coordination/http/replication/TestThreadPoolRequestReplicator.java index 9c56fb9599..e699fcac1a 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-cluster/src/test/java/org/apache/nifi/cluster/coordination/http/replication/TestThreadPoolRequestReplicator.java +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-cluster/src/test/java/org/apache/nifi/cluster/coordination/http/replication/TestThreadPoolRequestReplicator.java @@ -22,6 +22,13 @@ import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertNull; import static org.junit.Assert.assertTrue; +import com.sun.jersey.api.client.Client; +import com.sun.jersey.api.client.ClientHandlerException; +import com.sun.jersey.api.client.ClientResponse; +import com.sun.jersey.api.client.ClientResponse.Status; +import com.sun.jersey.api.client.WebResource; +import com.sun.jersey.core.header.InBoundHeaders; +import com.sun.jersey.core.header.OutBoundHeaders; import java.io.ByteArrayInputStream; import java.net.SocketTimeoutException; import java.net.URI; @@ -34,9 +41,7 @@ import java.util.Map; import java.util.Set; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicInteger; - import javax.ws.rs.HttpMethod; - import org.apache.commons.collections4.map.MultiValueMap; import org.apache.nifi.cluster.coordination.ClusterCoordinator; import org.apache.nifi.cluster.coordination.node.NodeConnectionState; @@ -57,14 +62,6 @@ import org.mockito.internal.util.reflection.Whitebox; import org.mockito.invocation.InvocationOnMock; import org.mockito.stubbing.Answer; -import com.sun.jersey.api.client.Client; -import com.sun.jersey.api.client.ClientHandlerException; -import com.sun.jersey.api.client.ClientResponse; -import com.sun.jersey.api.client.ClientResponse.Status; -import com.sun.jersey.api.client.WebResource; -import com.sun.jersey.core.header.InBoundHeaders; -import com.sun.jersey.core.header.OutBoundHeaders; - public class TestThreadPoolRequestReplicator { @BeforeClass diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-cluster/src/test/java/org/apache/nifi/cluster/coordination/node/TestNodeClusterCoordinator.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-cluster/src/test/java/org/apache/nifi/cluster/coordination/node/TestNodeClusterCoordinator.java index 00edbc40f7..e55605bcd9 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-cluster/src/test/java/org/apache/nifi/cluster/coordination/node/TestNodeClusterCoordinator.java +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-cluster/src/test/java/org/apache/nifi/cluster/coordination/node/TestNodeClusterCoordinator.java @@ -32,7 +32,6 @@ import java.util.Map; import java.util.Set; import java.util.concurrent.atomic.AtomicReference; import java.util.stream.Collectors; - import org.apache.nifi.cluster.manager.exception.IllegalNodeDisconnectionException; import org.apache.nifi.cluster.protocol.ConnectionRequest; import org.apache.nifi.cluster.protocol.ConnectionResponse; diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-cluster/src/test/java/org/apache/nifi/cluster/integration/Node.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-cluster/src/test/java/org/apache/nifi/cluster/integration/Node.java index a8b51f2e1d..b9372a67e4 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-cluster/src/test/java/org/apache/nifi/cluster/integration/Node.java +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-cluster/src/test/java/org/apache/nifi/cluster/integration/Node.java @@ -28,7 +28,6 @@ import java.util.Set; import java.util.UUID; import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.TimeUnit; - import org.apache.commons.lang3.builder.HashCodeBuilder; import org.apache.nifi.authorization.Authorizer; import org.apache.nifi.cluster.ReportedEvent; @@ -65,7 +64,6 @@ import org.apache.nifi.registry.VariableRegistry; import org.apache.nifi.reporting.BulletinRepository; import org.apache.nifi.reporting.Severity; import org.apache.nifi.util.NiFiProperties; -import org.apache.nifi.web.Revision; import org.apache.nifi.web.revision.RevisionManager; import org.junit.Assert; import org.mockito.Mockito; @@ -117,7 +115,7 @@ public class Node { }; revisionManager = Mockito.mock(RevisionManager.class); - Mockito.when(revisionManager.getAllRevisions()).thenReturn(Collections. emptyList()); + Mockito.when(revisionManager.getAllRevisions()).thenReturn(Collections.emptyList()); electionManager = new CuratorLeaderElectionManager(4, nodeProperties); } diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/pom.xml b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/pom.xml index f46792fac9..094490b11b 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/pom.xml +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/pom.xml @@ -28,6 +28,10 @@ org.apache.nifi nifi-framework-core-api + + org.apache.nifi + nifi-properties-loader + org.apache.nifi nifi-api diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/connectable/StandardConnection.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/connectable/StandardConnection.java index 2a0f0dec10..6d2228128c 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/connectable/StandardConnection.java +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/connectable/StandardConnection.java @@ -16,6 +16,17 @@ */ package org.apache.nifi.connectable; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.UUID; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicLong; +import java.util.concurrent.atomic.AtomicReference; +import java.util.stream.Collectors; import org.apache.commons.collections4.CollectionUtils; import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.builder.EqualsBuilder; @@ -41,18 +52,6 @@ import org.apache.nifi.processor.FlowFileFilter; import org.apache.nifi.processor.Relationship; import org.apache.nifi.provenance.ProvenanceEventRepository; -import java.util.ArrayList; -import java.util.Collection; -import java.util.Collections; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.UUID; -import java.util.concurrent.atomic.AtomicInteger; -import java.util.concurrent.atomic.AtomicLong; -import java.util.concurrent.atomic.AtomicReference; -import java.util.stream.Collectors; - /** * Models a connection between connectable components. A connection may contain * one or more relationships that map the source component to the destination diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/FileSystemSwapManager.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/FileSystemSwapManager.java index a61d7fe967..3c4610fa2f 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/FileSystemSwapManager.java +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/FileSystemSwapManager.java @@ -37,7 +37,6 @@ import java.util.List; import java.util.Map; import java.util.UUID; import java.util.regex.Pattern; - import org.apache.nifi.controller.queue.FlowFileQueue; import org.apache.nifi.controller.queue.QueueSize; import org.apache.nifi.controller.repository.FlowFileRecord; @@ -351,7 +350,7 @@ public class FileSystemSwapManager implements FlowFileSwapManager { out.flush(); } - logger.info("Successfully swapped out {} FlowFiles from {} to Swap File {}", new Object[]{toSwap.size(), queue, swapLocation}); + logger.info("Successfully swapped out {} FlowFiles from {} to Swap File {}", toSwap.size(), queue, swapLocation); return toSwap.size(); } @@ -399,8 +398,8 @@ public class FileSystemSwapManager implements FlowFileSwapManager { } } catch (final EOFException eof) { final QueueSize queueSize = new QueueSize(numRecords, contentSize); - final SwapSummary summary = new StandardSwapSummary(queueSize, maxRecordId, Collections.emptyList()); - final SwapContents partialContents = new StandardSwapContents(summary, Collections.emptyList()); + final SwapSummary summary = new StandardSwapSummary(queueSize, maxRecordId, Collections.emptyList()); + final SwapContents partialContents = new StandardSwapContents(summary, Collections.emptyList()); throw new IncompleteSwapFileException(swapLocation, partialContents); } diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/cluster/ClusterProtocolHeartbeater.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/cluster/ClusterProtocolHeartbeater.java index efc05889d5..e2e7f9c5b4 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/cluster/ClusterProtocolHeartbeater.java +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/cluster/ClusterProtocolHeartbeater.java @@ -17,11 +17,9 @@ package org.apache.nifi.controller.cluster; import java.io.IOException; - import java.util.List; import java.util.Map; import java.util.stream.Collectors; - import org.apache.nifi.cluster.coordination.ClusterCoordinator; import org.apache.nifi.cluster.coordination.node.ClusterRoles; import org.apache.nifi.cluster.coordination.node.NodeConnectionStatus; diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/leader/election/CuratorLeaderElectionManager.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/leader/election/CuratorLeaderElectionManager.java index 977580e452..dfe456b30c 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/leader/election/CuratorLeaderElectionManager.java +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/leader/election/CuratorLeaderElectionManager.java @@ -18,7 +18,6 @@ package org.apache.nifi.controller.leader.election; import java.util.HashMap; import java.util.Map; - import org.apache.commons.lang3.StringUtils; import org.apache.curator.RetryPolicy; import org.apache.curator.framework.CuratorFramework; diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/encrypt/StringEncryptor.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/encrypt/StringEncryptor.java index 15043c3544..dc0e7c7bec 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/encrypt/StringEncryptor.java +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/encrypt/StringEncryptor.java @@ -17,8 +17,8 @@ package org.apache.nifi.encrypt; import java.security.Security; -import org.apache.nifi.util.NiFiProperties; import org.apache.commons.lang3.StringUtils; +import org.apache.nifi.util.NiFiProperties; import org.bouncycastle.jce.provider.BouncyCastleProvider; import org.jasypt.encryption.pbe.StandardPBEStringEncryptor; import org.jasypt.exceptions.EncryptionInitializationException; diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/remote/StandardRemoteProcessGroup.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/remote/StandardRemoteProcessGroup.java index 5fd0f1a8e4..989e8dd0b7 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/remote/StandardRemoteProcessGroup.java +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/remote/StandardRemoteProcessGroup.java @@ -16,10 +16,34 @@ */ package org.apache.nifi.remote; +import static java.util.Objects.requireNonNull; + import com.sun.jersey.api.client.ClientHandlerException; import com.sun.jersey.api.client.ClientResponse; import com.sun.jersey.api.client.ClientResponse.Status; import com.sun.jersey.api.client.UniformInterfaceException; +import java.io.File; +import java.io.IOException; +import java.net.URI; +import java.net.URISyntaxException; +import java.util.ArrayList; +import java.util.Date; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Iterator; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicReference; +import java.util.concurrent.locks.Lock; +import java.util.concurrent.locks.ReadWriteLock; +import java.util.concurrent.locks.ReentrantReadWriteLock; +import javax.net.ssl.SSLContext; +import javax.ws.rs.core.Response; import org.apache.nifi.authorization.Resource; import org.apache.nifi.authorization.resource.Authorizable; import org.apache.nifi.authorization.resource.ResourceFactory; @@ -52,31 +76,6 @@ import org.apache.nifi.web.api.dto.PortDTO; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import javax.net.ssl.SSLContext; -import javax.ws.rs.core.Response; -import java.io.File; -import java.io.IOException; -import java.net.URI; -import java.net.URISyntaxException; -import java.util.ArrayList; -import java.util.Date; -import java.util.HashMap; -import java.util.HashSet; -import java.util.Iterator; -import java.util.LinkedHashSet; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.concurrent.ScheduledExecutorService; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicBoolean; -import java.util.concurrent.atomic.AtomicReference; -import java.util.concurrent.locks.Lock; -import java.util.concurrent.locks.ReadWriteLock; -import java.util.concurrent.locks.ReentrantReadWriteLock; - -import static java.util.Objects.requireNonNull; - /** * Represents the Root Process Group of a remote NiFi Instance. Holds * information about that remote instance, as well as {@link IncomingPort}s and @@ -837,7 +836,7 @@ public class StandardRemoteProcessGroup implements RemoteProcessGroup { // perform the request final ControllerDTO dto; try ( - final SiteToSiteRestApiClient apiClient = getSiteToSiteRestApiClient();) { + final SiteToSiteRestApiClient apiClient = getSiteToSiteRestApiClient()) { dto = apiClient.getController(); } catch (IOException e) { writeLock.lock(); @@ -1202,8 +1201,8 @@ public class StandardRemoteProcessGroup implements RemoteProcessGroup { if (Response.Status.Family.SUCCESSFUL.equals(requestAccountResponse.getStatusInfo().getFamily())) { logger.info("{} Issued a Request to communicate with remote instance", this); } else { - logger.error("{} Failed to request account: got unexpected response code of {}:{}", new Object[]{ - this, requestAccountResponse.getStatus(), requestAccountResponse.getStatusInfo().getReasonPhrase()}); + logger.error("{} Failed to request account: got unexpected response code of {}:{}", this, + requestAccountResponse.getStatus(), requestAccountResponse.getStatusInfo().getReasonPhrase()); } } catch (final Exception re) { logger.error("{} Failed to request account due to {}", this, re.toString()); diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/resources/nifi-context.xml b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/resources/nifi-context.xml index 7ca0130a32..bf3fd2ae17 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/resources/nifi-context.xml +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/resources/nifi-context.xml @@ -16,18 +16,10 @@ + xsi:schemaLocation="http://www.springframework.org/schema/beans http://www.springframework.org/schema/beans/spring-beans-3.1.xsd"> - - - + diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/controller/TestFlowController.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/controller/TestFlowController.java index 602ddce7d6..bbcdc3b7b8 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/controller/TestFlowController.java +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/controller/TestFlowController.java @@ -16,6 +16,21 @@ */ package org.apache.nifi.controller; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotEquals; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertTrue; +import static org.mockito.Mockito.when; + +import java.io.File; +import java.io.FileInputStream; +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.util.HashMap; +import java.util.LinkedHashSet; +import java.util.Map; +import java.util.Set; import org.apache.commons.io.IOUtils; import org.apache.nifi.admin.service.AuditService; import org.apache.nifi.authorization.AbstractPolicyBasedAuthorizer; @@ -35,29 +50,13 @@ import org.apache.nifi.processor.Relationship; import org.apache.nifi.provenance.MockProvenanceRepository; import org.apache.nifi.registry.VariableRegistry; import org.apache.nifi.reporting.BulletinRepository; +import org.apache.nifi.util.FileBasedVariableRegistry; import org.apache.nifi.util.NiFiProperties; import org.junit.After; import org.junit.Before; import org.junit.Test; import org.mockito.Mockito; -import java.io.File; -import java.io.FileInputStream; -import java.io.IOException; -import java.nio.charset.StandardCharsets; -import java.util.HashMap; -import java.util.LinkedHashSet; -import java.util.Map; -import java.util.Set; -import org.apache.nifi.util.FileBasedVariableRegistry; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertNotEquals; -import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.assertTrue; -import static org.mockito.Mockito.when; - public class TestFlowController { private FlowController controller; diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/controller/scheduling/TestStandardProcessScheduler.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/controller/scheduling/TestStandardProcessScheduler.java index ff787be3c7..e8185cb66f 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/controller/scheduling/TestStandardProcessScheduler.java +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/controller/scheduling/TestStandardProcessScheduler.java @@ -29,7 +29,6 @@ import java.util.concurrent.Executors; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger; - import org.apache.nifi.annotation.lifecycle.OnDisabled; import org.apache.nifi.annotation.lifecycle.OnEnabled; import org.apache.nifi.annotation.lifecycle.OnScheduled; diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/controller/service/TestStandardControllerServiceProvider.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/controller/service/TestStandardControllerServiceProvider.java index 34033467dc..f0e1566404 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/controller/service/TestStandardControllerServiceProvider.java +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/controller/service/TestStandardControllerServiceProvider.java @@ -29,7 +29,6 @@ import java.util.List; import java.util.Map; import java.util.Set; import java.util.UUID; - import org.apache.nifi.components.state.StateManager; import org.apache.nifi.components.state.StateManagerProvider; import org.apache.nifi.controller.FlowController; @@ -430,7 +429,7 @@ public class TestStandardControllerServiceProvider { E.setProperty(ServiceA.OTHER_SERVICE.getName(), "A"); E.setProperty(ServiceA.OTHER_SERVICE_2.getName(), "F"); - provider.enableControllerServices(Arrays.asList(new ControllerServiceNode[]{A, B, C, D, E, F})); + provider.enableControllerServices(Arrays.asList(A, B, C, D, E, F)); assertTrue(A.isActive()); assertTrue(B.isActive()); @@ -473,7 +472,7 @@ public class TestStandardControllerServiceProvider { F.setProperty(ServiceA.OTHER_SERVICE.getName(), "D"); D.setProperty(ServiceA.OTHER_SERVICE.getName(), "C"); - provider.enableControllerServices(Arrays.asList(new ControllerServiceNode[]{C, F, A, B, D})); + provider.enableControllerServices(Arrays.asList(C, F, A, B, D)); assertTrue(A.isActive()); assertTrue(B.isActive()); @@ -516,7 +515,7 @@ public class TestStandardControllerServiceProvider { serviceNode7.setProperty(ServiceC.REQ_SERVICE_2.getName(), "3"); provider.enableControllerServices(Arrays.asList( - new ControllerServiceNode[]{serviceNode1, serviceNode2, serviceNode3, serviceNode4, serviceNode5, serviceNode7})); + serviceNode1, serviceNode2, serviceNode3, serviceNode4, serviceNode5, serviceNode7)); assertFalse(serviceNode1.isActive()); assertFalse(serviceNode2.isActive()); assertFalse(serviceNode3.isActive()); @@ -526,7 +525,7 @@ public class TestStandardControllerServiceProvider { provider.enableControllerService(serviceNode6); provider.enableControllerServices(Arrays.asList( - new ControllerServiceNode[]{serviceNode1, serviceNode2, serviceNode3, serviceNode4, serviceNode5})); + serviceNode1, serviceNode2, serviceNode3, serviceNode4, serviceNode5)); assertTrue(serviceNode1.isActive()); assertTrue(serviceNode2.isActive()); diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-nar-utils/src/main/java/org/apache/nifi/nar/NarThreadContextClassLoader.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-nar-utils/src/main/java/org/apache/nifi/nar/NarThreadContextClassLoader.java index 827abdd014..ff11a9e660 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-nar-utils/src/main/java/org/apache/nifi/nar/NarThreadContextClassLoader.java +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-nar-utils/src/main/java/org/apache/nifi/nar/NarThreadContextClassLoader.java @@ -173,14 +173,14 @@ public class NarThreadContextClassLoader extends URLClassLoader { * constructor or a constructor which takes a NiFiProperties object * (preferred). * - * @param type - * @param implementationClassName class - * @param typeDefinition def - * @param nifiProperties props + * @param the type to create an instance for + * @param implementationClassName the implementation class name + * @param typeDefinition the type definition + * @param nifiProperties the NiFiProperties instance * @return constructed instance - * @throws InstantiationException ex - * @throws IllegalAccessException ex - * @throws ClassNotFoundException ex + * @throws InstantiationException if there is an error instantiating the class + * @throws IllegalAccessException if there is an error accessing the type + * @throws ClassNotFoundException if the class cannot be found */ public static T createInstance(final String implementationClassName, final Class typeDefinition, final NiFiProperties nifiProperties) throws InstantiationException, IllegalAccessException, ClassNotFoundException { @@ -199,7 +199,7 @@ public class NarThreadContextClassLoader extends URLClassLoader { Thread.currentThread().setContextClassLoader(detectedClassLoaderForType); final Class desiredClass = rawClass.asSubclass(typeDefinition); - if (nifiProperties == null) { + if(nifiProperties == null){ return typeDefinition.cast(desiredClass.newInstance()); } Constructor constructor = null; diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-properties-loader/pom.xml b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-properties-loader/pom.xml new file mode 100644 index 0000000000..f540b226c3 --- /dev/null +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-properties-loader/pom.xml @@ -0,0 +1,53 @@ + + + 4.0.0 + + org.apache.nifi + nifi-framework + 1.0.0-SNAPSHOT + + + nifi-properties-loader + NiFi Properties Loader + Handles the loading of the nifi.properties file to an instance of NiFiProperties, and transparently + performs any decryption/retrieval of sensitive configuration properties. + + jar + + + + org.apache.nifi + nifi-properties + + + org.bouncycastle + bcprov-jdk15on + + + org.slf4j + jul-to-slf4j + + + ch.qos.logback + logback-classic + + + org.apache.commons + commons-lang3 + + + \ No newline at end of file diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-properties-loader/src/main/java/org/apache/nifi/properties/AESSensitivePropertyProvider.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-properties-loader/src/main/java/org/apache/nifi/properties/AESSensitivePropertyProvider.java new file mode 100644 index 0000000000..cab2b0d5be --- /dev/null +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-properties-loader/src/main/java/org/apache/nifi/properties/AESSensitivePropertyProvider.java @@ -0,0 +1,251 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.properties; + +import java.nio.charset.StandardCharsets; +import java.security.InvalidAlgorithmParameterException; +import java.security.InvalidKeyException; +import java.security.NoSuchAlgorithmException; +import java.security.NoSuchProviderException; +import java.security.SecureRandom; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.stream.Collectors; +import javax.crypto.BadPaddingException; +import javax.crypto.Cipher; +import javax.crypto.IllegalBlockSizeException; +import javax.crypto.NoSuchPaddingException; +import javax.crypto.SecretKey; +import javax.crypto.spec.IvParameterSpec; +import javax.crypto.spec.SecretKeySpec; +import org.apache.commons.lang3.StringUtils; +import org.bouncycastle.util.encoders.Base64; +import org.bouncycastle.util.encoders.DecoderException; +import org.bouncycastle.util.encoders.EncoderException; +import org.bouncycastle.util.encoders.Hex; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class AESSensitivePropertyProvider implements SensitivePropertyProvider { + private static final Logger logger = LoggerFactory.getLogger(AESSensitivePropertyProvider.class); + + private static final String IMPLEMENTATION_NAME = "AES Sensitive Property Provider"; + private static final String IMPLEMENTATION_KEY = "aes/gcm/"; + private static final String ALGORITHM = "AES/GCM/NoPadding"; + private static final String PROVIDER = "BC"; + private static final String DELIMITER = "||"; // "|" is not a valid Base64 character, so ensured not to be present in cipher text + private static final int IV_LENGTH = 12; + private static final int MIN_CIPHER_TEXT_LENGTH = IV_LENGTH * 4 / 3 + DELIMITER.length() + 1; + + private Cipher cipher; + private final SecretKey key; + + public AESSensitivePropertyProvider(String keyHex) throws NoSuchPaddingException, NoSuchAlgorithmException, NoSuchProviderException { + byte[] key = validateKey(keyHex); + + try { + cipher = Cipher.getInstance(ALGORITHM, PROVIDER); + // Only store the key if the cipher was initialized successfully + this.key = new SecretKeySpec(key, "AES"); + } catch (NoSuchAlgorithmException | NoSuchProviderException | NoSuchPaddingException e) { + logger.error("Encountered an error initializing the {}: {}", IMPLEMENTATION_NAME, e.getMessage()); + throw new SensitivePropertyProtectionException("Error initializing the protection cipher", e); + } + } + + private byte[] validateKey(String keyHex) { + if (keyHex == null || StringUtils.isBlank(keyHex)) { + throw new SensitivePropertyProtectionException("The key cannot be empty"); + } + keyHex = formatHexKey(keyHex); + if (!isHexKeyValid(keyHex)) { + throw new SensitivePropertyProtectionException("The key must be a valid hexadecimal key"); + } + byte[] key = Hex.decode(keyHex); + final List validKeyLengths = getValidKeyLengths(); + if (!validKeyLengths.contains(key.length * 8)) { + List validKeyLengthsAsStrings = validKeyLengths.stream().map(i -> Integer.toString(i)).collect(Collectors.toList()); + throw new SensitivePropertyProtectionException("The key (" + key.length * 8 + " bits) must be a valid length: " + StringUtils.join(validKeyLengthsAsStrings, ", ")); + } + return key; + } + + public AESSensitivePropertyProvider(byte[] key) throws NoSuchPaddingException, NoSuchAlgorithmException, NoSuchProviderException { + this(key == null ? "" : Hex.toHexString(key)); + } + + private static String formatHexKey(String input) { + if (input == null || StringUtils.isBlank(input)) { + return ""; + } + return input.replaceAll("[^0-9a-fA-F]", "").toLowerCase(); + } + + private static boolean isHexKeyValid(String key) { + if (key == null || StringUtils.isBlank(key)) { + return false; + } + // Key length is in "nibbles" (i.e. one hex char = 4 bits) + return getValidKeyLengths().contains(key.length() * 4) && key.matches("^[0-9a-fA-F]*$"); + } + + private static List getValidKeyLengths() { + List validLengths = new ArrayList<>(); + validLengths.add(128); + + try { + if (Cipher.getMaxAllowedKeyLength("AES") > 128) { + validLengths.add(192); + validLengths.add(256); + } else { + logger.warn("JCE Unlimited Strength Cryptography Jurisdiction policies are not available, so the max key length is 128 bits"); + } + } catch (NoSuchAlgorithmException e) { + logger.warn("Encountered an error determining the max key length", e); + } + + return validLengths; + } + + /** + * Returns the name of the underlying implementation. + * + * @return the name of this sensitive property provider + */ + @Override + public String getName() { + return IMPLEMENTATION_NAME; + } + + /** + * Returns the key used to identify the provider implementation in {@code nifi.properties}. + * + * @return the key to persist in the sibling property + */ + @Override + public String getIdentifierKey() { + return IMPLEMENTATION_KEY + Collections.max(getValidKeyLengths()).toString(); + } + + /** + * Returns the encrypted cipher text. + * + * @param unprotectedValue the sensitive value + * @return the value to persist in the {@code nifi.properties} file + * @throws SensitivePropertyProtectionException if there is an exception encrypting the value + */ + @Override + public String protect(String unprotectedValue) throws SensitivePropertyProtectionException { + if (unprotectedValue == null || unprotectedValue.trim().length() == 0) { + throw new IllegalArgumentException("Cannot encrypt an empty value"); + } + + // Generate IV + byte[] iv = generateIV(); + if (iv.length < IV_LENGTH) { + throw new IllegalArgumentException("The IV (" + iv.length + " bytes) must be at least " + IV_LENGTH + " bytes"); + } + + try { + // Initialize cipher for encryption + cipher.init(Cipher.ENCRYPT_MODE, this.key, new IvParameterSpec(iv)); + + byte[] plainBytes = unprotectedValue.getBytes(StandardCharsets.UTF_8); + byte[] cipherBytes = cipher.doFinal(plainBytes); + logger.info(getName() + " encrypted a sensitive value successfully"); + return base64Encode(iv) + DELIMITER + base64Encode(cipherBytes); + // return Base64.toBase64String(iv) + DELIMITER + Base64.toBase64String(cipherBytes); + } catch (BadPaddingException | IllegalBlockSizeException | EncoderException | InvalidAlgorithmParameterException | InvalidKeyException e) { + final String msg = "Error encrypting a protected value"; + logger.error(msg, e); + throw new SensitivePropertyProtectionException(msg, e); + } + } + + private String base64Encode(byte[] input) { + return Base64.toBase64String(input).replaceAll("=", ""); + } + + /** + * Generates a new random IV of 12 bytes using {@link java.security.SecureRandom}. + * + * @return the IV + */ + private byte[] generateIV() { + byte[] iv = new byte[IV_LENGTH]; + new SecureRandom().nextBytes(iv); + return iv; + } + + /** + * Returns the decrypted plaintext. + * + * @param protectedValue the cipher text read from the {@code nifi.properties} file + * @return the raw value to be used by the application + * @throws SensitivePropertyProtectionException if there is an error decrypting the cipher text + */ + @Override + public String unprotect(String protectedValue) throws SensitivePropertyProtectionException { + if (protectedValue == null || protectedValue.trim().length() < MIN_CIPHER_TEXT_LENGTH) { + throw new IllegalArgumentException("Cannot decrypt a cipher text shorter than " + MIN_CIPHER_TEXT_LENGTH + " chars"); + } + + if (!protectedValue.contains(DELIMITER)) { + throw new IllegalArgumentException("The cipher text does not contain the delimiter " + DELIMITER + " -- it should be of the form Base64(IV) || Base64(cipherText)"); + } + + final String IV_B64 = protectedValue.substring(0, protectedValue.indexOf(DELIMITER)); + byte[] iv = Base64.decode(IV_B64); + if (iv.length < IV_LENGTH) { + throw new IllegalArgumentException("The IV (" + iv.length + " bytes) must be at least " + IV_LENGTH + " bytes"); + } + + String CIPHERTEXT_B64 = protectedValue.substring(protectedValue.indexOf(DELIMITER) + 2); + + // Restore the = padding if necessary to reconstitute the GCM MAC check + if (CIPHERTEXT_B64.length() % 4 != 0) { + final int paddedLength = CIPHERTEXT_B64.length() + 4 - (CIPHERTEXT_B64.length() % 4); + CIPHERTEXT_B64 = StringUtils.rightPad(CIPHERTEXT_B64, paddedLength, '='); + } + + try { + byte[] cipherBytes = Base64.decode(CIPHERTEXT_B64); + + cipher.init(Cipher.DECRYPT_MODE, this.key, new IvParameterSpec(iv)); + byte[] plainBytes = cipher.doFinal(cipherBytes); + logger.info(getName() + " decrypted a sensitive value successfully"); + return new String(plainBytes, StandardCharsets.UTF_8); + } catch (BadPaddingException | IllegalBlockSizeException | DecoderException | InvalidAlgorithmParameterException | InvalidKeyException e) { + final String msg = "Error decrypting a protected value"; + logger.error(msg, e); + throw new SensitivePropertyProtectionException(msg, e); + } + } + + public static int getIvLength() { + return IV_LENGTH; + } + + public static int getMinCipherTextLength() { + return MIN_CIPHER_TEXT_LENGTH; + } + + public static String getDelimiter() { + return DELIMITER; + } +} diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-properties-loader/src/main/java/org/apache/nifi/properties/AESSensitivePropertyProviderFactory.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-properties-loader/src/main/java/org/apache/nifi/properties/AESSensitivePropertyProviderFactory.java new file mode 100644 index 0000000000..56a1cc0509 --- /dev/null +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-properties-loader/src/main/java/org/apache/nifi/properties/AESSensitivePropertyProviderFactory.java @@ -0,0 +1,53 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.properties; + +import java.security.NoSuchAlgorithmException; +import java.security.NoSuchProviderException; +import javax.crypto.NoSuchPaddingException; +import org.apache.commons.lang3.StringUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class AESSensitivePropertyProviderFactory implements SensitivePropertyProviderFactory { + private static final Logger logger = LoggerFactory.getLogger(AESSensitivePropertyProviderFactory.class); + + private String keyHex; + + public AESSensitivePropertyProviderFactory(String keyHex) { + this.keyHex = keyHex; + } + + public SensitivePropertyProvider getProvider() throws SensitivePropertyProtectionException { + try { + if (keyHex != null && !StringUtils.isBlank(keyHex)) { + return new AESSensitivePropertyProvider(keyHex); + } else { + throw new SensitivePropertyProtectionException("The provider factory cannot generate providers without a key"); + } + } catch (NoSuchAlgorithmException | NoSuchProviderException | NoSuchPaddingException e) { + String msg = "Error creating AES Sensitive Property Provider"; + logger.warn(msg, e); + throw new SensitivePropertyProtectionException(msg, e); + } + } + + @Override + public String toString() { + return "SensitivePropertyProviderFactory for creating AESSensitivePropertyProviders"; + } +} diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-properties-loader/src/main/java/org/apache/nifi/properties/MultipleSensitivePropertyProtectionException.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-properties-loader/src/main/java/org/apache/nifi/properties/MultipleSensitivePropertyProtectionException.java new file mode 100644 index 0000000000..3b6f3cd796 --- /dev/null +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-properties-loader/src/main/java/org/apache/nifi/properties/MultipleSensitivePropertyProtectionException.java @@ -0,0 +1,128 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.properties; + +import java.util.Collection; +import java.util.HashSet; +import java.util.Set; +import org.apache.commons.lang3.StringUtils; + +public class MultipleSensitivePropertyProtectionException extends SensitivePropertyProtectionException { + + private Set failedKeys; + + /** + * Constructs a new throwable with {@code null} as its detail message. + * The cause is not initialized, and may subsequently be initialized by a + * call to {@link #initCause}. + *

    + *

    The {@link #fillInStackTrace()} method is called to initialize + * the stack trace data in the newly created throwable. + */ + public MultipleSensitivePropertyProtectionException() { + } + + /** + * Constructs a new throwable with the specified detail message. The + * cause is not initialized, and may subsequently be initialized by + * a call to {@link #initCause}. + *

    + *

    The {@link #fillInStackTrace()} method is called to initialize + * the stack trace data in the newly created throwable. + * + * @param message the detail message. The detail message is saved for + * later retrieval by the {@link #getMessage()} method. + */ + public MultipleSensitivePropertyProtectionException(String message) { + super(message); + } + + /** + * Constructs a new throwable with the specified detail message and + * cause.

    Note that the detail message associated with + * {@code cause} is not automatically incorporated in + * this throwable's detail message. + *

    + *

    The {@link #fillInStackTrace()} method is called to initialize + * the stack trace data in the newly created throwable. + * + * @param message the detail message (which is saved for later retrieval + * by the {@link #getMessage()} method). + * @param cause the cause (which is saved for later retrieval by the + * {@link #getCause()} method). (A {@code null} value is + * permitted, and indicates that the cause is nonexistent or + * unknown.) + * @since 1.4 + */ + public MultipleSensitivePropertyProtectionException(String message, Throwable cause) { + super(message, cause); + } + + /** + * Constructs a new throwable with the specified cause and a detail + * message of {@code (cause==null ? null : cause.toString())} (which + * typically contains the class and detail message of {@code cause}). + * This constructor is useful for throwables that are little more than + * wrappers for other throwables (for example, PrivilegedActionException). + *

    + *

    The {@link #fillInStackTrace()} method is called to initialize + * the stack trace data in the newly created throwable. + * + * @param cause the cause (which is saved for later retrieval by the + * {@link #getCause()} method). (A {@code null} value is + * permitted, and indicates that the cause is nonexistent or + * unknown.) + * @since 1.4 + */ + public MultipleSensitivePropertyProtectionException(Throwable cause) { + super(cause); + } + + /** + * Constructs a new exception with the provided message and a unique set of the keys that caused the error. + * + * @param message the message + * @param failedKeys any failed keys + */ + public MultipleSensitivePropertyProtectionException(String message, Collection failedKeys) { + this(message, failedKeys, null); + } + + /** + * Constructs a new exception with the provided message and a unique set of the keys that caused the error. + * + * @param message the message + * @param failedKeys any failed keys + * @param cause the cause (which is saved for later retrieval by the + * {@link #getCause()} method). (A {@code null} value is + * permitted, and indicates that the cause is nonexistent or + * unknown.) + */ + public MultipleSensitivePropertyProtectionException(String message, Collection failedKeys, Throwable cause) { + super(message, cause); + this.failedKeys = new HashSet<>(failedKeys); + } + + public Set getFailedKeys() { + return this.failedKeys; + } + + @Override + public String toString() { + return "SensitivePropertyProtectionException for [" + StringUtils.join(this.failedKeys, ", ") + "]: " + getLocalizedMessage(); + } +} diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-properties-loader/src/main/java/org/apache/nifi/properties/NiFiPropertiesLoader.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-properties-loader/src/main/java/org/apache/nifi/properties/NiFiPropertiesLoader.java new file mode 100644 index 0000000000..831374c643 --- /dev/null +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-properties-loader/src/main/java/org/apache/nifi/properties/NiFiPropertiesLoader.java @@ -0,0 +1,254 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.properties; + +import java.io.BufferedInputStream; +import java.io.File; +import java.io.FileInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.nio.file.Files; +import java.nio.file.Paths; +import java.security.NoSuchAlgorithmException; +import java.security.Security; +import java.util.Optional; +import java.util.Properties; +import java.util.stream.Stream; +import javax.crypto.Cipher; +import org.apache.nifi.util.NiFiProperties; +import org.bouncycastle.jce.provider.BouncyCastleProvider; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class NiFiPropertiesLoader { + private static final Logger logger = LoggerFactory.getLogger(NiFiPropertiesLoader.class); + + private static final String RELATIVE_PATH = "conf/nifi.properties"; + + private static final String BOOTSTRAP_KEY_PREFIX = "nifi.bootstrap.sensitive.key="; + + private NiFiProperties instance; + private String keyHex; + + // Future enhancement: allow for external registration of new providers + private static SensitivePropertyProviderFactory sensitivePropertyProviderFactory; + + public NiFiPropertiesLoader() { + } + + /** + * Returns an instance of the loader configured with the key. + * + * @param keyHex the key used to encrypt any sensitive properties + * @return the configured loader + */ + public static NiFiPropertiesLoader withKey(String keyHex) { + NiFiPropertiesLoader loader = new NiFiPropertiesLoader(); + loader.setKeyHex(keyHex); + return loader; + } + + /** + * Sets the hexadecimal key used to unprotect properties encrypted with + * {@link AESSensitivePropertyProvider}. If the key has already been set, + * calling this method will throw a {@link RuntimeException}. + * + * @param keyHex the key in hexadecimal format + */ + public void setKeyHex(String keyHex) { + if (this.keyHex == null || this.keyHex.trim().isEmpty()) { + this.keyHex = keyHex; + } else { + throw new RuntimeException("Cannot overwrite an existing key"); + } + } + + /** + * Returns a {@link NiFiProperties} instance with any encrypted properties + * decrypted using the key from the {@code conf/bootstrap.conf} file. This + * method is exposed to allow Spring factory-method loading at application + * startup. + * + * @return the populated and decrypted NiFiProperties instance + * @throws IOException if there is a problem reading from the bootstrap.conf or nifi.properties files + */ + public static NiFiProperties loadDefaultWithKeyFromBootstrap() throws IOException { + try { + String keyHex = extractKeyFromBootstrapFile(); + return NiFiPropertiesLoader.withKey(keyHex).loadDefault(); + } catch (IOException e) { + logger.error("Encountered an exception loading the default nifi.properties file {} with the key provided in bootstrap.conf", getDefaultFilePath(), e); + throw e; + } + } + + private static String extractKeyFromBootstrapFile() throws IOException { + // Guess at location of bootstrap.conf file from nifi.properties file + String defaultNiFiPropertiesPath = getDefaultFilePath(); + File propertiesFile = new File(defaultNiFiPropertiesPath); + File confDir = new File(propertiesFile.getParent()); + if (confDir.exists() && confDir.canRead()) { + File expectedBootstrapFile = new File(confDir, "bootstrap.conf"); + if (expectedBootstrapFile.exists() && expectedBootstrapFile.canRead()) { + try (Stream stream = Files.lines(Paths.get(expectedBootstrapFile.getAbsolutePath()))) { + Optional keyLine = stream.filter(l -> l.startsWith(BOOTSTRAP_KEY_PREFIX)).findFirst(); + if (keyLine.isPresent()) { + return keyLine.get().split("=", 2)[1]; + } else { + logger.warn("No encryption key present in the bootstrap.conf file at {}", expectedBootstrapFile.getAbsolutePath()); + return ""; + } + } catch (IOException e) { + logger.error("Cannot read from bootstrap.conf file at {} to extract encryption key", expectedBootstrapFile.getAbsolutePath()); + throw new IOException("Cannot read from bootstrap.conf", e); + } + } else { + logger.error("Cannot read from bootstrap.conf file at {} to extract encryption key -- file is missing or permissions are incorrect", expectedBootstrapFile.getAbsolutePath()); + throw new IOException("Cannot read from bootstrap.conf"); + } + } else { + logger.error("Cannot read from bootstrap.conf file at {} to extract encryption key -- conf/ directory is missing or permissions are incorrect", confDir.getAbsolutePath()); + throw new IOException("Cannot read from bootstrap.conf"); + } + } + + private static String getDefaultFilePath() { + String systemPath = System.getProperty(NiFiProperties.PROPERTIES_FILE_PATH); + + if (systemPath == null || systemPath.trim().isEmpty()) { + logger.warn("The system variable {} is not set, so it is being set to '{}'", NiFiProperties.PROPERTIES_FILE_PATH, RELATIVE_PATH); + System.setProperty(NiFiProperties.PROPERTIES_FILE_PATH, RELATIVE_PATH); + systemPath = RELATIVE_PATH; + } + + logger.info("Determined default nifi.properties path to be '{}'", systemPath); + return systemPath; + } + + private NiFiProperties loadDefault() { + return load(getDefaultFilePath()); + } + + private static String getDefaultProviderKey() { + try { + return "aes/gcm/" + (Cipher.getMaxAllowedKeyLength("AES") > 128 ? "256" : "128"); + } catch (NoSuchAlgorithmException e) { + return "aes/gcm/128"; + } + } + + private void initializeSensitivePropertyProviderFactory() { + if (sensitivePropertyProviderFactory == null) { + sensitivePropertyProviderFactory = new AESSensitivePropertyProviderFactory(keyHex); + } + } + + private SensitivePropertyProvider getSensitivePropertyProvider() { + initializeSensitivePropertyProviderFactory(); + return sensitivePropertyProviderFactory.getProvider(); + } + + /** + * Returns a {@link ProtectedNiFiProperties} instance loaded from the serialized + * form in the file. Responsible for actually reading from disk and deserializing + * the properties. Returns a protected instance to allow for decryption operations. + * + * @param file the file containing serialized properties + * @return the ProtectedNiFiProperties instance + */ + ProtectedNiFiProperties readProtectedPropertiesFromDisk(File file) { + if (file == null || !file.exists() || !file.canRead()) { + String path = (file == null ? "missing file" : file.getAbsolutePath()); + logger.error("Cannot read from '{}' -- file is missing or not readable", path); + throw new IllegalArgumentException("NiFi properties file missing or unreadable"); + } + + Properties rawProperties = new Properties(); + + InputStream inStream = null; + try { + inStream = new BufferedInputStream(new FileInputStream(file)); + rawProperties.load(inStream); + logger.info("Loaded {} properties from {}", rawProperties.size(), file.getAbsolutePath()); + + ProtectedNiFiProperties protectedNiFiProperties = new ProtectedNiFiProperties(rawProperties); + return protectedNiFiProperties; + } catch (final Exception ex) { + logger.error("Cannot load properties file due to " + ex.getLocalizedMessage()); + throw new RuntimeException("Cannot load properties file due to " + + ex.getLocalizedMessage(), ex); + } finally { + if (null != inStream) { + try { + inStream.close(); + } catch (final Exception ex) { + /** + * do nothing * + */ + } + } + } + } + + /** + * Returns an instance of {@link NiFiProperties} loaded from the provided + * {@link File}. If any properties are protected, will attempt to use the + * appropriate {@link SensitivePropertyProvider} to unprotect them transparently. + * + * @param file the File containing the serialized properties + * @return the NiFiProperties instance + */ + public NiFiProperties load(File file) { + ProtectedNiFiProperties protectedNiFiProperties = readProtectedPropertiesFromDisk(file); + if (protectedNiFiProperties.hasProtectedKeys()) { + Security.addProvider(new BouncyCastleProvider()); + protectedNiFiProperties.addSensitivePropertyProvider(getSensitivePropertyProvider()); + } + + return protectedNiFiProperties.getUnprotectedProperties(); + } + + /** + * Returns an instance of {@link NiFiProperties}. If the path is empty, this + * will load the default properties file as specified by + * {@code NiFiProperties.PROPERTY_FILE_PATH}. + * + * @param path the path of the serialized properties file + * @return the NiFiProperties instance + * @see NiFiPropertiesLoader#load(File) + */ + public NiFiProperties load(String path) { + if (path != null && !path.trim().isEmpty()) { + return load(new File(path)); + } else { + return loadDefault(); + } + } + + /** + * Returns the loaded {@link NiFiProperties} instance. If none is currently loaded, attempts to load the default instance. + * + * @return the current NiFiProperties instance + */ + public NiFiProperties get() { + if (instance == null) { + instance = loadDefault(); + } + + return instance; + } +} diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-properties-loader/src/main/java/org/apache/nifi/properties/ProtectedNiFiProperties.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-properties-loader/src/main/java/org/apache/nifi/properties/ProtectedNiFiProperties.java new file mode 100644 index 0000000000..83320a0f10 --- /dev/null +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-properties-loader/src/main/java/org/apache/nifi/properties/ProtectedNiFiProperties.java @@ -0,0 +1,521 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.properties; + +import static java.util.Arrays.asList; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Properties; +import java.util.Set; +import org.apache.commons.lang3.StringUtils; +import org.apache.nifi.util.NiFiProperties; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Decorator class for intermediate phase when {@link NiFiPropertiesLoader} loads the + * raw properties file and performs unprotection activities before returning a clean + * implementation of {@link NiFiProperties}, likely {@link StandardNiFiProperties}. + * This encapsulates the sensitive property access logic from external consumers + * of {@code NiFiProperties}. + */ +class ProtectedNiFiProperties extends StandardNiFiProperties { + private static final Logger logger = LoggerFactory.getLogger(ProtectedNiFiProperties.class); + + private NiFiProperties niFiProperties; + + private Map localProviderCache = new HashMap<>(); + + // Additional "sensitive" property key + public static final String ADDITIONAL_SENSITIVE_PROPERTIES_KEY = "nifi.sensitive.props.additional.keys"; + + // Default list of "sensitive" property keys + public static final List DEFAULT_SENSITIVE_PROPERTIES = new ArrayList<>(asList(SECURITY_KEY_PASSWD, + SECURITY_KEYSTORE_PASSWD, SECURITY_TRUSTSTORE_PASSWD, SENSITIVE_PROPS_KEY)); + + public ProtectedNiFiProperties() { + this(new StandardNiFiProperties()); + } + + /** + * Creates an instance containing the provided {@link NiFiProperties}. + * + * @param props the NiFiProperties to contain + */ + public ProtectedNiFiProperties(NiFiProperties props) { + this.niFiProperties = props; + logger.debug("Loaded {} properties (including {} protection schemes) into ProtectedNiFiProperties", getPropertyKeysIncludingProtectionSchemes().size(), getProtectedPropertyKeys().size()); + } + + /** + * Creates an instance containing the provided raw {@link Properties}. + * + * @param rawProps the Properties to contain + */ + public ProtectedNiFiProperties(Properties rawProps) { + this(new StandardNiFiProperties(rawProps)); + } + + /** + * Retrieves the property value for the given property key. + * + * @param key the key of property value to lookup + * @return value of property at given key or null if not found + */ + @Override + public String getProperty(String key) { + return getInternalNiFiProperties().getProperty(key); + } + + /** + * Retrieves all known property keys. + * + * @return all known property keys + */ + @Override + public Set getPropertyKeys() { + Set filteredKeys = getPropertyKeysIncludingProtectionSchemes(); + filteredKeys.removeIf(p -> p.endsWith(".protected")); + return filteredKeys; + } + + /** + * Returns the internal representation of the {@link NiFiProperties} -- protected + * or not as determined by the current state. No guarantee is made to the + * protection state of these properties. If the internal reference is null, a new + * {@link StandardNiFiProperties} instance is created. + * + * @return the internal properties + */ + NiFiProperties getInternalNiFiProperties() { + if (this.niFiProperties == null) { + this.niFiProperties = new StandardNiFiProperties(); + } + + return this.niFiProperties; + } + + /** + * Returns the number of properties, excluding protection scheme properties. + *

    + * Example: + *

    + * key: E(value, key) + * key.protected: aes/gcm/256 + * key2: value2 + *

    + * would return size 2 + * + * @return the count of real properties + */ + @Override + public int size() { + return getPropertyKeys().size(); + } + + /** + * Returns the complete set of property keys, including any protection keys (i.e. 'x.y.z.protected'). + * + * @return the set of property keys + */ + Set getPropertyKeysIncludingProtectionSchemes() { + return getInternalNiFiProperties().getPropertyKeys(); + } + + /** + * Splits a single string containing multiple property keys into a List. Delimited by ',' or ';' and ignores leading and trailing whitespace around delimiter. + * + * @param multipleProperties a single String containing multiple properties, i.e. "nifi.property.1; nifi.property.2, nifi.property.3" + * @return a List containing the split and trimmed properties + */ + private static List splitMultipleProperties(String multipleProperties) { + if (multipleProperties == null || multipleProperties.trim().isEmpty()) { + return new ArrayList<>(0); + } else { + List properties = new ArrayList<>(asList(multipleProperties.split("\\s*[,;]\\s*"))); + for (int i = 0; i < properties.size(); i++) { + properties.set(i, properties.get(i).trim()); + } + return properties; + } + } + + /** + * Returns a list of the keys identifying "sensitive" properties. There is a default list, + * and additional keys can be provided in the {@code nifi.sensitive.props.additional.keys} property in {@code nifi.properties}. + * + * @return the list of sensitive property keys + */ + public List getSensitivePropertyKeys() { + String additionalPropertiesString = getProperty(ADDITIONAL_SENSITIVE_PROPERTIES_KEY); + if (additionalPropertiesString == null || additionalPropertiesString.trim().isEmpty()) { + return DEFAULT_SENSITIVE_PROPERTIES; + } else { + List additionalProperties = splitMultipleProperties(additionalPropertiesString); + /* Remove this key if it was accidentally provided as a sensitive key + * because we cannot protect it and read from it + */ + if (additionalProperties.contains(ADDITIONAL_SENSITIVE_PROPERTIES_KEY)) { + logger.warn("The key '{}' contains itself. This is poor practice and should be removed", ADDITIONAL_SENSITIVE_PROPERTIES_KEY); + additionalProperties.remove(ADDITIONAL_SENSITIVE_PROPERTIES_KEY); + } + additionalProperties.addAll(DEFAULT_SENSITIVE_PROPERTIES); + return additionalProperties; + } + } + + /** + * Returns true if any sensitive keys are protected. + * + * @return true if any key is protected; false otherwise + */ + public boolean hasProtectedKeys() { + List sensitiveKeys = getSensitivePropertyKeys(); + for (String k : sensitiveKeys) { + if (isPropertyProtected(k)) { + return true; + } + } + return false; + } + + /** + * Returns a Map of the keys identifying "sensitive" properties that are currently protected and the "protection" key for each. This may or may not include all properties marked as sensitive. + * + * @return the Map of protected property keys and the protection identifier for each + */ + public Map getProtectedPropertyKeys() { + List sensitiveKeys = getSensitivePropertyKeys(); + + // This is the Java 8 way, but can likely be optimized (and not sure of correctness) + // Map protectedProperties = sensitiveKeys.stream().filter(key -> + // getProperty(getProtectionKey(key)) != null).collect(Collectors.toMap(Function.identity(), key -> + // getProperty(getProtectionKey(key)))); + + // Groovy + // Map groovyProtectedProperties = sensitiveKeys.collectEntries { key -> + // [(key): getProperty(getProtectionKey(key))] }.findAll { k, v -> v } + + // Traditional way + Map traditionalProtectedProperties = new HashMap<>(); + for (String key : sensitiveKeys) { + String protection = getProperty(getProtectionKey(key)); + if (!StringUtils.isBlank(protection)) { + traditionalProtectedProperties.put(key, protection); + } + } + + return traditionalProtectedProperties; + } + + /** + * Returns the unique set of all protection schemes currently in use for this instance. + * + * @return the set of protection schemes + */ + public Set getProtectionSchemes() { + return new HashSet<>(getProtectedPropertyKeys().values()); + } + + /** + * Returns a percentage of the total number of properties marked as sensitive that are currently protected. + * + * @return the percent of sensitive properties marked as protected + */ + public int getPercentOfSensitivePropertiesProtected() { + return (int) Math.round(getProtectedPropertyKeys().size() / ((double) getSensitivePropertyKeys().size()) * 100); + } + + /** + * Returns true if the property identified by this key is considered sensitive in this instance of {@code NiFiProperties}. + * Some properties are sensitive by default, while others can be specified by + * {@link ProtectedNiFiProperties#ADDITIONAL_SENSITIVE_PROPERTIES_KEY}. + * + * @param key the key + * @return true if it is sensitive + * @see ProtectedNiFiProperties#getSensitivePropertyKeys() + */ + public boolean isPropertySensitive(String key) { + // If the explicit check for ADDITIONAL_SENSITIVE_PROPERTIES_KEY is not here, this will loop infinitely + return key != null && !key.equals(ADDITIONAL_SENSITIVE_PROPERTIES_KEY) && getSensitivePropertyKeys().contains(key.trim()); + } + + /** + * Returns true if the property identified by this key is considered protected in this instance of {@code NiFiProperties}. + * The property value is protected if the key is sensitive and the sibling key of key.protected is present. + * + * @param key the key + * @return true if it is currently marked as protected + * @see ProtectedNiFiProperties#getSensitivePropertyKeys() + */ + public boolean isPropertyProtected(String key) { + return key != null && isPropertySensitive(key) && !StringUtils.isBlank(getProperty(getProtectionKey(key))); + } + + /** + * Returns the sibling property key which specifies the protection scheme for this key. + *

    + * Example: + *

    + * nifi.sensitive.key=ABCXYZ + * nifi.sensitive.key.protected=aes/gcm/256 + *

    + * nifi.sensitive.key -> nifi.sensitive.key.protected + * + * @param key the key identifying the sensitive property + * @return the key identifying the protection scheme for the sensitive property + */ + public String getProtectionKey(String key) { + if (key == null || key.isEmpty()) { + throw new IllegalArgumentException("Cannot find protection key for null key"); + } + + return key + ".protected"; + } + + /** + * Returns the unprotected {@link NiFiProperties} instance. If none of the properties + * loaded are marked as protected, it will simply pass through the internal instance. + * If any are protected, it will drop the protection scheme keys and translate each + * protected value (encrypted, HSM-retrieved, etc.) into the raw value and store it + * under the original key. + *

    + * If any property fails to unprotect, it will save that key and continue. After + * attempting all properties, it will throw an exception containing all failed + * properties. This is necessary because the order is not enforced, so all failed + * properties should be gathered together. + * + * @return the NiFiProperties instance with all raw values + * @throws SensitivePropertyProtectionException if there is a problem unprotecting one or more keys + */ + public NiFiProperties getUnprotectedProperties() throws SensitivePropertyProtectionException { + if (hasProtectedKeys()) { + logger.info("There are {} protected properties of {} sensitive properties ({}%)", + getProtectedPropertyKeys().size(), + getSensitivePropertyKeys().size(), + getPercentOfSensitivePropertiesProtected()); + + Properties rawProperties = new Properties(); + + Set failedKeys = new HashSet<>(); + + for (String key : getPropertyKeys()) { + /* Three kinds of keys + * 1. protection schemes -- skip + * 2. protected keys -- unprotect and copy + * 3. normal keys -- copy over + */ + if (key.endsWith(".protected")) { + // Do nothing + } else if (isPropertyProtected(key)) { + try { + rawProperties.setProperty(key, unprotectValue(key, getProperty(key))); + } catch (SensitivePropertyProtectionException e) { + logger.warn("Failed to unprotect '{}'", key, e); + failedKeys.add(key); + } + } else { + rawProperties.setProperty(key, getProperty(key)); + } + } + + if (!failedKeys.isEmpty()) { + if (failedKeys.size() > 1) { + logger.warn("Combining {} failed keys [{}] into single exception", failedKeys.size(), StringUtils.join(failedKeys, ", ")); + throw new MultipleSensitivePropertyProtectionException("Failed to unprotect keys", failedKeys); + } else { + throw new SensitivePropertyProtectionException("Failed to unprotect key " + failedKeys.iterator().next()); + } + } + + NiFiProperties unprotected = new StandardNiFiProperties(rawProperties); + + return unprotected; + } else { + logger.debug("No protected properties"); + return getInternalNiFiProperties(); + } + } + + /** + * Registers a new {@link SensitivePropertyProvider}. This method will throw a {@link UnsupportedOperationException} if a provider is already registered for the protection scheme. + * + * @param sensitivePropertyProvider the provider + */ + void addSensitivePropertyProvider(SensitivePropertyProvider sensitivePropertyProvider) { + if (sensitivePropertyProvider == null) { + throw new IllegalArgumentException("Cannot add null SensitivePropertyProvider"); + } + + if (getSensitivePropertyProviders().containsKey(sensitivePropertyProvider.getIdentifierKey())) { + throw new UnsupportedOperationException("Cannot overwrite existing sensitive property provider registered for " + sensitivePropertyProvider.getIdentifierKey()); + } + + getSensitivePropertyProviders().put(sensitivePropertyProvider.getIdentifierKey(), sensitivePropertyProvider); + } + + private String getDefaultProtectionScheme() { + if (!getSensitivePropertyProviders().isEmpty()) { + List schemes = new ArrayList<>(getSensitivePropertyProviders().keySet()); + Collections.sort(schemes); + return schemes.get(0); + } else { + throw new IllegalStateException("No registered protection schemes"); + } + } + + /** + * Returns a new instance of {@link NiFiProperties} with all populated sensitive values protected by the default protection scheme. Plain non-sensitive values are copied directly. + * + * @return the protected properties in a {@link StandardNiFiProperties} object + * @throws IllegalStateException if no protection schemes are registered + */ + NiFiProperties protectPlainProperties() { + try { + return protectPlainProperties(getDefaultProtectionScheme()); + } catch (IllegalStateException e) { + final String msg = "Cannot protect properties with default scheme if no protection schemes are registered"; + logger.warn(msg); + throw new IllegalStateException(msg, e); + } + } + + /** + * Returns a new instance of {@link NiFiProperties} with all populated sensitive values protected by the provided protection scheme. Plain non-sensitive values are copied directly. + * + * @param protectionScheme the identifier key of the {@link SensitivePropertyProvider} to use + * @return the protected properties in a {@link StandardNiFiProperties} object + */ + NiFiProperties protectPlainProperties(String protectionScheme) { + SensitivePropertyProvider spp = getSensitivePropertyProvider(protectionScheme); + + // Make a new holder (settable) + Properties protectedProperties = new Properties(); + + // Copy over the plain keys + Set plainKeys = getPropertyKeys(); + plainKeys.removeAll(getSensitivePropertyKeys()); + for (String key : plainKeys) { + protectedProperties.setProperty(key, getInternalNiFiProperties().getProperty(key)); + } + + // Add the protected keys and the protection schemes + for (String key : getSensitivePropertyKeys()) { + final String plainValue = getInternalNiFiProperties().getProperty(key); + if (plainValue == null || plainValue.trim().isEmpty()) { + protectedProperties.setProperty(key, plainValue); + } else { + final String protectedValue = spp.protect(plainValue); + protectedProperties.setProperty(key, protectedValue); + protectedProperties.setProperty(getProtectionKey(key), protectionScheme); + } + } + + return new StandardNiFiProperties(protectedProperties); + } + + /** + * Returns the number of properties that are marked as protected in the provided {@link NiFiProperties} instance without requiring external creation of a {@link ProtectedNiFiProperties} instance. + * + * @param plainProperties the instance to count protected properties + * @return the number of protected properties + */ + public static int countProtectedProperties(NiFiProperties plainProperties) { + return new ProtectedNiFiProperties(plainProperties).getProtectedPropertyKeys().size(); + } + + /** + * Returns the number of properties that are marked as sensitive in the provided {@link NiFiProperties} instance without requiring external creation of a {@link ProtectedNiFiProperties} instance. + * + * @param plainProperties the instance to count sensitive properties + * @return the number of sensitive properties + */ + public static int countSensitiveProperties(NiFiProperties plainProperties) { + return new ProtectedNiFiProperties(plainProperties).getSensitivePropertyKeys().size(); + } + + @Override + public String toString() { + final Set providers = getSensitivePropertyProviders().keySet(); + return new StringBuilder("ProtectedNiFiProperties instance with ") + .append(size()).append(" properties (") + .append(getProtectedPropertyKeys().size()) + .append(" protected) and ") + .append(providers.size()) + .append(" sensitive property providers: ") + .append(StringUtils.join(providers, ", ")) + .toString(); + } + + /** + * Returns the local provider cache (null-safe) as a Map of protection schemes -> implementations. + * + * @return the map + */ + private Map getSensitivePropertyProviders() { + if (localProviderCache == null) { + localProviderCache = new HashMap<>(); + } + + return localProviderCache; + } + + private SensitivePropertyProvider getSensitivePropertyProvider(String protectionScheme) { + if (isProviderAvailable(protectionScheme)) { + return getSensitivePropertyProviders().get(protectionScheme); + } else { + throw new SensitivePropertyProtectionException("No provider available for " + protectionScheme); + } + } + + private boolean isProviderAvailable(String protectionScheme) { + return getSensitivePropertyProviders().containsKey(protectionScheme); + } + + /** + * If the value is protected, unprotects it and returns it. If not, returns the original value. + * + * @param key the retrieved property key + * @param retrievedValue the retrieved property value + * @return the unprotected value + */ + private String unprotectValue(String key, String retrievedValue) { + // Checks if the key is sensitive and marked as protected + if (isPropertyProtected(key)) { + final String protectionScheme = getProperty(getProtectionKey(key)); + + // No provider registered for this scheme, so just return the value + if (!isProviderAvailable(protectionScheme)) { + logger.warn("No provider available for {} so passing the protected {} value back", protectionScheme, key); + return retrievedValue; + } + + try { + SensitivePropertyProvider sensitivePropertyProvider = getSensitivePropertyProvider(protectionScheme); + return sensitivePropertyProvider.unprotect(retrievedValue); + } catch (SensitivePropertyProtectionException e) { + throw new SensitivePropertyProtectionException("Error unprotecting value for " + key, e.getCause()); + } + } + return retrievedValue; + } +} diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-properties-loader/src/main/java/org/apache/nifi/properties/SensitivePropertyProtectionException.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-properties-loader/src/main/java/org/apache/nifi/properties/SensitivePropertyProtectionException.java new file mode 100644 index 0000000000..2870c2a4b1 --- /dev/null +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-properties-loader/src/main/java/org/apache/nifi/properties/SensitivePropertyProtectionException.java @@ -0,0 +1,91 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.properties; + +public class SensitivePropertyProtectionException extends RuntimeException { + /** + * Constructs a new throwable with {@code null} as its detail message. + * The cause is not initialized, and may subsequently be initialized by a + * call to {@link #initCause}. + *

    + *

    The {@link #fillInStackTrace()} method is called to initialize + * the stack trace data in the newly created throwable. + */ + public SensitivePropertyProtectionException() { + } + + /** + * Constructs a new throwable with the specified detail message. The + * cause is not initialized, and may subsequently be initialized by + * a call to {@link #initCause}. + *

    + *

    The {@link #fillInStackTrace()} method is called to initialize + * the stack trace data in the newly created throwable. + * + * @param message the detail message. The detail message is saved for + * later retrieval by the {@link #getMessage()} method. + */ + public SensitivePropertyProtectionException(String message) { + super(message); + } + + /** + * Constructs a new throwable with the specified detail message and + * cause.

    Note that the detail message associated with + * {@code cause} is not automatically incorporated in + * this throwable's detail message. + *

    + *

    The {@link #fillInStackTrace()} method is called to initialize + * the stack trace data in the newly created throwable. + * + * @param message the detail message (which is saved for later retrieval + * by the {@link #getMessage()} method). + * @param cause the cause (which is saved for later retrieval by the + * {@link #getCause()} method). (A {@code null} value is + * permitted, and indicates that the cause is nonexistent or + * unknown.) + * @since 1.4 + */ + public SensitivePropertyProtectionException(String message, Throwable cause) { + super(message, cause); + } + + /** + * Constructs a new throwable with the specified cause and a detail + * message of {@code (cause==null ? null : cause.toString())} (which + * typically contains the class and detail message of {@code cause}). + * This constructor is useful for throwables that are little more than + * wrappers for other throwables (for example, PrivilegedActionException). + *

    + *

    The {@link #fillInStackTrace()} method is called to initialize + * the stack trace data in the newly created throwable. + * + * @param cause the cause (which is saved for later retrieval by the + * {@link #getCause()} method). (A {@code null} value is + * permitted, and indicates that the cause is nonexistent or + * unknown.) + * @since 1.4 + */ + public SensitivePropertyProtectionException(Throwable cause) { + super(cause); + } + + @Override + public String toString() { + return "SensitivePropertyProtectionException: " + getLocalizedMessage(); + } +} diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-properties-loader/src/main/java/org/apache/nifi/properties/SensitivePropertyProvider.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-properties-loader/src/main/java/org/apache/nifi/properties/SensitivePropertyProvider.java new file mode 100644 index 0000000000..b0c0be2e38 --- /dev/null +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-properties-loader/src/main/java/org/apache/nifi/properties/SensitivePropertyProvider.java @@ -0,0 +1,52 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.properties; + +public interface SensitivePropertyProvider { + + /** + * Returns the name of the underlying implementation. + * + * @return the name of this sensitive property provider + */ + String getName(); + + /** + * Returns the key used to identify the provider implementation in {@code nifi.properties}. + * + * @return the key to persist in the sibling property + */ + String getIdentifierKey(); + + /** + * Returns the "protected" form of this value. This is a form which can safely be persisted in the {@code nifi.properties} file without compromising the value. + * An encryption-based provider would return a cipher text, while a remote-lookup provider could return a unique ID to retrieve the secured value. + * + * @param unprotectedValue the sensitive value + * @return the value to persist in the {@code nifi.properties} file + */ + String protect(String unprotectedValue) throws SensitivePropertyProtectionException; + + /** + * Returns the "unprotected" form of this value. This is the raw sensitive value which is used by the application logic. + * An encryption-based provider would decrypt a cipher text and return the plaintext, while a remote-lookup provider could retrieve the secured value. + * + * @param protectedValue the protected value read from the {@code nifi.properties} file + * @return the raw value to be used by the application + */ + String unprotect(String protectedValue) throws SensitivePropertyProtectionException; +} diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-properties-loader/src/main/java/org/apache/nifi/properties/SensitivePropertyProviderFactory.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-properties-loader/src/main/java/org/apache/nifi/properties/SensitivePropertyProviderFactory.java new file mode 100644 index 0000000000..c800b3ad38 --- /dev/null +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-properties-loader/src/main/java/org/apache/nifi/properties/SensitivePropertyProviderFactory.java @@ -0,0 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.properties; + +public interface SensitivePropertyProviderFactory { + + SensitivePropertyProvider getProvider(); + +} diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-properties-loader/src/main/java/org/apache/nifi/properties/StandardNiFiProperties.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-properties-loader/src/main/java/org/apache/nifi/properties/StandardNiFiProperties.java new file mode 100644 index 0000000000..b7561ed3fe --- /dev/null +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-properties-loader/src/main/java/org/apache/nifi/properties/StandardNiFiProperties.java @@ -0,0 +1,81 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.properties; + +import java.util.Enumeration; +import java.util.HashSet; +import java.util.Properties; +import java.util.Set; +import org.apache.nifi.util.NiFiProperties; + +public class StandardNiFiProperties extends NiFiProperties { + + private Properties rawProperties = new Properties(); + + public StandardNiFiProperties() { + this(null); + } + + public StandardNiFiProperties(Properties props) { + this.rawProperties = props == null ? new Properties() : props; + } + + /** + * Retrieves the property value for the given property key. + * + * @param key the key of property value to lookup + * @return value of property at given key or null if not found + */ + @Override + public String getProperty(String key) { + return rawProperties.getProperty(key); + } + + /** + * Retrieves all known property keys. + * + * @return all known property keys + */ + @Override + public Set getPropertyKeys() { + Set propertyNames = new HashSet<>(); + Enumeration e = getRawProperties().propertyNames(); + for (; e.hasMoreElements(); ){ + propertyNames.add((String) e.nextElement()); + } + + return propertyNames; + } + + Properties getRawProperties() { + if (this.rawProperties == null) { + this.rawProperties = new Properties(); + } + + return this.rawProperties; + } + + @Override + public int size() { + return getRawProperties().size(); + } + + @Override + public String toString() { + return "StandardNiFiProperties instance with " + size() + " properties"; + } +} diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-properties-loader/src/test/groovy/org/apache/nifi/properties/AESSensitivePropertyProviderFactoryTest.groovy b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-properties-loader/src/test/groovy/org/apache/nifi/properties/AESSensitivePropertyProviderFactoryTest.groovy new file mode 100644 index 0000000000..b899ad22f0 --- /dev/null +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-properties-loader/src/test/groovy/org/apache/nifi/properties/AESSensitivePropertyProviderFactoryTest.groovy @@ -0,0 +1,97 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.properties + +import org.bouncycastle.jce.provider.BouncyCastleProvider +import org.junit.After +import org.junit.Before +import org.junit.BeforeClass +import org.junit.Test +import org.junit.runner.RunWith +import org.junit.runners.JUnit4 +import org.slf4j.Logger +import org.slf4j.LoggerFactory + +import java.security.Security + +@RunWith(JUnit4.class) +class AESSensitivePropertyProviderFactoryTest extends GroovyTestCase { + private static final Logger logger = LoggerFactory.getLogger(AESSensitivePropertyProviderFactoryTest.class) + + private static final String KEY_HEX = "0123456789ABCDEFFEDCBA9876543210" * 2 + + @BeforeClass + public static void setUpOnce() throws Exception { + Security.addProvider(new BouncyCastleProvider()) + + logger.metaClass.methodMissing = { String name, args -> + logger.info("[${name?.toUpperCase()}] ${(args as List).join(" ")}") + } + } + + @Before + public void setUp() throws Exception { + + } + + @After + public void tearDown() throws Exception { + + } + + @Test + public void testShouldGetProviderWithoutKey() throws Exception { + // Arrange + SensitivePropertyProviderFactory factory = new AESSensitivePropertyProviderFactory() + + // Act + SensitivePropertyProvider provider = factory.getProvider() + + // Assert + assert provider instanceof AESSensitivePropertyProvider + assert !provider.@key + assert !provider.@cipher + } + + @Test + public void testShouldGetProviderWithKey() throws Exception { + // Arrange + SensitivePropertyProviderFactory factory = new AESSensitivePropertyProviderFactory(KEY_HEX) + + // Act + SensitivePropertyProvider provider = factory.getProvider() + + // Assert + assert provider instanceof AESSensitivePropertyProvider + assert provider.@key + assert provider.@cipher + } + + @Test + public void testGetProviderShouldHandleEmptyKey() throws Exception { + // Arrange + SensitivePropertyProviderFactory factory = new AESSensitivePropertyProviderFactory("") + + // Act + SensitivePropertyProvider provider = factory.getProvider() + + // Assert + assert provider instanceof AESSensitivePropertyProvider + assert !provider.@key + assert !provider.@cipher + } +} \ No newline at end of file diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-properties-loader/src/test/groovy/org/apache/nifi/properties/AESSensitivePropertyProviderTest.groovy b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-properties-loader/src/test/groovy/org/apache/nifi/properties/AESSensitivePropertyProviderTest.groovy new file mode 100644 index 0000000000..3b06c40e5d --- /dev/null +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-properties-loader/src/test/groovy/org/apache/nifi/properties/AESSensitivePropertyProviderTest.groovy @@ -0,0 +1,463 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.properties + +import org.bouncycastle.jce.provider.BouncyCastleProvider +import org.bouncycastle.util.encoders.DecoderException +import org.bouncycastle.util.encoders.Hex +import org.junit.After +import org.junit.Assume +import org.junit.Before +import org.junit.BeforeClass +import org.junit.Test +import org.junit.runner.RunWith +import org.junit.runners.JUnit4 +import org.slf4j.Logger +import org.slf4j.LoggerFactory + +import javax.crypto.Cipher +import javax.crypto.spec.IvParameterSpec +import javax.crypto.spec.SecretKeySpec +import java.nio.charset.StandardCharsets +import java.security.SecureRandom +import java.security.Security + +@RunWith(JUnit4.class) +class AESSensitivePropertyProviderTest extends GroovyTestCase { + private static final Logger logger = LoggerFactory.getLogger(AESSensitivePropertyProviderTest.class) + + private static final String KEY_128_HEX = "0123456789ABCDEFFEDCBA9876543210" + private static final String KEY_256_HEX = KEY_128_HEX * 2 + private static final int IV_LENGTH = AESSensitivePropertyProvider.getIvLength() + + private static final List KEY_SIZES = getAvailableKeySizes() + + private static final SecureRandom secureRandom = new SecureRandom() + + private static final Base64.Encoder encoder = Base64.encoder + private static final Base64.Decoder decoder = Base64.decoder + + @BeforeClass + public static void setUpOnce() throws Exception { + Security.addProvider(new BouncyCastleProvider()) + + logger.metaClass.methodMissing = { String name, args -> + logger.info("[${name?.toUpperCase()}] ${(args as List).join(" ")}") + } + } + + @Before + public void setUp() throws Exception { + + } + + @After + public void tearDown() throws Exception { + + } + + private static Cipher getCipher(boolean encrypt = true, int keySize = 256, byte[] iv = [0x00] * IV_LENGTH) { + Cipher cipher = Cipher.getInstance("AES/GCM/NoPadding") + String key = getKeyOfSize(keySize) + cipher.init((encrypt ? Cipher.ENCRYPT_MODE : Cipher.DECRYPT_MODE) as int, new SecretKeySpec(Hex.decode(key), "AES"), new IvParameterSpec(iv)) + logger.setup("Initialized a cipher in ${encrypt ? "encrypt" : "decrypt"} mode with a key of length ${keySize} bits") + cipher + } + + private static String getKeyOfSize(int keySize = 256) { + switch (keySize) { + case 128: + return KEY_128_HEX + case 192: + case 256: + if (Cipher.getMaxAllowedKeyLength("AES") < keySize) { + throw new IllegalArgumentException("The JCE unlimited strength cryptographic jurisdiction policies are not installed, so the max key size is 128 bits") + } + return KEY_256_HEX[0..<(keySize / 4)] + default: + throw new IllegalArgumentException("Key size ${keySize} bits is not valid") + } + } + + private static List getAvailableKeySizes() { + if (Cipher.getMaxAllowedKeyLength("AES") > 128) { + [128, 192, 256] + } else { + [128] + } + } + + private static String manipulateString(String input, int start = 0, int end = input?.length()) { + if ((input[start..end] as List).unique().size() == 1) { + throw new IllegalArgumentException("Can't manipulate a String where the entire range is identical [${input[start..end]}]") + } + List shuffled = input[start..end] as List + Collections.shuffle(shuffled) + String reconstituted = input[0.. CIPHER_TEXTS = KEY_SIZES.collectEntries { int keySize -> + SensitivePropertyProvider spp = new AESSensitivePropertyProvider(Hex.decode(getKeyOfSize(keySize))) + logger.info("Initialized ${spp.name} with key size ${keySize}") + [(keySize): spp.protect(PLAINTEXT)] + } + CIPHER_TEXTS.each { ks, ct -> logger.info("Encrypted for ${ks} length key: ${ct}") } + + // Assert + + // The IV generation is part of #protect, so the expected cipher text values must be generated after #protect has run + Map decryptionCiphers = CIPHER_TEXTS.collectEntries { int keySize, String cipherText -> + // The 12 byte IV is the first 16 Base64-encoded characters of the "complete" cipher text + byte[] iv = decoder.decode(cipherText[0..<16]) + [(keySize): getCipher(false, keySize, iv)] + } + Map plaintexts = decryptionCiphers.collectEntries { Map.Entry e -> + String cipherTextWithoutIVAndDelimiter = CIPHER_TEXTS[e.key][18..-1] + String plaintext = new String(e.value.doFinal(decoder.decode(cipherTextWithoutIVAndDelimiter)), StandardCharsets.UTF_8) + [(e.key): plaintext] + } + CIPHER_TEXTS.each { key, ct -> logger.expected("Cipher text for ${key} length key: ${ct}") } + + assert plaintexts.every { int ks, String pt -> pt == PLAINTEXT } + } + + @Test + public void testShouldHandleProtectEmptyValue() throws Exception { + final List EMPTY_PLAINTEXTS = ["", " ", null] + + // Act + KEY_SIZES.collectEntries { int keySize -> + SensitivePropertyProvider spp = new AESSensitivePropertyProvider(Hex.decode(getKeyOfSize(keySize))) + logger.info("Initialized ${spp.name} with key size ${keySize}") + EMPTY_PLAINTEXTS.each { String emptyPlaintext -> + def msg = shouldFail(IllegalArgumentException) { + spp.protect(emptyPlaintext) + } + logger.expected("${msg} for keySize ${keySize} and plaintext [${emptyPlaintext}]") + + // Assert + assert msg == "Cannot encrypt an empty value" + } + } + } + + @Test + public void testShouldUnprotectValue() throws Exception { + // Arrange + final String PLAINTEXT = "This is a plaintext value" + + Map encryptionCiphers = KEY_SIZES.collectEntries { int keySize -> + byte[] iv = new byte[IV_LENGTH] + secureRandom.nextBytes(iv) + [(keySize): getCipher(true, keySize, iv)] + } + + Map CIPHER_TEXTS = encryptionCiphers.collectEntries { Map.Entry e -> + String iv = encoder.encodeToString(e.value.getIV()) + String cipherText = encoder.encodeToString(e.value.doFinal(PLAINTEXT.getBytes(StandardCharsets.UTF_8))) + [(e.key): "${iv}||${cipherText}"] + } + CIPHER_TEXTS.each { key, ct -> logger.expected("Cipher text for ${key} length key: ${ct}") } + + // Act + Map plaintexts = CIPHER_TEXTS.collectEntries { int keySize, String cipherText -> + SensitivePropertyProvider spp = new AESSensitivePropertyProvider(Hex.decode(getKeyOfSize(keySize))) + logger.info("Initialized ${spp.name} with key size ${keySize}") + [(keySize): spp.unprotect(cipherText)] + } + plaintexts.each { ks, pt -> logger.info("Decrypted for ${ks} length key: ${pt}") } + + // Assert + assert plaintexts.every { int ks, String pt -> pt == PLAINTEXT } + } + + /** + * Tests inputs where the entire String is empty/blank space/{@code null}. + * + * @throws Exception + */ + @Test + public void testShouldHandleUnprotectEmptyValue() throws Exception { + // Arrange + final List EMPTY_CIPHER_TEXTS = ["", " ", null] + + // Act + KEY_SIZES.each { int keySize -> + SensitivePropertyProvider spp = new AESSensitivePropertyProvider(Hex.decode(getKeyOfSize(keySize))) + logger.info("Initialized ${spp.name} with key size ${keySize}") + EMPTY_CIPHER_TEXTS.each { String emptyCipherText -> + def msg = shouldFail(IllegalArgumentException) { + spp.unprotect(emptyCipherText) + } + logger.expected("${msg} for keySize ${keySize} and cipher text [${emptyCipherText}]") + + // Assert + assert msg == "Cannot decrypt a cipher text shorter than ${AESSensitivePropertyProvider.minCipherTextLength} chars".toString() + } + } + } + + @Test + public void testShouldHandleUnprotectMalformedValue() throws Exception { + // Arrange + final String PLAINTEXT = "This is a plaintext value" + + // Act + KEY_SIZES.each { int keySize -> + SensitivePropertyProvider spp = new AESSensitivePropertyProvider(Hex.decode(getKeyOfSize(keySize))) + logger.info("Initialized ${spp.name} with key size ${keySize}") + String cipherText = spp.protect(PLAINTEXT) + // Swap two characters in the cipher text + final String MALFORMED_CIPHER_TEXT = manipulateString(cipherText, 25, 28) + logger.info("Manipulated ${cipherText} to\n${MALFORMED_CIPHER_TEXT.padLeft(163)}") + + def msg = shouldFail(SensitivePropertyProtectionException) { + spp.unprotect(MALFORMED_CIPHER_TEXT) + } + logger.expected("${msg} for keySize ${keySize} and cipher text [${MALFORMED_CIPHER_TEXT}]") + + // Assert + assert msg == "Error decrypting a protected value" + } + } + + @Test + public void testShouldHandleUnprotectMissingIV() throws Exception { + // Arrange + final String PLAINTEXT = "This is a plaintext value" + + // Act + KEY_SIZES.each { int keySize -> + SensitivePropertyProvider spp = new AESSensitivePropertyProvider(Hex.decode(getKeyOfSize(keySize))) + logger.info("Initialized ${spp.name} with key size ${keySize}") + String cipherText = spp.protect(PLAINTEXT) + // Remove the IV from the "complete" cipher text + final String MISSING_IV_CIPHER_TEXT = cipherText[18..-1] + logger.info("Manipulated ${cipherText} to\n${MISSING_IV_CIPHER_TEXT.padLeft(163)}") + + def msg = shouldFail(IllegalArgumentException) { + spp.unprotect(MISSING_IV_CIPHER_TEXT) + } + logger.expected("${msg} for keySize ${keySize} and cipher text [${MISSING_IV_CIPHER_TEXT}]") + + // Remove the IV from the "complete" cipher text but keep the delimiter + final String MISSING_IV_CIPHER_TEXT_WITH_DELIMITER = cipherText[16..-1] + logger.info("Manipulated ${cipherText} to\n${MISSING_IV_CIPHER_TEXT_WITH_DELIMITER.padLeft(163)}") + + def msgWithDelimiter = shouldFail(DecoderException) { + spp.unprotect(MISSING_IV_CIPHER_TEXT_WITH_DELIMITER) + } + logger.expected("${msgWithDelimiter} for keySize ${keySize} and cipher text [${MISSING_IV_CIPHER_TEXT_WITH_DELIMITER}]") + + // Assert + assert msg == "The cipher text does not contain the delimiter || -- it should be of the form Base64(IV) || Base64(cipherText)" + + // Assert + assert msgWithDelimiter =~ "unable to decode base64 string" + } + } + + /** + * Tests inputs which have a valid IV and delimiter but no "cipher text". + * + * @throws Exception + */ + @Test + public void testShouldHandleUnprotectEmptyCipherText() throws Exception { + // Arrange + final String IV_AND_DELIMITER = "${encoder.encodeToString("Bad IV value".getBytes(StandardCharsets.UTF_8))}||" + logger.info("IV and delimiter: ${IV_AND_DELIMITER}") + + final List EMPTY_CIPHER_TEXTS = ["", " ", "\n"].collect { "${IV_AND_DELIMITER}${it}" } + + // Act + KEY_SIZES.each { int keySize -> + SensitivePropertyProvider spp = new AESSensitivePropertyProvider(Hex.decode(getKeyOfSize(keySize))) + logger.info("Initialized ${spp.name} with key size ${keySize}") + EMPTY_CIPHER_TEXTS.each { String emptyCipherText -> + def msg = shouldFail(IllegalArgumentException) { + spp.unprotect(emptyCipherText) + } + logger.expected("${msg} for keySize ${keySize} and cipher text [${emptyCipherText}]") + + // Assert + assert msg == "Cannot decrypt a cipher text shorter than ${AESSensitivePropertyProvider.minCipherTextLength} chars".toString() + } + } + } + + @Test + public void testShouldHandleUnprotectMalformedIV() throws Exception { + // Arrange + final String PLAINTEXT = "This is a plaintext value" + + // Act + KEY_SIZES.each { int keySize -> + SensitivePropertyProvider spp = new AESSensitivePropertyProvider(Hex.decode(getKeyOfSize(keySize))) + logger.info("Initialized ${spp.name} with key size ${keySize}") + String cipherText = spp.protect(PLAINTEXT) + // Swap two characters in the IV + final String MALFORMED_IV_CIPHER_TEXT = manipulateString(cipherText, 8, 11) + logger.info("Manipulated ${cipherText} to\n${MALFORMED_IV_CIPHER_TEXT.padLeft(163)}") + + def msg = shouldFail(SensitivePropertyProtectionException) { + spp.unprotect(MALFORMED_IV_CIPHER_TEXT) + } + logger.expected("${msg} for keySize ${keySize} and cipher text [${MALFORMED_IV_CIPHER_TEXT}]") + + // Assert + assert msg == "Error decrypting a protected value" + } + } + + @Test + public void testShouldGetImplementationKeyWithDifferentMaxKeyLengths() throws Exception { + // Arrange + final int MAX_KEY_SIZE = getAvailableKeySizes().max() + final String EXPECTED_IMPL_KEY = "aes/gcm/${MAX_KEY_SIZE}" + logger.expected("Implementation key: ${EXPECTED_IMPL_KEY}") + + // Act + String key = new AESSensitivePropertyProvider(getKeyOfSize(MAX_KEY_SIZE)).getIdentifierKey() + logger.info("Implementation key: ${key}") + + // Assert + assert key == EXPECTED_IMPL_KEY + } + + @Test + public void testShouldNotAllowEmptyKey() throws Exception { + // Arrange + final String INVALID_KEY = "" + + // Act + def msg = shouldFail(SensitivePropertyProtectionException) { + AESSensitivePropertyProvider spp = new AESSensitivePropertyProvider(INVALID_KEY) + } + + // Assert + assert msg == "The key cannot be empty" + } + + @Test + public void testShouldNotAllowIncorrectlySizedKey() throws Exception { + // Arrange + final String INVALID_KEY = "Z" * 31 + + // Act + def msg = shouldFail(SensitivePropertyProtectionException) { + AESSensitivePropertyProvider spp = new AESSensitivePropertyProvider(INVALID_KEY) + } + + // Assert + assert msg == "The key must be a valid hexadecimal key" + } + + @Test + public void testShouldNotAllowInvalidKey() throws Exception { + // Arrange + final String INVALID_KEY = "Z" * 32 + + // Act + def msg = shouldFail(SensitivePropertyProtectionException) { + AESSensitivePropertyProvider spp = new AESSensitivePropertyProvider(INVALID_KEY) + } + + // Assert + assert msg == "The key must be a valid hexadecimal key" + } + + /** + * This test is to ensure internal consistency and allow for encrypting value for various property files + */ + @Test + public void testShouldEncryptArbitraryValues() { + // Arrange + def values = ["thisIsABadSensitiveKeyPassword", "thisIsABadKeystorePassword", "thisIsABadKeyPassword", "thisIsABadTruststorePassword", "This is an encrypted banner message"] + + String key = getKeyOfSize(128) + // key = "0" * 64 + + SensitivePropertyProvider spp = new AESSensitivePropertyProvider(key) + + // Act + def encryptedValues = values.collect { String v -> + def encryptedValue = spp.protect(v) + logger.info("${v} -> ${encryptedValue}") + def (String iv, String cipherText) = encryptedValue.tokenize("||") + logger.info("Normal Base64 encoding would be ${encoder.encodeToString(decoder.decode(iv))}||${encoder.encodeToString(decoder.decode(cipherText))}") + encryptedValue + } + + // Assert + assert values == encryptedValues.collect { spp.unprotect(it) } + } + + /** + * This test is to ensure external compatibility in case someone encodes the encrypted value with Base64 and does not remove the padding + */ + @Test + public void testShouldDecryptPaddedValue() { + // Arrange + Assume.assumeTrue("JCE unlimited strength crypto policy must be installed for this test", Cipher.getMaxAllowedKeyLength("AES") > 128) + + final String EXPECTED_VALUE = "thisIsABadKeyPassword" + String cipherText = "ac/BaE35SL/esLiJ||+ULRvRLYdIDA2VqpE0eQXDEMjaLBMG2kbKOdOwBk/hGebDKlVg==" + String unpaddedCipherText = cipherText.replaceAll("=", "") + + String key = getKeyOfSize(256) + + SensitivePropertyProvider spp = new AESSensitivePropertyProvider(key) + + // Act + String rawValue = spp.unprotect(cipherText) + logger.info("Decrypted ${cipherText} to ${rawValue}") + String rawUnpaddedValue = spp.unprotect(unpaddedCipherText) + logger.info("Decrypted ${unpaddedCipherText} to ${rawUnpaddedValue}") + + // Assert + assert rawValue == EXPECTED_VALUE + assert rawUnpaddedValue == EXPECTED_VALUE + } +} diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-properties-loader/src/test/groovy/org/apache/nifi/properties/NiFiPropertiesLoaderGroovyTest.groovy b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-properties-loader/src/test/groovy/org/apache/nifi/properties/NiFiPropertiesLoaderGroovyTest.groovy new file mode 100644 index 0000000000..e1a0c65c37 --- /dev/null +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-properties-loader/src/test/groovy/org/apache/nifi/properties/NiFiPropertiesLoaderGroovyTest.groovy @@ -0,0 +1,393 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.properties + +import org.apache.nifi.util.NiFiProperties +import org.bouncycastle.jce.provider.BouncyCastleProvider +import org.junit.After +import org.junit.AfterClass +import org.junit.Before +import org.junit.BeforeClass +import org.junit.Test +import org.junit.runner.RunWith +import org.junit.runners.JUnit4 +import org.slf4j.Logger +import org.slf4j.LoggerFactory + +import javax.crypto.Cipher +import java.nio.file.Files +import java.nio.file.attribute.PosixFilePermission +import java.security.Security + +@RunWith(JUnit4.class) +class NiFiPropertiesLoaderGroovyTest extends GroovyTestCase { + private static final Logger logger = LoggerFactory.getLogger(NiFiPropertiesLoaderGroovyTest.class) + + final def DEFAULT_SENSITIVE_PROPERTIES = [ + "nifi.sensitive.props.key", + "nifi.security.keystorePasswd", + "nifi.security.keyPasswd", + "nifi.security.truststorePasswd" + ] + + final def COMMON_ADDITIONAL_SENSITIVE_PROPERTIES = [ + "nifi.sensitive.props.algorithm", + "nifi.kerberos.service.principal", + "nifi.kerberos.krb5.file", + "nifi.kerberos.keytab.location" + ] + + private static final String KEY_HEX = "0123456789ABCDEFFEDCBA9876543210" * 2 + + private static String originalPropertiesPath = System.getProperty(NiFiProperties.PROPERTIES_FILE_PATH) + private + final Set ownerReadWrite = [PosixFilePermission.OWNER_WRITE, PosixFilePermission.OWNER_READ] + + @BeforeClass + public static void setUpOnce() throws Exception { + Security.addProvider(new BouncyCastleProvider()) + + logger.metaClass.methodMissing = { String name, args -> + logger.info("[${name?.toUpperCase()}] ${(args as List).join(" ")}") + } + } + + @Before + public void setUp() throws Exception { + } + + @After + public void tearDown() throws Exception { + // Clear the sensitive property providers between runs +// if (ProtectedNiFiProperties.@localProviderCache) { +// ProtectedNiFiProperties.@localProviderCache = [:] +// } + NiFiPropertiesLoader.@sensitivePropertyProviderFactory = null + } + + @AfterClass + public static void tearDownOnce() { + if (originalPropertiesPath) { + System.setProperty(NiFiProperties.PROPERTIES_FILE_PATH, originalPropertiesPath) + } + } + + @Test + public void testConstructorShouldCreateNewInstance() throws Exception { + // Arrange + + // Act + NiFiPropertiesLoader niFiPropertiesLoader = new NiFiPropertiesLoader() + + // Assert + assert !niFiPropertiesLoader.@instance + assert !niFiPropertiesLoader.@keyHex + } + + @Test + public void testShouldCreateInstanceWithKey() throws Exception { + // Arrange + + // Act + NiFiPropertiesLoader niFiPropertiesLoader = NiFiPropertiesLoader.withKey(KEY_HEX) + + // Assert + assert !niFiPropertiesLoader.@instance + assert niFiPropertiesLoader.@keyHex == KEY_HEX + } + + @Test + public void testShouldGetDefaultProviderKey() throws Exception { + // Arrange + final String EXPECTED_PROVIDER_KEY = "aes/gcm/${Cipher.getMaxAllowedKeyLength("AES") > 128 ? 256 : 128}" + logger.info("Expected provider key: ${EXPECTED_PROVIDER_KEY}") + + // Act + String defaultKey = NiFiPropertiesLoader.getDefaultProviderKey() + logger.info("Default key: ${defaultKey}") + // Assert + assert defaultKey == EXPECTED_PROVIDER_KEY + } + + @Test + public void testShouldInitializeSensitivePropertyProviderFactory() throws Exception { + // Arrange + NiFiPropertiesLoader niFiPropertiesLoader = new NiFiPropertiesLoader() + + // Act + niFiPropertiesLoader.initializeSensitivePropertyProviderFactory() + + // Assert + assert niFiPropertiesLoader.@sensitivePropertyProviderFactory + } + + @Test + public void testShouldLoadUnprotectedPropertiesFromFile() throws Exception { + // Arrange + File unprotectedFile = new File("src/test/resources/conf/nifi.properties") + NiFiPropertiesLoader niFiPropertiesLoader = new NiFiPropertiesLoader() + + // Act + NiFiProperties niFiProperties = niFiPropertiesLoader.load(unprotectedFile) + + // Assert + assert niFiProperties.size() > 0 + + // Ensure it is not a ProtectedNiFiProperties + assert niFiProperties instanceof StandardNiFiProperties + } + + @Test + public void testShouldNotLoadUnprotectedPropertiesFromNullFile() throws Exception { + // Arrange + NiFiPropertiesLoader niFiPropertiesLoader = new NiFiPropertiesLoader() + + // Act + def msg = shouldFail(IllegalArgumentException) { + NiFiProperties niFiProperties = niFiPropertiesLoader.load(null as File) + } + logger.expected(msg) + + // Assert + assert msg == "NiFi properties file missing or unreadable" + } + + @Test + public void testShouldNotLoadUnprotectedPropertiesFromMissingFile() throws Exception { + // Arrange + File missingFile = new File("src/test/resources/conf/nifi_missing.properties") + assert !missingFile.exists() + + NiFiPropertiesLoader niFiPropertiesLoader = new NiFiPropertiesLoader() + + // Act + def msg = shouldFail(IllegalArgumentException) { + NiFiProperties niFiProperties = niFiPropertiesLoader.load(missingFile) + } + logger.expected(msg) + + // Assert + assert msg == "NiFi properties file missing or unreadable" + } + + @Test + public void testShouldNotLoadUnprotectedPropertiesFromUnreadableFile() throws Exception { + // Arrange + File unreadableFile = new File("src/test/resources/conf/nifi_no_permissions.properties") + Files.setPosixFilePermissions(unreadableFile.toPath(), [] as Set) + assert !unreadableFile.canRead() + + NiFiPropertiesLoader niFiPropertiesLoader = new NiFiPropertiesLoader() + + // Act + def msg = shouldFail(IllegalArgumentException) { + NiFiProperties niFiProperties = niFiPropertiesLoader.load(unreadableFile) + } + logger.expected(msg) + + // Assert + assert msg == "NiFi properties file missing or unreadable" + + // Clean up to allow for indexing, etc. + Files.setPosixFilePermissions(unreadableFile.toPath(), ownerReadWrite) + } + + @Test + public void testShouldLoadUnprotectedPropertiesFromPath() throws Exception { + // Arrange + File unprotectedFile = new File("src/test/resources/conf/nifi.properties") + NiFiPropertiesLoader niFiPropertiesLoader = new NiFiPropertiesLoader() + + // Act + NiFiProperties niFiProperties = niFiPropertiesLoader.load(unprotectedFile.path) + + // Assert + assert niFiProperties.size() > 0 + + // Ensure it is not a ProtectedNiFiProperties + assert niFiProperties instanceof StandardNiFiProperties + } + + @Test + public void testShouldLoadUnprotectedPropertiesFromProtectedFile() throws Exception { + // Arrange + File protectedFile = new File("src/test/resources/conf/nifi_with_sensitive_properties_protected_aes.properties") + NiFiPropertiesLoader niFiPropertiesLoader = NiFiPropertiesLoader.withKey(KEY_HEX) + + final def EXPECTED_PLAIN_VALUES = [ + (NiFiProperties.SENSITIVE_PROPS_KEY): "thisIsABadSensitiveKeyPassword", + (NiFiProperties.SECURITY_KEYSTORE_PASSWD): "thisIsABadKeystorePassword", + (NiFiProperties.SECURITY_KEY_PASSWD): "thisIsABadKeyPassword", + ] + + // This method is covered in tests above, so safe to use here to retrieve protected properties + ProtectedNiFiProperties protectedNiFiProperties = niFiPropertiesLoader.readProtectedPropertiesFromDisk(protectedFile) + int totalKeysCount = protectedNiFiProperties.getPropertyKeysIncludingProtectionSchemes().size() + int protectedKeysCount = protectedNiFiProperties.getProtectedPropertyKeys().size() + logger.info("Read ${totalKeysCount} total properties (${protectedKeysCount} protected) from ${protectedFile.canonicalPath}") + + // Act + NiFiProperties niFiProperties = niFiPropertiesLoader.load(protectedFile) + + // Assert + assert niFiProperties.size() == totalKeysCount - protectedKeysCount + + // Ensure that any key marked as protected above is different in this instance + protectedNiFiProperties.getProtectedPropertyKeys().keySet().each { String key -> + String plainValue = niFiProperties.getProperty(key) + String protectedValue = protectedNiFiProperties.getProperty(key) + + logger.info("Checking that [${protectedValue}] -> [${plainValue}] == [${EXPECTED_PLAIN_VALUES[key]}]") + + assert plainValue == EXPECTED_PLAIN_VALUES[key] + assert plainValue != protectedValue + assert plainValue.length() <= protectedValue.length() + } + + // Ensure it is not a ProtectedNiFiProperties + assert niFiProperties instanceof StandardNiFiProperties + } + + @Test + public void testShouldExtractKeyFromBootstrapFile() throws Exception { + // Arrange + def defaultNiFiPropertiesFilePath = "src/test/resources/bootstrap_tests/conf/nifi.properties" + System.setProperty(NiFiProperties.PROPERTIES_FILE_PATH, defaultNiFiPropertiesFilePath) + + // Act + String key = NiFiPropertiesLoader.extractKeyFromBootstrapFile() + + // Assert + assert key == KEY_HEX + } + + @Test + public void testShouldNotExtractKeyFromBootstrapFileWithoutKeyLine() throws Exception { + // Arrange + def defaultNiFiPropertiesFilePath = "src/test/resources/bootstrap_tests/missing_key_line/nifi.properties" + System.setProperty(NiFiProperties.PROPERTIES_FILE_PATH, defaultNiFiPropertiesFilePath) + + // Act + String key = NiFiPropertiesLoader.extractKeyFromBootstrapFile() + + // Assert + assert key == "" + } + + @Test + public void testShouldNotExtractKeyFromBootstrapFileWithoutKey() throws Exception { + // Arrange + def defaultNiFiPropertiesFilePath = "src/test/resources/bootstrap_tests/missing_key_line/nifi.properties" + System.setProperty(NiFiProperties.PROPERTIES_FILE_PATH, defaultNiFiPropertiesFilePath) + + // Act + String key = NiFiPropertiesLoader.extractKeyFromBootstrapFile() + + // Assert + assert key == "" + } + + @Test + public void testShouldNotExtractKeyFromMissingBootstrapFile() throws Exception { + // Arrange + def defaultNiFiPropertiesFilePath = "src/test/resources/bootstrap_tests/missing_bootstrap/nifi.properties" + System.setProperty(NiFiProperties.PROPERTIES_FILE_PATH, defaultNiFiPropertiesFilePath) + + // Act + def msg = shouldFail(IOException) { + String key = NiFiPropertiesLoader.extractKeyFromBootstrapFile() + } + logger.expected(msg) + + // Assert + assert msg == "Cannot read from bootstrap.conf" + } + + @Test + public void testShouldNotExtractKeyFromUnreadableBootstrapFile() throws Exception { + // Arrange + File unreadableFile = new File("src/test/resources/bootstrap_tests/unreadable_bootstrap/bootstrap.conf") + Set originalPermissions = Files.getPosixFilePermissions(unreadableFile.toPath()) + Files.setPosixFilePermissions(unreadableFile.toPath(), [] as Set) + assert !unreadableFile.canRead() + + def defaultNiFiPropertiesFilePath = "src/test/resources/bootstrap_tests/unreadable_bootstrap/nifi.properties" + System.setProperty(NiFiProperties.PROPERTIES_FILE_PATH, defaultNiFiPropertiesFilePath) + + // Act + def msg = shouldFail(IOException) { + String key = NiFiPropertiesLoader.extractKeyFromBootstrapFile() + } + logger.expected(msg) + + // Assert + assert msg == "Cannot read from bootstrap.conf" + + // Clean up to allow for indexing, etc. + Files.setPosixFilePermissions(unreadableFile.toPath(), originalPermissions) + } + + @Test + public void testShouldNotExtractKeyFromUnreadableConfDir() throws Exception { + // Arrange + File unreadableDir = new File("src/test/resources/bootstrap_tests/unreadable_conf") + Set originalPermissions = Files.getPosixFilePermissions(unreadableDir.toPath()) + Files.setPosixFilePermissions(unreadableDir.toPath(), [] as Set) + assert !unreadableDir.canRead() + + def defaultNiFiPropertiesFilePath = "src/test/resources/bootstrap_tests/unreadable_conf/nifi.properties" + System.setProperty(NiFiProperties.PROPERTIES_FILE_PATH, defaultNiFiPropertiesFilePath) + + // Act + def msg = shouldFail(IOException) { + String key = NiFiPropertiesLoader.extractKeyFromBootstrapFile() + } + logger.expected(msg) + + // Assert + assert msg == "Cannot read from bootstrap.conf" + + // Clean up to allow for indexing, etc. + Files.setPosixFilePermissions(unreadableDir.toPath(), originalPermissions) + } + + @Test + public void testShouldLoadUnprotectedPropertiesFromProtectedDefaultFileAndUseBootstrapKey() throws Exception { + // Arrange + File protectedFile = new File("src/test/resources/bootstrap_tests/conf/nifi_with_sensitive_properties_protected_aes.properties") + System.setProperty(NiFiProperties.PROPERTIES_FILE_PATH, protectedFile.path) + NiFiPropertiesLoader niFiPropertiesLoader = NiFiPropertiesLoader.withKey(KEY_HEX) + + NiFiProperties normalReadProperties = niFiPropertiesLoader.load(protectedFile) + logger.info("Read ${normalReadProperties.size()} total properties from ${protectedFile.canonicalPath}") + + // Act + NiFiProperties niFiProperties = NiFiPropertiesLoader.loadDefaultWithKeyFromBootstrap() + + // Assert + assert niFiProperties.size() == normalReadProperties.size() + + + def readPropertiesAndValues = niFiProperties.getPropertyKeys().collectEntries { + [(it): niFiProperties.getProperty(it)] + } + def expectedPropertiesAndValues = normalReadProperties.getPropertyKeys().collectEntries { + [(it): normalReadProperties.getProperty(it)] + } + assert readPropertiesAndValues == expectedPropertiesAndValues + } +} diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-properties-loader/src/test/groovy/org/apache/nifi/properties/ProtectedNiFiPropertiesGroovyTest.groovy b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-properties-loader/src/test/groovy/org/apache/nifi/properties/ProtectedNiFiPropertiesGroovyTest.groovy new file mode 100644 index 0000000000..bf4e677840 --- /dev/null +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-properties-loader/src/test/groovy/org/apache/nifi/properties/ProtectedNiFiPropertiesGroovyTest.groovy @@ -0,0 +1,860 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.properties + +import org.apache.nifi.util.NiFiProperties +import org.bouncycastle.jce.provider.BouncyCastleProvider +import org.junit.After +import org.junit.AfterClass +import org.junit.Before +import org.junit.BeforeClass +import org.junit.Test +import org.junit.runner.RunWith +import org.junit.runners.JUnit4 +import org.slf4j.Logger +import org.slf4j.LoggerFactory + +import java.security.Security + +@RunWith(JUnit4.class) +class ProtectedNiFiPropertiesGroovyTest extends GroovyTestCase { + private static final Logger logger = LoggerFactory.getLogger(ProtectedNiFiPropertiesGroovyTest.class) + + final def DEFAULT_SENSITIVE_PROPERTIES = [ + "nifi.sensitive.props.key", + "nifi.security.keystorePasswd", + "nifi.security.keyPasswd", + "nifi.security.truststorePasswd" + ] + + final def COMMON_ADDITIONAL_SENSITIVE_PROPERTIES = [ + "nifi.sensitive.props.algorithm", + "nifi.kerberos.service.principal", + "nifi.kerberos.krb5.file", + "nifi.kerberos.keytab.location" + ] + + private static final String KEY_HEX = "0123456789ABCDEFFEDCBA9876543210" * 2 + + private static String originalPropertiesPath = System.getProperty(NiFiProperties.PROPERTIES_FILE_PATH) + + @BeforeClass + public static void setUpOnce() throws Exception { + Security.addProvider(new BouncyCastleProvider()) + + logger.metaClass.methodMissing = { String name, args -> + logger.info("[${name?.toUpperCase()}] ${(args as List).join(" ")}") + } + } + + @Before + public void setUp() throws Exception { + } + + @After + public void tearDown() throws Exception { + } + + @AfterClass + public static void tearDownOnce() { + if (originalPropertiesPath) { + System.setProperty(NiFiProperties.PROPERTIES_FILE_PATH, originalPropertiesPath) + } + } + + private static ProtectedNiFiProperties loadFromFile(String propertiesFilePath) { + String filePath + try { + filePath = ProtectedNiFiPropertiesGroovyTest.class.getResource(propertiesFilePath).toURI().getPath() + } catch (URISyntaxException ex) { + throw new RuntimeException("Cannot load properties file due to " + + ex.getLocalizedMessage(), ex) + } + + File file = new File(filePath) + + if (file == null || !file.exists() || !file.canRead()) { + String path = (file == null ? "missing file" : file.getAbsolutePath()) + logger.error("Cannot read from '{}' -- file is missing or not readable", path) + throw new IllegalArgumentException("NiFi properties file missing or unreadable") + } + + Properties rawProperties = new Properties() + + InputStream inStream = null + try { + inStream = new BufferedInputStream(new FileInputStream(file)) + rawProperties.load(inStream) + logger.info("Loaded {} properties from {}", rawProperties.size(), file.getAbsolutePath()) + + ProtectedNiFiProperties protectedNiFiProperties = new ProtectedNiFiProperties(rawProperties) + + // If it has protected keys, inject the SPP + if (protectedNiFiProperties.hasProtectedKeys()) { + protectedNiFiProperties.addSensitivePropertyProvider(new AESSensitivePropertyProvider(KEY_HEX)) + } + + return protectedNiFiProperties + } catch (final Exception ex) { + logger.error("Cannot load properties file due to " + ex.getLocalizedMessage()) + throw new RuntimeException("Cannot load properties file due to " + + ex.getLocalizedMessage(), ex) + } finally { + if (null != inStream) { + try { + inStream.close() + } catch (final Exception ex) { + /** + * do nothing * + */ + } + } + } + } + + @Test + public void testConstructorShouldCreateNewInstance() throws Exception { + // Arrange + + // Act + NiFiProperties niFiProperties = new StandardNiFiProperties() + logger.info("niFiProperties has ${niFiProperties.size()} properties: ${niFiProperties.getPropertyKeys()}") + + // Assert + assert niFiProperties.size() == 0 + assert niFiProperties.getPropertyKeys() == [] as Set + } + + @Test + public void testConstructorShouldAcceptRawProperties() throws Exception { + // Arrange + Properties rawProperties = new Properties() + rawProperties.setProperty("key", "value") + logger.info("rawProperties has ${rawProperties.size()} properties: ${rawProperties.stringPropertyNames()}") + assert rawProperties.size() == 1 + + // Act + NiFiProperties niFiProperties = new StandardNiFiProperties(rawProperties) + logger.info("niFiProperties has ${niFiProperties.size()} properties: ${niFiProperties.getPropertyKeys()}") + + // Assert + assert niFiProperties.size() == 1 + assert niFiProperties.getPropertyKeys() == ["key"] as Set + } + + @Test + public void testConstructorShouldAcceptNiFiProperties() throws Exception { + // Arrange + Properties rawProperties = new Properties() + rawProperties.setProperty("key", "value") + rawProperties.setProperty("key.protected", "value2") + NiFiProperties niFiProperties = new StandardNiFiProperties(rawProperties) + logger.info("niFiProperties has ${niFiProperties.size()} properties: ${niFiProperties.getPropertyKeys()}") + assert niFiProperties.size() == 2 + + // Act + ProtectedNiFiProperties protectedNiFiProperties = new ProtectedNiFiProperties(niFiProperties) + logger.info("protectedNiFiProperties has ${protectedNiFiProperties.size()} properties: ${protectedNiFiProperties.getPropertyKeys()}") + + // Assert + def allKeys = protectedNiFiProperties.getPropertyKeysIncludingProtectionSchemes() + assert allKeys == ["key", "key.protected"] as Set + assert allKeys.size() == niFiProperties.size() + + } + + @Test + public void testShouldAllowMultipleInstances() throws Exception { + // Arrange + Properties rawProperties = new Properties() + rawProperties.setProperty("key", "value") + logger.info("rawProperties has ${rawProperties.size()} properties: ${rawProperties.stringPropertyNames()}") + assert rawProperties.size() == 1 + + // Act + NiFiProperties niFiProperties = new StandardNiFiProperties(rawProperties) + logger.info("niFiProperties has ${niFiProperties.size()} properties: ${niFiProperties.getPropertyKeys()}") + NiFiProperties emptyProperties = new StandardNiFiProperties() + logger.info("emptyProperties has ${emptyProperties.size()} properties: ${emptyProperties.getPropertyKeys()}") + + // Assert + assert niFiProperties.size() == 1 + assert niFiProperties.getPropertyKeys() == ["key"] as Set + + assert emptyProperties.size() == 0 + assert emptyProperties.getPropertyKeys() == [] as Set + } + + @Test + public void testShouldDetectIfPropertyIsSensitive() throws Exception { + // Arrange + final String INSENSITIVE_PROPERTY_KEY = "nifi.ui.banner.text" + final String SENSITIVE_PROPERTY_KEY = "nifi.security.keystorePasswd" + + ProtectedNiFiProperties properties = loadFromFile("/conf/nifi.properties") + + // Act + boolean bannerIsSensitive = properties.isPropertySensitive(INSENSITIVE_PROPERTY_KEY) + logger.info("${INSENSITIVE_PROPERTY_KEY} is ${bannerIsSensitive ? "SENSITIVE" : "NOT SENSITIVE"}") + boolean passwordIsSensitive = properties.isPropertySensitive(SENSITIVE_PROPERTY_KEY) + logger.info("${SENSITIVE_PROPERTY_KEY} is ${passwordIsSensitive ? "SENSITIVE" : "NOT SENSITIVE"}") + + // Assert + assert !bannerIsSensitive + assert passwordIsSensitive + } + + @Test + public void testShouldGetDefaultSensitiveProperties() throws Exception { + // Arrange + logger.expected("${DEFAULT_SENSITIVE_PROPERTIES.size()} default sensitive properties: ${DEFAULT_SENSITIVE_PROPERTIES.join(", ")}") + + ProtectedNiFiProperties properties = loadFromFile("/conf/nifi.properties") + + // Act + List defaultSensitiveProperties = properties.getSensitivePropertyKeys() + logger.info("${defaultSensitiveProperties.size()} default sensitive properties: ${defaultSensitiveProperties.join(", ")}") + + // Assert + assert defaultSensitiveProperties.size() == DEFAULT_SENSITIVE_PROPERTIES.size() + assert defaultSensitiveProperties.containsAll(DEFAULT_SENSITIVE_PROPERTIES) + } + + @Test + public void testShouldGetAdditionalSensitiveProperties() throws Exception { + // Arrange + def completeSensitiveProperties = DEFAULT_SENSITIVE_PROPERTIES + ["nifi.ui.banner.text", "nifi.version"] + logger.expected("${completeSensitiveProperties.size()} total sensitive properties: ${completeSensitiveProperties.join(", ")}") + + ProtectedNiFiProperties properties = loadFromFile("/conf/nifi_with_additional_sensitive_keys.properties") + + // Act + List retrievedSensitiveProperties = properties.getSensitivePropertyKeys() + logger.info("${retrievedSensitiveProperties.size()} retrieved sensitive properties: ${retrievedSensitiveProperties.join(", ")}") + + // Assert + assert retrievedSensitiveProperties.size() == completeSensitiveProperties.size() + assert retrievedSensitiveProperties.containsAll(completeSensitiveProperties) + } + + // TODO: Add negative tests (fuzz additional.keys property, etc.) + + @Test + public void testGetAdditionalSensitivePropertiesShouldNotIncludeSelf() throws Exception { + // Arrange + def completeSensitiveProperties = DEFAULT_SENSITIVE_PROPERTIES + ["nifi.ui.banner.text", "nifi.version"] + logger.expected("${completeSensitiveProperties.size()} total sensitive properties: ${completeSensitiveProperties.join(", ")}") + + ProtectedNiFiProperties properties = loadFromFile("/conf/nifi_with_additional_sensitive_keys.properties") + + // Act + List retrievedSensitiveProperties = properties.getSensitivePropertyKeys() + logger.info("${retrievedSensitiveProperties.size()} retrieved sensitive properties: ${retrievedSensitiveProperties.join(", ")}") + + // Assert + assert retrievedSensitiveProperties.size() == completeSensitiveProperties.size() + assert retrievedSensitiveProperties.containsAll(completeSensitiveProperties) + } + + /** + * In the default (no protection enabled) scenario, a call to retrieve a sensitive property should return the raw value transparently. + * @throws Exception + */ + @Test + public void testShouldGetUnprotectedValueOfSensitiveProperty() throws Exception { + // Arrange + final String KEYSTORE_PASSWORD_KEY = "nifi.security.keystorePasswd" + final String EXPECTED_KEYSTORE_PASSWORD = "thisIsABadKeystorePassword" + + ProtectedNiFiProperties properties = loadFromFile("/conf/nifi_with_sensitive_properties_unprotected.properties") + + boolean isSensitive = properties.isPropertySensitive(KEYSTORE_PASSWORD_KEY) + boolean isProtected = properties.isPropertyProtected(KEYSTORE_PASSWORD_KEY) + logger.info("The property is ${isSensitive ? "sensitive" : "not sensitive"} and ${isProtected ? "protected" : "not protected"}") + + // Act + String retrievedKeystorePassword = properties.getProperty(KEYSTORE_PASSWORD_KEY) + logger.info("${KEYSTORE_PASSWORD_KEY}: ${retrievedKeystorePassword}") + + // Assert + assert retrievedKeystorePassword == EXPECTED_KEYSTORE_PASSWORD + assert isSensitive + assert !isProtected + } + + /** + * In the default (no protection enabled) scenario, a call to retrieve a sensitive property (which is empty) should return the raw value transparently. + * @throws Exception + */ + @Test + public void testShouldGetEmptyUnprotectedValueOfSensitiveProperty() throws Exception { + // Arrange + final String TRUSTSTORE_PASSWORD_KEY = "nifi.security.truststorePasswd" + final String EXPECTED_TRUSTSTORE_PASSWORD = "" + + ProtectedNiFiProperties properties = loadFromFile("/conf/nifi_with_sensitive_properties_unprotected.properties") + + boolean isSensitive = properties.isPropertySensitive(TRUSTSTORE_PASSWORD_KEY) + boolean isProtected = properties.isPropertyProtected(TRUSTSTORE_PASSWORD_KEY) + logger.info("The property is ${isSensitive ? "sensitive" : "not sensitive"} and ${isProtected ? "protected" : "not protected"}") + + // Act + NiFiProperties unprotectedProperties = properties.getUnprotectedProperties() + String retrievedTruststorePassword = unprotectedProperties.getProperty(TRUSTSTORE_PASSWORD_KEY) + logger.info("${TRUSTSTORE_PASSWORD_KEY}: ${retrievedTruststorePassword}") + + // Assert + assert retrievedTruststorePassword == EXPECTED_TRUSTSTORE_PASSWORD + assert isSensitive + assert !isProtected + } + + /** + * The new model no longer needs to maintain the protected state -- it is used as a wrapper/decorator during load to unprotect the sensitive properties and then return an instance of raw properties. + * + * @throws Exception + */ + @Test + public void testShouldGetUnprotectedValueOfSensitivePropertyWhenProtected() throws Exception { + // Arrange + final String KEYSTORE_PASSWORD_KEY = "nifi.security.keystorePasswd" + final String EXPECTED_KEYSTORE_PASSWORD = "thisIsABadKeystorePassword" + + ProtectedNiFiProperties properties = loadFromFile("/conf/nifi_with_sensitive_properties_protected_aes.properties") + + boolean isSensitive = properties.isPropertySensitive(KEYSTORE_PASSWORD_KEY) + boolean isProtected = properties.isPropertyProtected(KEYSTORE_PASSWORD_KEY) + logger.info("The property is ${isSensitive ? "sensitive" : "not sensitive"} and ${isProtected ? "protected" : "not protected"}") + + // Act + NiFiProperties unprotectedProperties = properties.getUnprotectedProperties() + String retrievedKeystorePassword = unprotectedProperties.getProperty(KEYSTORE_PASSWORD_KEY) + logger.info("${KEYSTORE_PASSWORD_KEY}: ${retrievedKeystorePassword}") + + // Assert + assert retrievedKeystorePassword == EXPECTED_KEYSTORE_PASSWORD + assert isSensitive + assert isProtected + } + + /** + * In the protection enabled scenario, a call to retrieve a sensitive property should handle if the property is protected with an unknown protection scheme. + * @throws Exception + */ + @Test + public void testGetValueOfSensitivePropertyShouldHandleUnknownProtectionScheme() throws Exception { + // Arrange + final String KEYSTORE_PASSWORD_KEY = "nifi.security.keystorePasswd" + + // Raw properties + Properties rawProperties = new Properties() + rawProperties.load(new File("src/test/resources/conf/nifi_with_sensitive_properties_protected_unknown.properties").newInputStream()) + final String RAW_KEYSTORE_PASSWORD = rawProperties.getProperty(KEYSTORE_PASSWORD_KEY) + logger.info("Raw value for ${KEYSTORE_PASSWORD_KEY}: ${RAW_KEYSTORE_PASSWORD}") + + ProtectedNiFiProperties properties = loadFromFile("/conf/nifi_with_sensitive_properties_protected_unknown.properties") + + boolean isSensitive = properties.isPropertySensitive(KEYSTORE_PASSWORD_KEY) + boolean isProtected = properties.isPropertyProtected(KEYSTORE_PASSWORD_KEY) + + // While the value is "protected", the scheme is not registered, so treat it as raw + logger.info("The property is ${isSensitive ? "sensitive" : "not sensitive"} and ${isProtected ? "protected" : "not protected"}") + + // Act + NiFiProperties unprotectedProperties = properties.getUnprotectedProperties() + String retrievedKeystorePassword = unprotectedProperties.getProperty(KEYSTORE_PASSWORD_KEY) + logger.info("${KEYSTORE_PASSWORD_KEY}: ${retrievedKeystorePassword}") + + // Assert + assert retrievedKeystorePassword == RAW_KEYSTORE_PASSWORD + assert isSensitive + assert isProtected + } + + /** + * In the protection enabled scenario, a call to retrieve a sensitive property should handle if the property is unable to be unprotected due to a malformed value. + * @throws Exception + */ + @Test + public void testGetValueOfSensitivePropertyShouldHandleSingleMalformedValue() throws Exception { + // Arrange + final String KEYSTORE_PASSWORD_KEY = "nifi.security.keystorePasswd" + + // Raw properties + Properties rawProperties = new Properties() + rawProperties.load(new File("src/test/resources/conf/nifi_with_sensitive_properties_protected_aes_single_malformed.properties").newInputStream()) + final String RAW_KEYSTORE_PASSWORD = rawProperties.getProperty(KEYSTORE_PASSWORD_KEY) + logger.info("Raw value for ${KEYSTORE_PASSWORD_KEY}: ${RAW_KEYSTORE_PASSWORD}") + + ProtectedNiFiProperties properties = loadFromFile("/conf/nifi_with_sensitive_properties_protected_aes_single_malformed.properties") + + boolean isSensitive = properties.isPropertySensitive(KEYSTORE_PASSWORD_KEY) + boolean isProtected = properties.isPropertyProtected(KEYSTORE_PASSWORD_KEY) + logger.info("The property is ${isSensitive ? "sensitive" : "not sensitive"} and ${isProtected ? "protected" : "not protected"}") + + // Act + def msg = shouldFail(SensitivePropertyProtectionException) { + NiFiProperties unprotectedProperties = properties.getUnprotectedProperties() + String retrievedKeystorePassword = unprotectedProperties.getProperty(KEYSTORE_PASSWORD_KEY) + logger.info("${KEYSTORE_PASSWORD_KEY}: ${retrievedKeystorePassword}") + } + logger.expected(msg) + + // Assert + assert msg =~ "Failed to unprotect key ${KEYSTORE_PASSWORD_KEY}" + assert isSensitive + assert isProtected + } + + /** + * In the protection enabled scenario, a call to retrieve a sensitive property should handle if the property is unable to be unprotected due to a malformed value. + * @throws Exception + */ + @Test + public void testGetValueOfSensitivePropertyShouldHandleMultipleMalformedValues() throws Exception { + // Arrange + + // Raw properties + Properties rawProperties = new Properties() + rawProperties.load(new File("src/test/resources/conf/nifi_with_sensitive_properties_protected_aes_multiple_malformed.properties").newInputStream()) + + ProtectedNiFiProperties properties = loadFromFile("/conf/nifi_with_sensitive_properties_protected_aes_multiple_malformed.properties") + + // Iterate over the protected keys and track the ones that fail to decrypt + SensitivePropertyProvider spp = new AESSensitivePropertyProvider(KEY_HEX) + Set malformedKeys = properties.getProtectedPropertyKeys() + .findAll { String key, String scheme -> scheme == spp.identifierKey } + .keySet().collect { String key -> + try { + String rawValue = spp.unprotect(properties.getProperty(key)) + return + } catch (SensitivePropertyProtectionException e) { + logger.expected("Caught a malformed value for ${key}") + return key + } + } + + logger.expected("Malformed keys: ${malformedKeys.join(", ")}") + + // Act + def e = groovy.test.GroovyAssert.shouldFail(SensitivePropertyProtectionException) { + NiFiProperties unprotectedProperties = properties.getUnprotectedProperties() + } + logger.expected(e.getMessage()) + + // Assert + assert e instanceof MultipleSensitivePropertyProtectionException + assert e.getMessage() =~ "Failed to unprotect keys" + assert e.getFailedKeys() == malformedKeys + + } + + /** + * In the default (no protection enabled) scenario, a call to retrieve a sensitive property (which is empty) should return the raw value transparently. + * @throws Exception + */ + @Test + public void testShouldGetEmptyUnprotectedValueOfSensitivePropertyWithDefault() throws Exception { + // Arrange + final String TRUSTSTORE_PASSWORD_KEY = "nifi.security.truststorePasswd" + final String EXPECTED_TRUSTSTORE_PASSWORD = "" + final String DEFAULT_VALUE = "defaultValue" + + // Raw properties + Properties rawProperties = new Properties() + rawProperties.load(new File("src/test/resources/conf/nifi_with_sensitive_properties_unprotected.properties").newInputStream()) + final String RAW_TRUSTSTORE_PASSWORD = rawProperties.getProperty(TRUSTSTORE_PASSWORD_KEY) + logger.info("Raw value for ${TRUSTSTORE_PASSWORD_KEY}: ${RAW_TRUSTSTORE_PASSWORD}") + assert RAW_TRUSTSTORE_PASSWORD == EXPECTED_TRUSTSTORE_PASSWORD + + ProtectedNiFiProperties properties = loadFromFile("/conf/nifi_with_sensitive_properties_unprotected.properties") + + boolean isSensitive = properties.isPropertySensitive(TRUSTSTORE_PASSWORD_KEY) + boolean isProtected = properties.isPropertyProtected(TRUSTSTORE_PASSWORD_KEY) + logger.info("The property is ${isSensitive ? "sensitive" : "not sensitive"} and ${isProtected ? "protected" : "not protected"}") + + // Act + String retrievedTruststorePassword = properties.getProperty(TRUSTSTORE_PASSWORD_KEY, DEFAULT_VALUE) + logger.info("${TRUSTSTORE_PASSWORD_KEY}: ${retrievedTruststorePassword}") + + // Assert + assert retrievedTruststorePassword == DEFAULT_VALUE + assert isSensitive + assert !isProtected + } + + /** + * In the protection enabled scenario, a call to retrieve a sensitive property should return the raw value transparently. + * @throws Exception + */ + @Test + public void testShouldGetUnprotectedValueOfSensitivePropertyWhenProtectedWithDefault() throws Exception { + // Arrange + final String KEYSTORE_PASSWORD_KEY = "nifi.security.keystorePasswd" + final String EXPECTED_KEYSTORE_PASSWORD = "thisIsABadKeystorePassword" + final String DEFAULT_VALUE = "defaultValue" + + // Raw properties + Properties rawProperties = new Properties() + rawProperties.load(new File("src/test/resources/conf/nifi_with_sensitive_properties_protected_aes.properties").newInputStream()) + final String RAW_KEYSTORE_PASSWORD = rawProperties.getProperty(KEYSTORE_PASSWORD_KEY) + logger.info("Raw value for ${KEYSTORE_PASSWORD_KEY}: ${RAW_KEYSTORE_PASSWORD}") + + ProtectedNiFiProperties properties = loadFromFile("/conf/nifi_with_sensitive_properties_protected_aes.properties") + + boolean isSensitive = properties.isPropertySensitive(KEYSTORE_PASSWORD_KEY) + boolean isProtected = properties.isPropertyProtected(KEYSTORE_PASSWORD_KEY) + logger.info("The property is ${isSensitive ? "sensitive" : "not sensitive"} and ${isProtected ? "protected" : "not protected"}") + + // Act + NiFiProperties unprotectedProperties = properties.getUnprotectedProperties() + String retrievedKeystorePassword = unprotectedProperties.getProperty(KEYSTORE_PASSWORD_KEY, DEFAULT_VALUE) + logger.info("${KEYSTORE_PASSWORD_KEY}: ${retrievedKeystorePassword}") + + // Assert + assert retrievedKeystorePassword == EXPECTED_KEYSTORE_PASSWORD + assert isSensitive + assert isProtected + } + + // TODO: Test getProtected with multiple providers + + /** + * In the protection enabled scenario, a call to retrieve a sensitive property should handle if the internal cache of providers is empty. + * @throws Exception + */ + @Test + public void testGetValueOfSensitivePropertyShouldHandleInvalidatedInternalCache() throws Exception { + // Arrange + final String KEYSTORE_PASSWORD_KEY = "nifi.security.keystorePasswd" + final String EXPECTED_KEYSTORE_PASSWORD = "thisIsABadKeystorePassword" + + ProtectedNiFiProperties properties = loadFromFile("/conf/nifi_with_sensitive_properties_protected_aes.properties") + + final String RAW_PASSWORD = properties.getProperty(KEYSTORE_PASSWORD_KEY) + logger.info("Read raw value from properties: ${RAW_PASSWORD}") + + // Overwrite the internal cache + properties.localProviderCache = [:] + + boolean isSensitive = properties.isPropertySensitive(KEYSTORE_PASSWORD_KEY) + boolean isProtected = properties.isPropertyProtected(KEYSTORE_PASSWORD_KEY) + logger.info("The property is ${isSensitive ? "sensitive" : "not sensitive"} and ${isProtected ? "protected" : "not protected"}") + + // Act + NiFiProperties unprotectedProperties = properties.getUnprotectedProperties() + String retrievedKeystorePassword = unprotectedProperties.getProperty(KEYSTORE_PASSWORD_KEY) + logger.info("${KEYSTORE_PASSWORD_KEY}: ${retrievedKeystorePassword}") + + // Assert + assert retrievedKeystorePassword == RAW_PASSWORD + assert isSensitive + assert isProtected + } + + @Test + public void testShouldDetectIfPropertyIsProtected() throws Exception { + // Arrange + final String UNPROTECTED_PROPERTY_KEY = "nifi.security.truststorePasswd" + final String PROTECTED_PROPERTY_KEY = "nifi.security.keystorePasswd" + + ProtectedNiFiProperties properties = loadFromFile("/conf/nifi_with_sensitive_properties_protected_aes.properties") + + // Act + boolean unprotectedPasswordIsSensitive = properties.isPropertySensitive(UNPROTECTED_PROPERTY_KEY) + boolean unprotectedPasswordIsProtected = properties.isPropertyProtected(UNPROTECTED_PROPERTY_KEY) + logger.info("${UNPROTECTED_PROPERTY_KEY} is ${unprotectedPasswordIsSensitive ? "SENSITIVE" : "NOT SENSITIVE"}") + logger.info("${UNPROTECTED_PROPERTY_KEY} is ${unprotectedPasswordIsProtected ? "PROTECTED" : "NOT PROTECTED"}") + boolean protectedPasswordIsSensitive = properties.isPropertySensitive(PROTECTED_PROPERTY_KEY) + boolean protectedPasswordIsProtected = properties.isPropertyProtected(PROTECTED_PROPERTY_KEY) + logger.info("${PROTECTED_PROPERTY_KEY} is ${protectedPasswordIsSensitive ? "SENSITIVE" : "NOT SENSITIVE"}") + logger.info("${PROTECTED_PROPERTY_KEY} is ${protectedPasswordIsProtected ? "PROTECTED" : "NOT PROTECTED"}") + + // Assert + assert unprotectedPasswordIsSensitive + assert !unprotectedPasswordIsProtected + + assert protectedPasswordIsSensitive + assert protectedPasswordIsProtected + } + + @Test + public void testShouldDetectIfPropertyWithEmptyProtectionSchemeIsProtected() throws Exception { + // Arrange + final String UNPROTECTED_PROPERTY_KEY = "nifi.sensitive.props.key" + + ProtectedNiFiProperties properties = loadFromFile("/conf/nifi_with_sensitive_properties_unprotected_extra_line.properties") + + // Act + boolean unprotectedPasswordIsSensitive = properties.isPropertySensitive(UNPROTECTED_PROPERTY_KEY) + boolean unprotectedPasswordIsProtected = properties.isPropertyProtected(UNPROTECTED_PROPERTY_KEY) + logger.info("${UNPROTECTED_PROPERTY_KEY} is ${unprotectedPasswordIsSensitive ? "SENSITIVE" : "NOT SENSITIVE"}") + logger.info("${UNPROTECTED_PROPERTY_KEY} is ${unprotectedPasswordIsProtected ? "PROTECTED" : "NOT PROTECTED"}") + + // Assert + assert unprotectedPasswordIsSensitive + assert !unprotectedPasswordIsProtected + } + + @Test + public void testShouldGetPercentageOfSensitivePropertiesProtected_0() throws Exception { + // Arrange + ProtectedNiFiProperties properties = loadFromFile("/conf/nifi.properties") + + logger.info("Sensitive property keys: ${properties.getSensitivePropertyKeys()}") + logger.info("Protected property keys: ${properties.getProtectedPropertyKeys().keySet()}") + + // Act + double percentProtected = properties.getPercentOfSensitivePropertiesProtected() + logger.info("${percentProtected}% (${properties.getProtectedPropertyKeys().size()} of ${properties.getSensitivePropertyKeys().size()}) protected") + + // Assert + assert percentProtected == 0.0 + } + + @Test + public void testShouldGetPercentageOfSensitivePropertiesProtected_50() throws Exception { + // Arrange + ProtectedNiFiProperties properties = loadFromFile("/conf/nifi_with_sensitive_properties_protected_aes.properties") + + logger.info("Sensitive property keys: ${properties.getSensitivePropertyKeys()}") + logger.info("Protected property keys: ${properties.getProtectedPropertyKeys().keySet()}") + + // Act + double percentProtected = properties.getPercentOfSensitivePropertiesProtected() + logger.info("${percentProtected}% (${properties.getProtectedPropertyKeys().size()} of ${properties.getSensitivePropertyKeys().size()}) protected") + + // Assert + assert percentProtected == 50.0 + } + + @Test + public void testShouldGetPercentageOfSensitivePropertiesProtected_100() throws Exception { + // Arrange + ProtectedNiFiProperties properties = loadFromFile("/conf/nifi_with_all_sensitive_properties_protected_aes.properties") + + logger.info("Sensitive property keys: ${properties.getSensitivePropertyKeys()}") + logger.info("Protected property keys: ${properties.getProtectedPropertyKeys().keySet()}") + + // Act + double percentProtected = properties.getPercentOfSensitivePropertiesProtected() + logger.info("${percentProtected}% (${properties.getProtectedPropertyKeys().size()} of ${properties.getSensitivePropertyKeys().size()}) protected") + + // Assert + assert percentProtected == 100.0 + } + + @Test + public void testInstanceWithNoProtectedPropertiesShouldNotLoadSPP() throws Exception { + // Arrange + ProtectedNiFiProperties properties = loadFromFile("/conf/nifi.properties") + assert properties.@localProviderCache?.isEmpty() + + logger.info("Has protected properties: ${properties.hasProtectedKeys()}") + assert !properties.hasProtectedKeys() + + // Act + Map localCache = properties.@localProviderCache + logger.info("Internal cache ${localCache} has ${localCache.size()} providers loaded") + + // Assert + assert localCache.isEmpty() + } + + @Test + public void testShouldAddSensitivePropertyProvider() throws Exception { + // Arrange + ProtectedNiFiProperties properties = new ProtectedNiFiProperties() + assert properties.getSensitivePropertyProviders().isEmpty() + + SensitivePropertyProvider mockProvider = + [unprotect : { String input -> + logger.mock("Mock call to #unprotect(${input})") + input.reverse() + }, + getIdentifierKey: { -> "mockProvider" }] as SensitivePropertyProvider + + // Act + properties.addSensitivePropertyProvider(mockProvider) + + // Assert + assert properties.getSensitivePropertyProviders().size() == 1 + } + + @Test + public void testShouldNotAddNullSensitivePropertyProvider() throws Exception { + // Arrange + ProtectedNiFiProperties properties = new ProtectedNiFiProperties() + assert properties.getSensitivePropertyProviders().isEmpty() + + // Act + def msg = shouldFail(IllegalArgumentException) { + properties.addSensitivePropertyProvider(null) + } + logger.expected(msg) + + // Assert + assert properties.getSensitivePropertyProviders().size() == 0 + assert msg == "Cannot add null SensitivePropertyProvider" + } + + @Test + public void testShouldNotAllowOverwriteOfProvider() throws Exception { + // Arrange + ProtectedNiFiProperties properties = new ProtectedNiFiProperties() + assert properties.getSensitivePropertyProviders().isEmpty() + + SensitivePropertyProvider mockProvider = + [unprotect : { String input -> + logger.mock("Mock call to 1#unprotect(${input})") + input.reverse() + }, + getIdentifierKey: { -> "mockProvider" }] as SensitivePropertyProvider + properties.addSensitivePropertyProvider(mockProvider) + assert properties.getSensitivePropertyProviders().size() == 1 + + SensitivePropertyProvider mockProvider2 = + [unprotect : { String input -> + logger.mock("Mock call to 2#unprotect(${input})") + input.reverse() + }, + getIdentifierKey: { -> "mockProvider" }] as SensitivePropertyProvider + + // Act + def msg = shouldFail(UnsupportedOperationException) { + properties.addSensitivePropertyProvider(mockProvider2) + } + logger.expected(msg) + + // Assert + assert msg == "Cannot overwrite existing sensitive property provider registered for mockProvider" + assert properties.getSensitivePropertyProviders().size() == 1 + } + + @Test + void testGetUnprotectedPropertiesShouldReturnInternalInstanceWhenNoneProtected() { + // Arrange + String noProtectedPropertiesPath = "/conf/nifi.properties" + ProtectedNiFiProperties protectedNiFiProperties = loadFromFile(noProtectedPropertiesPath) + logger.info("Loaded ${protectedNiFiProperties.size()} properties from ${noProtectedPropertiesPath}") + + int hashCode = protectedNiFiProperties.internalNiFiProperties.hashCode() + logger.info("Hash code of internal instance: ${hashCode}") + + // Act + NiFiProperties unprotectedNiFiProperties = protectedNiFiProperties.getUnprotectedProperties() + logger.info("Unprotected ${unprotectedNiFiProperties.size()} properties") + + // Assert + assert unprotectedNiFiProperties.size() == protectedNiFiProperties.size() + assert unprotectedNiFiProperties.getPropertyKeys().every { + !unprotectedNiFiProperties.getProperty(it).endsWith(".protected") + } + logger.info("Hash code from returned unprotected instance: ${unprotectedNiFiProperties.hashCode()}") + assert unprotectedNiFiProperties.hashCode() == hashCode + } + + @Test + void testGetUnprotectedPropertiesShouldDecryptProtectedProperties() { + // Arrange + String noProtectedPropertiesPath = "/conf/nifi_with_sensitive_properties_protected_aes.properties" + ProtectedNiFiProperties protectedNiFiProperties = loadFromFile(noProtectedPropertiesPath) + logger.info("Loaded ${protectedNiFiProperties.size()} properties from ${noProtectedPropertiesPath}") + + int protectedPropertyCount = protectedNiFiProperties.getProtectedPropertyKeys().size() + int protectionSchemeCount = protectedNiFiProperties + .getPropertyKeys().findAll { it.endsWith(".protected") } + .size() + int expectedUnprotectedPropertyCount = protectedNiFiProperties.size() - protectionSchemeCount + + String protectedProps = protectedNiFiProperties + .getProtectedPropertyKeys() + .collectEntries { + [(it.key): protectedNiFiProperties.getProperty(it.key)] + }.entrySet() + .join("\n") + + logger.info("Detected ${protectedPropertyCount} protected properties and ${protectionSchemeCount} protection scheme properties") + logger.info("Protected properties: \n${protectedProps}") + + logger.info("Expected unprotected property count: ${expectedUnprotectedPropertyCount}") + + int hashCode = protectedNiFiProperties.internalNiFiProperties.hashCode() + logger.info("Hash code of internal instance: ${hashCode}") + + // Act + NiFiProperties unprotectedNiFiProperties = protectedNiFiProperties.getUnprotectedProperties() + logger.info("Unprotected ${unprotectedNiFiProperties.size()} properties") + + // Assert + assert unprotectedNiFiProperties.size() == expectedUnprotectedPropertyCount + assert unprotectedNiFiProperties.getPropertyKeys().every { + !unprotectedNiFiProperties.getProperty(it).endsWith(".protected") + } + logger.info("Hash code from returned unprotected instance: ${unprotectedNiFiProperties.hashCode()}") + assert unprotectedNiFiProperties.hashCode() != hashCode + } + + @Test + void testShouldCalculateSize() { + // Arrange + Properties rawProperties = [key: "protectedValue", "key.protected": "scheme", "key2": "value2"] as Properties + ProtectedNiFiProperties protectedNiFiProperties = new ProtectedNiFiProperties(rawProperties) + logger.info("Raw properties (${rawProperties.size()}): ${rawProperties.keySet().join(", ")}") + + // Act + int protectedSize = protectedNiFiProperties.size() + logger.info("Protected properties (${protectedNiFiProperties.size()}): ${protectedNiFiProperties.getPropertyKeys().join(", ")}") + + // Assert + assert protectedSize == rawProperties.size() - 1 + } + + @Test + void testGetPropertyKeysShouldMatchSize() { + // Arrange + Properties rawProperties = [key: "protectedValue", "key.protected": "scheme", "key2": "value2"] as Properties + ProtectedNiFiProperties protectedNiFiProperties = new ProtectedNiFiProperties(rawProperties) + logger.info("Raw properties (${rawProperties.size()}): ${rawProperties.keySet().join(", ")}") + + // Act + def filteredKeys = protectedNiFiProperties.getPropertyKeys() + logger.info("Protected properties (${protectedNiFiProperties.size()}): ${filteredKeys.join(", ")}") + + // Assert + assert protectedNiFiProperties.size() == rawProperties.size() - 1 + assert filteredKeys == rawProperties.keySet() - "key.protected" + } + + @Test + void testShouldGetPropertyKeysIncludingProtectionSchemes() { + // Arrange + Properties rawProperties = [key: "protectedValue", "key.protected": "scheme", "key2": "value2"] as Properties + ProtectedNiFiProperties protectedNiFiProperties = new ProtectedNiFiProperties(rawProperties) + logger.info("Raw properties (${rawProperties.size()}): ${rawProperties.keySet().join(", ")}") + + // Act + def allKeys = protectedNiFiProperties.getPropertyKeysIncludingProtectionSchemes() + logger.info("Protected properties with schemes (${allKeys.size()}): ${allKeys.join(", ")}") + + // Assert + assert allKeys.size() == rawProperties.size() + assert allKeys == rawProperties.keySet() + } + + // TODO: Add tests for protectPlainProperties +} diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-properties-loader/src/test/groovy/org/apache/nifi/properties/StandardNiFiPropertiesGroovyTest.groovy b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-properties-loader/src/test/groovy/org/apache/nifi/properties/StandardNiFiPropertiesGroovyTest.groovy new file mode 100644 index 0000000000..c9492fb90f --- /dev/null +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-properties-loader/src/test/groovy/org/apache/nifi/properties/StandardNiFiPropertiesGroovyTest.groovy @@ -0,0 +1,150 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.properties + +import org.apache.nifi.util.NiFiProperties +import org.junit.After +import org.junit.AfterClass +import org.junit.Before +import org.junit.BeforeClass +import org.junit.Test +import org.junit.runner.RunWith +import org.junit.runners.JUnit4 +import org.slf4j.Logger +import org.slf4j.LoggerFactory + +@RunWith(JUnit4.class) +class StandardNiFiPropertiesGroovyTest extends GroovyTestCase { + private static final Logger logger = LoggerFactory.getLogger(StandardNiFiPropertiesGroovyTest.class) + + private static String originalPropertiesPath = System.getProperty(NiFiProperties.PROPERTIES_FILE_PATH) + + @BeforeClass + public static void setUpOnce() throws Exception { + logger.metaClass.methodMissing = { String name, args -> + logger.info("[${name?.toUpperCase()}] ${(args as List).join(" ")}") + } + } + + @Before + public void setUp() throws Exception { + } + + @After + public void tearDown() throws Exception { + } + + @AfterClass + public static void tearDownOnce() { + if (originalPropertiesPath) { + System.setProperty(NiFiProperties.PROPERTIES_FILE_PATH, originalPropertiesPath) + } + } + + private static StandardNiFiProperties loadFromFile(String propertiesFilePath) { + String filePath; + try { + filePath = StandardNiFiPropertiesGroovyTest.class.getResource(propertiesFilePath).toURI().getPath(); + } catch (URISyntaxException ex) { + throw new RuntimeException("Cannot load properties file due to " + + ex.getLocalizedMessage(), ex); + } + + System.setProperty(NiFiProperties.PROPERTIES_FILE_PATH, filePath); + + StandardNiFiProperties properties = new StandardNiFiProperties(); + + // clear out existing properties + for (String prop : properties.stringPropertyNames()) { + properties.remove(prop); + } + + InputStream inStream = null; + try { + inStream = new BufferedInputStream(new FileInputStream(filePath)); + properties.load(inStream); + } catch (final Exception ex) { + throw new RuntimeException("Cannot load properties file due to " + + ex.getLocalizedMessage(), ex); + } finally { + if (null != inStream) { + try { + inStream.close(); + } catch (Exception ex) { + /** + * do nothing * + */ + } + } + } + + return properties; + } + + @Test + public void testConstructorShouldCreateNewInstance() throws Exception { + // Arrange + + // Act + NiFiProperties niFiProperties = new StandardNiFiProperties() + logger.info("niFiProperties has ${niFiProperties.size()} properties: ${niFiProperties.getPropertyKeys()}") + + // Assert + assert niFiProperties.size() == 0 + assert niFiProperties.getPropertyKeys() == [] as Set + } + + @Test + public void testConstructorShouldAcceptRawProperties() throws Exception { + // Arrange + Properties rawProperties = new Properties() + rawProperties.setProperty("key", "value") + logger.info("rawProperties has ${rawProperties.size()} properties: ${rawProperties.stringPropertyNames()}") + assert rawProperties.size() == 1 + + // Act + NiFiProperties niFiProperties = new StandardNiFiProperties(rawProperties) + logger.info("niFiProperties has ${niFiProperties.size()} properties: ${niFiProperties.getPropertyKeys()}") + + // Assert + assert niFiProperties.size() == 1 + assert niFiProperties.getPropertyKeys() == ["key"] as Set + } + + @Test + public void testShouldAllowMultipleInstances() throws Exception { + // Arrange + Properties rawProperties = new Properties() + rawProperties.setProperty("key", "value") + logger.info("rawProperties has ${rawProperties.size()} properties: ${rawProperties.stringPropertyNames()}") + assert rawProperties.size() == 1 + + // Act + NiFiProperties niFiProperties = new StandardNiFiProperties(rawProperties) + logger.info("niFiProperties has ${niFiProperties.size()} properties: ${niFiProperties.getPropertyKeys()}") + NiFiProperties emptyProperties = new StandardNiFiProperties() + logger.info("emptyProperties has ${emptyProperties.size()} properties: ${emptyProperties.getPropertyKeys()}") + + + // Assert + assert niFiProperties.size() == 1 + assert niFiProperties.getPropertyKeys() == ["key"] as Set + + assert emptyProperties.size() == 0 + assert emptyProperties.getPropertyKeys() == [] as Set + } +} diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-properties-loader/src/test/resources/bootstrap_tests/conf/bootstrap.conf b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-properties-loader/src/test/resources/bootstrap_tests/conf/bootstrap.conf new file mode 100644 index 0000000000..9225126caa --- /dev/null +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-properties-loader/src/test/resources/bootstrap_tests/conf/bootstrap.conf @@ -0,0 +1,74 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# Java command to use when running NiFi +java=java + +# Username to use when running NiFi. This value will be ignored on Windows. +run.as= + +# Configure where NiFi's lib and conf directories live +lib.dir=./lib +conf.dir=./conf + +# How long to wait after telling NiFi to shutdown before explicitly killing the Process +graceful.shutdown.seconds=20 + +# Disable JSR 199 so that we can use JSP's without running a JDK +java.arg.1=-Dorg.apache.jasper.compiler.disablejsr199=true + +# JVM memory settings +java.arg.2=-Xms512m +java.arg.3=-Xmx512m + +# Enable Remote Debugging +#java.arg.debug=-agentlib:jdwp=transport=dt_socket,server=y,suspend=n,address=8000 + +java.arg.4=-Djava.net.preferIPv4Stack=true + +# allowRestrictedHeaders is required for Cluster/Node communications to work properly +java.arg.5=-Dsun.net.http.allowRestrictedHeaders=true +java.arg.6=-Djava.protocol.handler.pkgs=sun.net.www.protocol + +# The G1GC is still considered experimental but has proven to be very advantageous in providing great +# performance without significant "stop-the-world" delays. +java.arg.13=-XX:+UseG1GC + +#Set headless mode by default +java.arg.14=-Djava.awt.headless=true + +# Master key in hexadecimal format for encrypted sensitive configuration values +nifi.bootstrap.sensitive.key=0123456789ABCDEFFEDCBA98765432100123456789ABCDEFFEDCBA9876543210 + +### +# Notification Services for notifying interested parties when NiFi is stopped, started, dies +### + +# XML File that contains the definitions of the notification services +notification.services.file=./conf/bootstrap-notification-services.xml + +# In the case that we are unable to send a notification for an event, how many times should we retry? +notification.max.attempts=5 + +# Comma-separated list of identifiers that are present in the notification.services.file; which services should be used to notify when NiFi is started? +#nifi.start.notification.services=email-notification + +# Comma-separated list of identifiers that are present in the notification.services.file; which services should be used to notify when NiFi is stopped? +#nifi.stop.notification.services=email-notification + +# Comma-separated list of identifiers that are present in the notification.services.file; which services should be used to notify when NiFi dies? +#nifi.dead.notification.services=email-notification \ No newline at end of file diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-properties-loader/src/test/resources/bootstrap_tests/conf/nifi_with_sensitive_properties_protected_aes.properties b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-properties-loader/src/test/resources/bootstrap_tests/conf/nifi_with_sensitive_properties_protected_aes.properties new file mode 100644 index 0000000000..bee200e3f2 --- /dev/null +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-properties-loader/src/test/resources/bootstrap_tests/conf/nifi_with_sensitive_properties_protected_aes.properties @@ -0,0 +1,129 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Core Properties # +nifi.version=nifi-test 3.0.0 +nifi.flow.configuration.file=./target/flow.xml.gz +nifi.flow.configuration.archive.dir=./target/archive/ +nifi.flowcontroller.autoResumeState=true +nifi.flowcontroller.graceful.shutdown.period=10 sec +nifi.flowservice.writedelay.interval=2 sec +nifi.administrative.yield.duration=30 sec + +nifi.reporting.task.configuration.file=./target/reporting-tasks.xml +nifi.controller.service.configuration.file=./target/controller-services.xml +nifi.templates.directory=./target/templates +nifi.ui.banner.text=UI Banner Text +nifi.ui.autorefresh.interval=30 sec +nifi.nar.library.directory=./target/resources/NiFiProperties/lib/ +nifi.nar.library.directory.alt=./target/resources/NiFiProperties/lib2/ +nifi.nar.working.directory=./target/work/nar/ + +# H2 Settings +nifi.database.directory=./target/database_repository +nifi.h2.url.append=;LOCK_TIMEOUT=25000;WRITE_DELAY=0;AUTO_SERVER=FALSE + +# FlowFile Repository +nifi.flowfile.repository.directory=./target/test-repo +nifi.flowfile.repository.partitions=1 +nifi.flowfile.repository.checkpoint.interval=2 mins +nifi.queue.swap.threshold=20000 +nifi.swap.storage.directory=./target/test-repo/swap +nifi.swap.in.period=5 sec +nifi.swap.in.threads=1 +nifi.swap.out.period=5 sec +nifi.swap.out.threads=4 + +# Content Repository +nifi.content.claim.max.appendable.size=10 MB +nifi.content.claim.max.flow.files=100 +nifi.content.repository.directory.default=./target/content_repository + +# Provenance Repository Properties +nifi.provenance.repository.storage.directory=./target/provenance_repository +nifi.provenance.repository.max.storage.time=24 hours +nifi.provenance.repository.max.storage.size=1 GB +nifi.provenance.repository.rollover.time=30 secs +nifi.provenance.repository.rollover.size=100 MB + +# Site to Site properties +nifi.remote.input.socket.port=9990 +nifi.remote.input.secure=true + +# web properties # +nifi.web.war.directory=./target/lib +nifi.web.http.host= +nifi.web.http.port= +nifi.web.https.host=nifi.nifi.apache.org +nifi.web.https.port=8443 +nifi.web.jetty.working.directory=./target/work/jetty + +# security properties # +nifi.sensitive.props.key=n2z+tTTbHuZ4V4V2||uWhdasyDXD4ZG2lMAes/vqh6u4vaz4xgL4aEbF4Y/dXevqk3ulRcOwf1vc4RDQ== +nifi.sensitive.props.key.protected=aes/gcm/256 +nifi.sensitive.props.algorithm=PBEWITHMD5AND256BITAES-CBC-OPENSSL +nifi.sensitive.props.provider=BC +nifi.sensitive.props.additional.keys= + +nifi.security.keystore=/path/to/keystore.jks +nifi.security.keystoreType=JKS +nifi.security.keystorePasswd=oBjT92hIGRElIGOh||MZ6uYuWNBrOA6usq/Jt3DaD2e4otNirZDytac/w/KFe0HOkrJR03vcbo +nifi.security.keystorePasswd.protected=aes/gcm/256 +nifi.security.keyPasswd=ac/BaE35SL/esLiJ||+ULRvRLYdIDA2VqpE0eQXDEMjaLBMG2kbKOdOwBk/hGebDKlVg== +nifi.security.keyPasswd.protected=aes/gcm/256 +nifi.security.truststore= +nifi.security.truststoreType= +nifi.security.truststorePasswd=/X/RSlNr2QCJ1Kwe||dENJevX5P61ix+97airrtoBQoyasMFS6DG6fHbX+SZtw2VAMllSSnDeT97Q= +nifi.security.truststorePasswd.protected=aes/gcm/256 +nifi.security.needClientAuth= +nifi.security.user.authorizer= + +# cluster common properties (cluster manager and nodes must have same values) # +nifi.cluster.protocol.heartbeat.interval=5 sec +nifi.cluster.protocol.is.secure=false +nifi.cluster.protocol.socket.timeout=30 sec +nifi.cluster.protocol.connection.handshake.timeout=45 sec +# if multicast is used, then nifi.cluster.protocol.multicast.xxx properties must be configured # +nifi.cluster.protocol.use.multicast=false +nifi.cluster.protocol.multicast.address= +nifi.cluster.protocol.multicast.port= +nifi.cluster.protocol.multicast.service.broadcast.delay=500 ms +nifi.cluster.protocol.multicast.service.locator.attempts=3 +nifi.cluster.protocol.multicast.service.locator.attempts.delay=1 sec + +# cluster node properties (only configure for cluster nodes) # +nifi.cluster.is.node=false +nifi.cluster.node.address= +nifi.cluster.node.protocol.port= +nifi.cluster.node.protocol.threads=2 +# if multicast is not used, nifi.cluster.node.unicast.xxx must have same values as nifi.cluster.manager.xxx # +nifi.cluster.node.unicast.manager.address= +nifi.cluster.node.unicast.manager.protocol.port= +nifi.cluster.node.unicast.manager.authority.provider.port= + +# cluster manager properties (only configure for cluster manager) # +nifi.cluster.is.manager=false +nifi.cluster.manager.address= +nifi.cluster.manager.protocol.port= +nifi.cluster.manager.authority.provider.port= +nifi.cluster.manager.authority.provider.threads=10 +nifi.cluster.manager.node.firewall.file= +nifi.cluster.manager.node.event.history.size=10 +nifi.cluster.manager.node.api.connection.timeout=30 sec +nifi.cluster.manager.node.api.read.timeout=30 sec +nifi.cluster.manager.node.api.request.threads=10 +nifi.cluster.manager.flow.retrieval.delay=5 sec +nifi.cluster.manager.protocol.threads=10 +nifi.cluster.manager.safemode.duration=0 sec diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-properties-loader/src/test/resources/bootstrap_tests/missing_bootstrap/nifi.properties b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-properties-loader/src/test/resources/bootstrap_tests/missing_bootstrap/nifi.properties new file mode 100644 index 0000000000..ae1e83eeb3 --- /dev/null +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-properties-loader/src/test/resources/bootstrap_tests/missing_bootstrap/nifi.properties @@ -0,0 +1,14 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-properties-loader/src/test/resources/bootstrap_tests/missing_key/bootstrap.conf b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-properties-loader/src/test/resources/bootstrap_tests/missing_key/bootstrap.conf new file mode 100644 index 0000000000..17acc6202d --- /dev/null +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-properties-loader/src/test/resources/bootstrap_tests/missing_key/bootstrap.conf @@ -0,0 +1,74 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# Java command to use when running NiFi +java=java + +# Username to use when running NiFi. This value will be ignored on Windows. +run.as= + +# Configure where NiFi's lib and conf directories live +lib.dir=./lib +conf.dir=./conf + +# How long to wait after telling NiFi to shutdown before explicitly killing the Process +graceful.shutdown.seconds=20 + +# Disable JSR 199 so that we can use JSP's without running a JDK +java.arg.1=-Dorg.apache.jasper.compiler.disablejsr199=true + +# JVM memory settings +java.arg.2=-Xms512m +java.arg.3=-Xmx512m + +# Enable Remote Debugging +#java.arg.debug=-agentlib:jdwp=transport=dt_socket,server=y,suspend=n,address=8000 + +java.arg.4=-Djava.net.preferIPv4Stack=true + +# allowRestrictedHeaders is required for Cluster/Node communications to work properly +java.arg.5=-Dsun.net.http.allowRestrictedHeaders=true +java.arg.6=-Djava.protocol.handler.pkgs=sun.net.www.protocol + +# The G1GC is still considered experimental but has proven to be very advantageous in providing great +# performance without significant "stop-the-world" delays. +java.arg.13=-XX:+UseG1GC + +#Set headless mode by default +java.arg.14=-Djava.awt.headless=true + +# Master key in hexadecimal format for encrypted sensitive configuration values +nifi.bootstrap.sensitive.key= + +### +# Notification Services for notifying interested parties when NiFi is stopped, started, dies +### + +# XML File that contains the definitions of the notification services +notification.services.file=./conf/bootstrap-notification-services.xml + +# In the case that we are unable to send a notification for an event, how many times should we retry? +notification.max.attempts=5 + +# Comma-separated list of identifiers that are present in the notification.services.file; which services should be used to notify when NiFi is started? +#nifi.start.notification.services=email-notification + +# Comma-separated list of identifiers that are present in the notification.services.file; which services should be used to notify when NiFi is stopped? +#nifi.stop.notification.services=email-notification + +# Comma-separated list of identifiers that are present in the notification.services.file; which services should be used to notify when NiFi dies? +#nifi.dead.notification.services=email-notification \ No newline at end of file diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-properties-loader/src/test/resources/bootstrap_tests/missing_key/nifi.properties b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-properties-loader/src/test/resources/bootstrap_tests/missing_key/nifi.properties new file mode 100644 index 0000000000..ae1e83eeb3 --- /dev/null +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-properties-loader/src/test/resources/bootstrap_tests/missing_key/nifi.properties @@ -0,0 +1,14 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-properties-loader/src/test/resources/bootstrap_tests/missing_key_line/bootstrap.conf b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-properties-loader/src/test/resources/bootstrap_tests/missing_key_line/bootstrap.conf new file mode 100644 index 0000000000..fc917c743f --- /dev/null +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-properties-loader/src/test/resources/bootstrap_tests/missing_key_line/bootstrap.conf @@ -0,0 +1,71 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# Java command to use when running NiFi +java=java + +# Username to use when running NiFi. This value will be ignored on Windows. +run.as= + +# Configure where NiFi's lib and conf directories live +lib.dir=./lib +conf.dir=./conf + +# How long to wait after telling NiFi to shutdown before explicitly killing the Process +graceful.shutdown.seconds=20 + +# Disable JSR 199 so that we can use JSP's without running a JDK +java.arg.1=-Dorg.apache.jasper.compiler.disablejsr199=true + +# JVM memory settings +java.arg.2=-Xms512m +java.arg.3=-Xmx512m + +# Enable Remote Debugging +#java.arg.debug=-agentlib:jdwp=transport=dt_socket,server=y,suspend=n,address=8000 + +java.arg.4=-Djava.net.preferIPv4Stack=true + +# allowRestrictedHeaders is required for Cluster/Node communications to work properly +java.arg.5=-Dsun.net.http.allowRestrictedHeaders=true +java.arg.6=-Djava.protocol.handler.pkgs=sun.net.www.protocol + +# The G1GC is still considered experimental but has proven to be very advantageous in providing great +# performance without significant "stop-the-world" delays. +java.arg.13=-XX:+UseG1GC + +#Set headless mode by default +java.arg.14=-Djava.awt.headless=true + +### +# Notification Services for notifying interested parties when NiFi is stopped, started, dies +### + +# XML File that contains the definitions of the notification services +notification.services.file=./conf/bootstrap-notification-services.xml + +# In the case that we are unable to send a notification for an event, how many times should we retry? +notification.max.attempts=5 + +# Comma-separated list of identifiers that are present in the notification.services.file; which services should be used to notify when NiFi is started? +#nifi.start.notification.services=email-notification + +# Comma-separated list of identifiers that are present in the notification.services.file; which services should be used to notify when NiFi is stopped? +#nifi.stop.notification.services=email-notification + +# Comma-separated list of identifiers that are present in the notification.services.file; which services should be used to notify when NiFi dies? +#nifi.dead.notification.services=email-notification \ No newline at end of file diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-properties-loader/src/test/resources/bootstrap_tests/missing_key_line/nifi.properties b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-properties-loader/src/test/resources/bootstrap_tests/missing_key_line/nifi.properties new file mode 100644 index 0000000000..ae1e83eeb3 --- /dev/null +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-properties-loader/src/test/resources/bootstrap_tests/missing_key_line/nifi.properties @@ -0,0 +1,14 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-properties-loader/src/test/resources/bootstrap_tests/unreadable_bootstrap/bootstrap.conf b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-properties-loader/src/test/resources/bootstrap_tests/unreadable_bootstrap/bootstrap.conf new file mode 100755 index 0000000000..9225126caa --- /dev/null +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-properties-loader/src/test/resources/bootstrap_tests/unreadable_bootstrap/bootstrap.conf @@ -0,0 +1,74 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# Java command to use when running NiFi +java=java + +# Username to use when running NiFi. This value will be ignored on Windows. +run.as= + +# Configure where NiFi's lib and conf directories live +lib.dir=./lib +conf.dir=./conf + +# How long to wait after telling NiFi to shutdown before explicitly killing the Process +graceful.shutdown.seconds=20 + +# Disable JSR 199 so that we can use JSP's without running a JDK +java.arg.1=-Dorg.apache.jasper.compiler.disablejsr199=true + +# JVM memory settings +java.arg.2=-Xms512m +java.arg.3=-Xmx512m + +# Enable Remote Debugging +#java.arg.debug=-agentlib:jdwp=transport=dt_socket,server=y,suspend=n,address=8000 + +java.arg.4=-Djava.net.preferIPv4Stack=true + +# allowRestrictedHeaders is required for Cluster/Node communications to work properly +java.arg.5=-Dsun.net.http.allowRestrictedHeaders=true +java.arg.6=-Djava.protocol.handler.pkgs=sun.net.www.protocol + +# The G1GC is still considered experimental but has proven to be very advantageous in providing great +# performance without significant "stop-the-world" delays. +java.arg.13=-XX:+UseG1GC + +#Set headless mode by default +java.arg.14=-Djava.awt.headless=true + +# Master key in hexadecimal format for encrypted sensitive configuration values +nifi.bootstrap.sensitive.key=0123456789ABCDEFFEDCBA98765432100123456789ABCDEFFEDCBA9876543210 + +### +# Notification Services for notifying interested parties when NiFi is stopped, started, dies +### + +# XML File that contains the definitions of the notification services +notification.services.file=./conf/bootstrap-notification-services.xml + +# In the case that we are unable to send a notification for an event, how many times should we retry? +notification.max.attempts=5 + +# Comma-separated list of identifiers that are present in the notification.services.file; which services should be used to notify when NiFi is started? +#nifi.start.notification.services=email-notification + +# Comma-separated list of identifiers that are present in the notification.services.file; which services should be used to notify when NiFi is stopped? +#nifi.stop.notification.services=email-notification + +# Comma-separated list of identifiers that are present in the notification.services.file; which services should be used to notify when NiFi dies? +#nifi.dead.notification.services=email-notification \ No newline at end of file diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-properties-loader/src/test/resources/bootstrap_tests/unreadable_bootstrap/nifi.properties b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-properties-loader/src/test/resources/bootstrap_tests/unreadable_bootstrap/nifi.properties new file mode 100644 index 0000000000..ae1e83eeb3 --- /dev/null +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-properties-loader/src/test/resources/bootstrap_tests/unreadable_bootstrap/nifi.properties @@ -0,0 +1,14 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-properties-loader/src/test/resources/bootstrap_tests/unreadable_conf/bootstrap.conf b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-properties-loader/src/test/resources/bootstrap_tests/unreadable_conf/bootstrap.conf new file mode 100755 index 0000000000..9225126caa --- /dev/null +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-properties-loader/src/test/resources/bootstrap_tests/unreadable_conf/bootstrap.conf @@ -0,0 +1,74 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# Java command to use when running NiFi +java=java + +# Username to use when running NiFi. This value will be ignored on Windows. +run.as= + +# Configure where NiFi's lib and conf directories live +lib.dir=./lib +conf.dir=./conf + +# How long to wait after telling NiFi to shutdown before explicitly killing the Process +graceful.shutdown.seconds=20 + +# Disable JSR 199 so that we can use JSP's without running a JDK +java.arg.1=-Dorg.apache.jasper.compiler.disablejsr199=true + +# JVM memory settings +java.arg.2=-Xms512m +java.arg.3=-Xmx512m + +# Enable Remote Debugging +#java.arg.debug=-agentlib:jdwp=transport=dt_socket,server=y,suspend=n,address=8000 + +java.arg.4=-Djava.net.preferIPv4Stack=true + +# allowRestrictedHeaders is required for Cluster/Node communications to work properly +java.arg.5=-Dsun.net.http.allowRestrictedHeaders=true +java.arg.6=-Djava.protocol.handler.pkgs=sun.net.www.protocol + +# The G1GC is still considered experimental but has proven to be very advantageous in providing great +# performance without significant "stop-the-world" delays. +java.arg.13=-XX:+UseG1GC + +#Set headless mode by default +java.arg.14=-Djava.awt.headless=true + +# Master key in hexadecimal format for encrypted sensitive configuration values +nifi.bootstrap.sensitive.key=0123456789ABCDEFFEDCBA98765432100123456789ABCDEFFEDCBA9876543210 + +### +# Notification Services for notifying interested parties when NiFi is stopped, started, dies +### + +# XML File that contains the definitions of the notification services +notification.services.file=./conf/bootstrap-notification-services.xml + +# In the case that we are unable to send a notification for an event, how many times should we retry? +notification.max.attempts=5 + +# Comma-separated list of identifiers that are present in the notification.services.file; which services should be used to notify when NiFi is started? +#nifi.start.notification.services=email-notification + +# Comma-separated list of identifiers that are present in the notification.services.file; which services should be used to notify when NiFi is stopped? +#nifi.stop.notification.services=email-notification + +# Comma-separated list of identifiers that are present in the notification.services.file; which services should be used to notify when NiFi dies? +#nifi.dead.notification.services=email-notification \ No newline at end of file diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-properties-loader/src/test/resources/bootstrap_tests/unreadable_conf/nifi.properties b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-properties-loader/src/test/resources/bootstrap_tests/unreadable_conf/nifi.properties new file mode 100644 index 0000000000..ae1e83eeb3 --- /dev/null +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-properties-loader/src/test/resources/bootstrap_tests/unreadable_conf/nifi.properties @@ -0,0 +1,14 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-properties-loader/src/test/resources/conf/nifi.blank.properties b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-properties-loader/src/test/resources/conf/nifi.blank.properties new file mode 100644 index 0000000000..898cebf10c --- /dev/null +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-properties-loader/src/test/resources/conf/nifi.blank.properties @@ -0,0 +1,124 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Core Properties # +nifi.version=nifi-test 3.0.0 +nifi.flow.configuration.file=./target/flow.xml.gz +nifi.flow.configuration.archive.dir=./target/archive/ +nifi.flowcontroller.autoResumeState=true +nifi.flowcontroller.graceful.shutdown.period=10 sec +nifi.flowservice.writedelay.interval=2 sec +nifi.administrative.yield.duration=30 sec + +nifi.reporting.task.configuration.file=./target/reporting-tasks.xml +nifi.controller.service.configuration.file=./target/controller-services.xml +nifi.templates.directory=./target/templates +nifi.ui.banner.text=UI Banner Text +nifi.ui.autorefresh.interval=30 sec +nifi.nar.library.directory= +nifi.custom.nar.library.directory.alt= +nifi.nar.working.directory=./target/work/nar/ + +# H2 Settings +nifi.database.directory=./target/database_repository +nifi.h2.url.append=;LOCK_TIMEOUT=25000;WRITE_DELAY=0;AUTO_SERVER=FALSE + +# FlowFile Repository +nifi.flowfile.repository.directory=./target/test-repo +nifi.flowfile.repository.partitions=1 +nifi.flowfile.repository.checkpoint.interval=2 mins +nifi.queue.swap.threshold=20000 +nifi.swap.storage.directory=./target/test-repo/swap +nifi.swap.in.period=5 sec +nifi.swap.in.threads=1 +nifi.swap.out.period=5 sec +nifi.swap.out.threads=4 + +# Content Repository +nifi.content.claim.max.appendable.size=10 MB +nifi.content.claim.max.flow.files=100 +nifi.content.repository.directory.default=./target/content_repository + +# Provenance Repository Properties +nifi.provenance.repository.storage.directory=./target/provenance_repository +nifi.provenance.repository.max.storage.time=24 hours +nifi.provenance.repository.max.storage.size=1 GB +nifi.provenance.repository.rollover.time=30 secs +nifi.provenance.repository.rollover.size=100 MB + +# Site to Site properties +nifi.remote.input.socket.port=9990 +nifi.remote.input.secure=true + +# web properties # +nifi.web.war.directory=./target/lib +nifi.web.http.host= +nifi.web.http.port=8080 +nifi.web.https.host= +nifi.web.https.port= +nifi.web.jetty.working.directory=./target/work/jetty + +# security properties # +nifi.sensitive.props.key=key +nifi.sensitive.props.algorithm=PBEWITHMD5AND256BITAES-CBC-OPENSSL +nifi.sensitive.props.provider=BC + +nifi.security.keystore= +nifi.security.keystoreType= +nifi.security.keystorePasswd= +nifi.security.keyPasswd= +nifi.security.truststore= +nifi.security.truststoreType= +nifi.security.truststorePasswd= +nifi.security.needClientAuth= +nifi.security.user.authorizer= + +# cluster common properties (cluster manager and nodes must have same values) # +nifi.cluster.protocol.heartbeat.interval=5 sec +nifi.cluster.protocol.is.secure=false +nifi.cluster.protocol.socket.timeout=30 sec +nifi.cluster.protocol.connection.handshake.timeout=45 sec +# if multicast is used, then nifi.cluster.protocol.multicast.xxx properties must be configured # +nifi.cluster.protocol.use.multicast=false +nifi.cluster.protocol.multicast.address= +nifi.cluster.protocol.multicast.port= +nifi.cluster.protocol.multicast.service.broadcast.delay=500 ms +nifi.cluster.protocol.multicast.service.locator.attempts=3 +nifi.cluster.protocol.multicast.service.locator.attempts.delay=1 sec + +# cluster node properties (only configure for cluster nodes) # +nifi.cluster.is.node=false +nifi.cluster.node.address= +nifi.cluster.node.protocol.port= +nifi.cluster.node.protocol.threads=2 +# if multicast is not used, nifi.cluster.node.unicast.xxx must have same values as nifi.cluster.manager.xxx # +nifi.cluster.node.unicast.manager.address= +nifi.cluster.node.unicast.manager.protocol.port= +nifi.cluster.node.unicast.manager.authority.provider.port= + +# cluster manager properties (only configure for cluster manager) # +nifi.cluster.is.manager=false +nifi.cluster.manager.address= +nifi.cluster.manager.protocol.port= +nifi.cluster.manager.authority.provider.port= +nifi.cluster.manager.authority.provider.threads=10 +nifi.cluster.manager.node.firewall.file= +nifi.cluster.manager.node.event.history.size=10 +nifi.cluster.manager.node.api.connection.timeout=30 sec +nifi.cluster.manager.node.api.read.timeout=30 sec +nifi.cluster.manager.node.api.request.threads=10 +nifi.cluster.manager.flow.retrieval.delay=5 sec +nifi.cluster.manager.protocol.threads=10 +nifi.cluster.manager.safemode.duration=0 sec diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-properties-loader/src/test/resources/conf/nifi.missing.properties b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-properties-loader/src/test/resources/conf/nifi.missing.properties new file mode 100644 index 0000000000..786b05f47b --- /dev/null +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-properties-loader/src/test/resources/conf/nifi.missing.properties @@ -0,0 +1,122 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Core Properties # +nifi.version=nifi-test 3.0.0 +nifi.flow.configuration.file=./target/flow.xml.gz +nifi.flow.configuration.archive.dir=./target/archive/ +nifi.flowcontroller.autoResumeState=true +nifi.flowcontroller.graceful.shutdown.period=10 sec +nifi.flowservice.writedelay.interval=2 sec +nifi.administrative.yield.duration=30 sec + +nifi.reporting.task.configuration.file=./target/reporting-tasks.xml +nifi.controller.service.configuration.file=./target/controller-services.xml +nifi.templates.directory=./target/templates +nifi.ui.banner.text=UI Banner Text +nifi.ui.autorefresh.interval=30 sec +nifi.nar.working.directory=./target/work/nar/ + +# H2 Settings +nifi.database.directory=./target/database_repository +nifi.h2.url.append=;LOCK_TIMEOUT=25000;WRITE_DELAY=0;AUTO_SERVER=FALSE + +# FlowFile Repository +nifi.flowfile.repository.directory=./target/test-repo +nifi.flowfile.repository.partitions=1 +nifi.flowfile.repository.checkpoint.interval=2 mins +nifi.queue.swap.threshold=20000 +nifi.swap.storage.directory=./target/test-repo/swap +nifi.swap.in.period=5 sec +nifi.swap.in.threads=1 +nifi.swap.out.period=5 sec +nifi.swap.out.threads=4 + +# Content Repository +nifi.content.claim.max.appendable.size=10 MB +nifi.content.claim.max.flow.files=100 +nifi.content.repository.directory.default=./target/content_repository + +# Provenance Repository Properties +nifi.provenance.repository.storage.directory=./target/provenance_repository +nifi.provenance.repository.max.storage.time=24 hours +nifi.provenance.repository.max.storage.size=1 GB +nifi.provenance.repository.rollover.time=30 secs +nifi.provenance.repository.rollover.size=100 MB + +# Site to Site properties +nifi.remote.input.socket.port=9990 +nifi.remote.input.secure=true + +# web properties # +nifi.web.war.directory=./target/lib +nifi.web.http.host= +nifi.web.http.port=8080 +nifi.web.https.host= +nifi.web.https.port= +nifi.web.jetty.working.directory=./target/work/jetty + +# security properties # +nifi.sensitive.props.key=key +nifi.sensitive.props.algorithm=PBEWITHMD5AND256BITAES-CBC-OPENSSL +nifi.sensitive.props.provider=BC + +nifi.security.keystore= +nifi.security.keystoreType= +nifi.security.keystorePasswd= +nifi.security.keyPasswd= +nifi.security.truststore= +nifi.security.truststoreType= +nifi.security.truststorePasswd= +nifi.security.needClientAuth= +nifi.security.user.authorizer= + +# cluster common properties (cluster manager and nodes must have same values) # +nifi.cluster.protocol.heartbeat.interval=5 sec +nifi.cluster.protocol.is.secure=false +nifi.cluster.protocol.socket.timeout=30 sec +nifi.cluster.protocol.connection.handshake.timeout=45 sec +# if multicast is used, then nifi.cluster.protocol.multicast.xxx properties must be configured # +nifi.cluster.protocol.use.multicast=false +nifi.cluster.protocol.multicast.address= +nifi.cluster.protocol.multicast.port= +nifi.cluster.protocol.multicast.service.broadcast.delay=500 ms +nifi.cluster.protocol.multicast.service.locator.attempts=3 +nifi.cluster.protocol.multicast.service.locator.attempts.delay=1 sec + +# cluster node properties (only configure for cluster nodes) # +nifi.cluster.is.node=false +nifi.cluster.node.address= +nifi.cluster.node.protocol.port= +nifi.cluster.node.protocol.threads=2 +# if multicast is not used, nifi.cluster.node.unicast.xxx must have same values as nifi.cluster.manager.xxx # +nifi.cluster.node.unicast.manager.address= +nifi.cluster.node.unicast.manager.protocol.port= +nifi.cluster.node.unicast.manager.authority.provider.port= + +# cluster manager properties (only configure for cluster manager) # +nifi.cluster.is.manager=false +nifi.cluster.manager.address= +nifi.cluster.manager.protocol.port= +nifi.cluster.manager.authority.provider.port= +nifi.cluster.manager.authority.provider.threads=10 +nifi.cluster.manager.node.firewall.file= +nifi.cluster.manager.node.event.history.size=10 +nifi.cluster.manager.node.api.connection.timeout=30 sec +nifi.cluster.manager.node.api.read.timeout=30 sec +nifi.cluster.manager.node.api.request.threads=10 +nifi.cluster.manager.flow.retrieval.delay=5 sec +nifi.cluster.manager.protocol.threads=10 +nifi.cluster.manager.safemode.duration=0 sec diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-properties-loader/src/test/resources/conf/nifi.properties b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-properties-loader/src/test/resources/conf/nifi.properties new file mode 100644 index 0000000000..f9d9b78ee8 --- /dev/null +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-properties-loader/src/test/resources/conf/nifi.properties @@ -0,0 +1,124 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Core Properties # +nifi.version=nifi-test 3.0.0 +nifi.flow.configuration.file=./target/flow.xml.gz +nifi.flow.configuration.archive.dir=./target/archive/ +nifi.flowcontroller.autoResumeState=true +nifi.flowcontroller.graceful.shutdown.period=10 sec +nifi.flowservice.writedelay.interval=2 sec +nifi.administrative.yield.duration=30 sec + +nifi.reporting.task.configuration.file=./target/reporting-tasks.xml +nifi.controller.service.configuration.file=./target/controller-services.xml +nifi.templates.directory=./target/templates +nifi.ui.banner.text=UI Banner Text +nifi.ui.autorefresh.interval=30 sec +nifi.nar.library.directory=./target/resources/NiFiProperties/lib/ +nifi.nar.library.directory.alt=./target/resources/NiFiProperties/lib2/ +nifi.nar.working.directory=./target/work/nar/ + +# H2 Settings +nifi.database.directory=./target/database_repository +nifi.h2.url.append=;LOCK_TIMEOUT=25000;WRITE_DELAY=0;AUTO_SERVER=FALSE + +# FlowFile Repository +nifi.flowfile.repository.directory=./target/test-repo +nifi.flowfile.repository.partitions=1 +nifi.flowfile.repository.checkpoint.interval=2 mins +nifi.queue.swap.threshold=20000 +nifi.swap.storage.directory=./target/test-repo/swap +nifi.swap.in.period=5 sec +nifi.swap.in.threads=1 +nifi.swap.out.period=5 sec +nifi.swap.out.threads=4 + +# Content Repository +nifi.content.claim.max.appendable.size=10 MB +nifi.content.claim.max.flow.files=100 +nifi.content.repository.directory.default=./target/content_repository + +# Provenance Repository Properties +nifi.provenance.repository.storage.directory=./target/provenance_repository +nifi.provenance.repository.max.storage.time=24 hours +nifi.provenance.repository.max.storage.size=1 GB +nifi.provenance.repository.rollover.time=30 secs +nifi.provenance.repository.rollover.size=100 MB + +# Site to Site properties +nifi.remote.input.socket.port=9990 +nifi.remote.input.secure=true + +# web properties # +nifi.web.war.directory=./target/lib +nifi.web.http.host= +nifi.web.http.port=8080 +nifi.web.https.host= +nifi.web.https.port= +nifi.web.jetty.working.directory=./target/work/jetty + +# security properties # +nifi.sensitive.props.key=key +nifi.sensitive.props.algorithm=PBEWITHMD5AND256BITAES-CBC-OPENSSL +nifi.sensitive.props.provider=BC + +nifi.security.keystore= +nifi.security.keystoreType= +nifi.security.keystorePasswd= +nifi.security.keyPasswd= +nifi.security.truststore= +nifi.security.truststoreType= +nifi.security.truststorePasswd= +nifi.security.needClientAuth= +nifi.security.user.authorizer= + +# cluster common properties (cluster manager and nodes must have same values) # +nifi.cluster.protocol.heartbeat.interval=5 sec +nifi.cluster.protocol.is.secure=false +nifi.cluster.protocol.socket.timeout=30 sec +nifi.cluster.protocol.connection.handshake.timeout=45 sec +# if multicast is used, then nifi.cluster.protocol.multicast.xxx properties must be configured # +nifi.cluster.protocol.use.multicast=false +nifi.cluster.protocol.multicast.address= +nifi.cluster.protocol.multicast.port= +nifi.cluster.protocol.multicast.service.broadcast.delay=500 ms +nifi.cluster.protocol.multicast.service.locator.attempts=3 +nifi.cluster.protocol.multicast.service.locator.attempts.delay=1 sec + +# cluster node properties (only configure for cluster nodes) # +nifi.cluster.is.node=false +nifi.cluster.node.address= +nifi.cluster.node.protocol.port= +nifi.cluster.node.protocol.threads=2 +# if multicast is not used, nifi.cluster.node.unicast.xxx must have same values as nifi.cluster.manager.xxx # +nifi.cluster.node.unicast.manager.address= +nifi.cluster.node.unicast.manager.protocol.port= +nifi.cluster.node.unicast.manager.authority.provider.port= + +# cluster manager properties (only configure for cluster manager) # +nifi.cluster.is.manager=false +nifi.cluster.manager.address= +nifi.cluster.manager.protocol.port= +nifi.cluster.manager.authority.provider.port= +nifi.cluster.manager.authority.provider.threads=10 +nifi.cluster.manager.node.firewall.file= +nifi.cluster.manager.node.event.history.size=10 +nifi.cluster.manager.node.api.connection.timeout=30 sec +nifi.cluster.manager.node.api.read.timeout=30 sec +nifi.cluster.manager.node.api.request.threads=10 +nifi.cluster.manager.flow.retrieval.delay=5 sec +nifi.cluster.manager.protocol.threads=10 +nifi.cluster.manager.safemode.duration=0 sec diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-properties-loader/src/test/resources/conf/nifi_no_permissions.properties b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-properties-loader/src/test/resources/conf/nifi_no_permissions.properties new file mode 100644 index 0000000000..ae1e83eeb3 --- /dev/null +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-properties-loader/src/test/resources/conf/nifi_no_permissions.properties @@ -0,0 +1,14 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-properties-loader/src/test/resources/conf/nifi_with_additional_sensitive_keys.properties b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-properties-loader/src/test/resources/conf/nifi_with_additional_sensitive_keys.properties new file mode 100644 index 0000000000..85728e69b8 --- /dev/null +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-properties-loader/src/test/resources/conf/nifi_with_additional_sensitive_keys.properties @@ -0,0 +1,125 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Core Properties # +nifi.version=nifi-test 3.0.0 +nifi.flow.configuration.file=./target/flow.xml.gz +nifi.flow.configuration.archive.dir=./target/archive/ +nifi.flowcontroller.autoResumeState=true +nifi.flowcontroller.graceful.shutdown.period=10 sec +nifi.flowservice.writedelay.interval=2 sec +nifi.administrative.yield.duration=30 sec + +nifi.reporting.task.configuration.file=./target/reporting-tasks.xml +nifi.controller.service.configuration.file=./target/controller-services.xml +nifi.templates.directory=./target/templates +nifi.ui.banner.text=UI Banner Text +nifi.ui.autorefresh.interval=30 sec +nifi.nar.library.directory=./target/resources/NiFiProperties/lib/ +nifi.nar.library.directory.alt=./target/resources/NiFiProperties/lib2/ +nifi.nar.working.directory=./target/work/nar/ + +# H2 Settings +nifi.database.directory=./target/database_repository +nifi.h2.url.append=;LOCK_TIMEOUT=25000;WRITE_DELAY=0;AUTO_SERVER=FALSE + +# FlowFile Repository +nifi.flowfile.repository.directory=./target/test-repo +nifi.flowfile.repository.partitions=1 +nifi.flowfile.repository.checkpoint.interval=2 mins +nifi.queue.swap.threshold=20000 +nifi.swap.storage.directory=./target/test-repo/swap +nifi.swap.in.period=5 sec +nifi.swap.in.threads=1 +nifi.swap.out.period=5 sec +nifi.swap.out.threads=4 + +# Content Repository +nifi.content.claim.max.appendable.size=10 MB +nifi.content.claim.max.flow.files=100 +nifi.content.repository.directory.default=./target/content_repository + +# Provenance Repository Properties +nifi.provenance.repository.storage.directory=./target/provenance_repository +nifi.provenance.repository.max.storage.time=24 hours +nifi.provenance.repository.max.storage.size=1 GB +nifi.provenance.repository.rollover.time=30 secs +nifi.provenance.repository.rollover.size=100 MB + +# Site to Site properties +nifi.remote.input.socket.port=9990 +nifi.remote.input.secure=true + +# web properties # +nifi.web.war.directory=./target/lib +nifi.web.http.host= +nifi.web.http.port=8080 +nifi.web.https.host= +nifi.web.https.port= +nifi.web.jetty.working.directory=./target/work/jetty + +# security properties # +nifi.sensitive.props.key=key +nifi.sensitive.props.algorithm=PBEWITHMD5AND256BITAES-CBC-OPENSSL +nifi.sensitive.props.provider=BC +nifi.sensitive.props.additional.keys=nifi.ui.banner.text, nifi.version + +nifi.security.keystore= +nifi.security.keystoreType= +nifi.security.keystorePasswd= +nifi.security.keyPasswd= +nifi.security.truststore= +nifi.security.truststoreType= +nifi.security.truststorePasswd= +nifi.security.needClientAuth= +nifi.security.user.authorizer= + +# cluster common properties (cluster manager and nodes must have same values) # +nifi.cluster.protocol.heartbeat.interval=5 sec +nifi.cluster.protocol.is.secure=false +nifi.cluster.protocol.socket.timeout=30 sec +nifi.cluster.protocol.connection.handshake.timeout=45 sec +# if multicast is used, then nifi.cluster.protocol.multicast.xxx properties must be configured # +nifi.cluster.protocol.use.multicast=false +nifi.cluster.protocol.multicast.address= +nifi.cluster.protocol.multicast.port= +nifi.cluster.protocol.multicast.service.broadcast.delay=500 ms +nifi.cluster.protocol.multicast.service.locator.attempts=3 +nifi.cluster.protocol.multicast.service.locator.attempts.delay=1 sec + +# cluster node properties (only configure for cluster nodes) # +nifi.cluster.is.node=false +nifi.cluster.node.address= +nifi.cluster.node.protocol.port= +nifi.cluster.node.protocol.threads=2 +# if multicast is not used, nifi.cluster.node.unicast.xxx must have same values as nifi.cluster.manager.xxx # +nifi.cluster.node.unicast.manager.address= +nifi.cluster.node.unicast.manager.protocol.port= +nifi.cluster.node.unicast.manager.authority.provider.port= + +# cluster manager properties (only configure for cluster manager) # +nifi.cluster.is.manager=false +nifi.cluster.manager.address= +nifi.cluster.manager.protocol.port= +nifi.cluster.manager.authority.provider.port= +nifi.cluster.manager.authority.provider.threads=10 +nifi.cluster.manager.node.firewall.file= +nifi.cluster.manager.node.event.history.size=10 +nifi.cluster.manager.node.api.connection.timeout=30 sec +nifi.cluster.manager.node.api.read.timeout=30 sec +nifi.cluster.manager.node.api.request.threads=10 +nifi.cluster.manager.flow.retrieval.delay=5 sec +nifi.cluster.manager.protocol.threads=10 +nifi.cluster.manager.safemode.duration=0 sec diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-properties-loader/src/test/resources/conf/nifi_with_all_sensitive_properties_protected_aes.properties b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-properties-loader/src/test/resources/conf/nifi_with_all_sensitive_properties_protected_aes.properties new file mode 100644 index 0000000000..bee200e3f2 --- /dev/null +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-properties-loader/src/test/resources/conf/nifi_with_all_sensitive_properties_protected_aes.properties @@ -0,0 +1,129 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Core Properties # +nifi.version=nifi-test 3.0.0 +nifi.flow.configuration.file=./target/flow.xml.gz +nifi.flow.configuration.archive.dir=./target/archive/ +nifi.flowcontroller.autoResumeState=true +nifi.flowcontroller.graceful.shutdown.period=10 sec +nifi.flowservice.writedelay.interval=2 sec +nifi.administrative.yield.duration=30 sec + +nifi.reporting.task.configuration.file=./target/reporting-tasks.xml +nifi.controller.service.configuration.file=./target/controller-services.xml +nifi.templates.directory=./target/templates +nifi.ui.banner.text=UI Banner Text +nifi.ui.autorefresh.interval=30 sec +nifi.nar.library.directory=./target/resources/NiFiProperties/lib/ +nifi.nar.library.directory.alt=./target/resources/NiFiProperties/lib2/ +nifi.nar.working.directory=./target/work/nar/ + +# H2 Settings +nifi.database.directory=./target/database_repository +nifi.h2.url.append=;LOCK_TIMEOUT=25000;WRITE_DELAY=0;AUTO_SERVER=FALSE + +# FlowFile Repository +nifi.flowfile.repository.directory=./target/test-repo +nifi.flowfile.repository.partitions=1 +nifi.flowfile.repository.checkpoint.interval=2 mins +nifi.queue.swap.threshold=20000 +nifi.swap.storage.directory=./target/test-repo/swap +nifi.swap.in.period=5 sec +nifi.swap.in.threads=1 +nifi.swap.out.period=5 sec +nifi.swap.out.threads=4 + +# Content Repository +nifi.content.claim.max.appendable.size=10 MB +nifi.content.claim.max.flow.files=100 +nifi.content.repository.directory.default=./target/content_repository + +# Provenance Repository Properties +nifi.provenance.repository.storage.directory=./target/provenance_repository +nifi.provenance.repository.max.storage.time=24 hours +nifi.provenance.repository.max.storage.size=1 GB +nifi.provenance.repository.rollover.time=30 secs +nifi.provenance.repository.rollover.size=100 MB + +# Site to Site properties +nifi.remote.input.socket.port=9990 +nifi.remote.input.secure=true + +# web properties # +nifi.web.war.directory=./target/lib +nifi.web.http.host= +nifi.web.http.port= +nifi.web.https.host=nifi.nifi.apache.org +nifi.web.https.port=8443 +nifi.web.jetty.working.directory=./target/work/jetty + +# security properties # +nifi.sensitive.props.key=n2z+tTTbHuZ4V4V2||uWhdasyDXD4ZG2lMAes/vqh6u4vaz4xgL4aEbF4Y/dXevqk3ulRcOwf1vc4RDQ== +nifi.sensitive.props.key.protected=aes/gcm/256 +nifi.sensitive.props.algorithm=PBEWITHMD5AND256BITAES-CBC-OPENSSL +nifi.sensitive.props.provider=BC +nifi.sensitive.props.additional.keys= + +nifi.security.keystore=/path/to/keystore.jks +nifi.security.keystoreType=JKS +nifi.security.keystorePasswd=oBjT92hIGRElIGOh||MZ6uYuWNBrOA6usq/Jt3DaD2e4otNirZDytac/w/KFe0HOkrJR03vcbo +nifi.security.keystorePasswd.protected=aes/gcm/256 +nifi.security.keyPasswd=ac/BaE35SL/esLiJ||+ULRvRLYdIDA2VqpE0eQXDEMjaLBMG2kbKOdOwBk/hGebDKlVg== +nifi.security.keyPasswd.protected=aes/gcm/256 +nifi.security.truststore= +nifi.security.truststoreType= +nifi.security.truststorePasswd=/X/RSlNr2QCJ1Kwe||dENJevX5P61ix+97airrtoBQoyasMFS6DG6fHbX+SZtw2VAMllSSnDeT97Q= +nifi.security.truststorePasswd.protected=aes/gcm/256 +nifi.security.needClientAuth= +nifi.security.user.authorizer= + +# cluster common properties (cluster manager and nodes must have same values) # +nifi.cluster.protocol.heartbeat.interval=5 sec +nifi.cluster.protocol.is.secure=false +nifi.cluster.protocol.socket.timeout=30 sec +nifi.cluster.protocol.connection.handshake.timeout=45 sec +# if multicast is used, then nifi.cluster.protocol.multicast.xxx properties must be configured # +nifi.cluster.protocol.use.multicast=false +nifi.cluster.protocol.multicast.address= +nifi.cluster.protocol.multicast.port= +nifi.cluster.protocol.multicast.service.broadcast.delay=500 ms +nifi.cluster.protocol.multicast.service.locator.attempts=3 +nifi.cluster.protocol.multicast.service.locator.attempts.delay=1 sec + +# cluster node properties (only configure for cluster nodes) # +nifi.cluster.is.node=false +nifi.cluster.node.address= +nifi.cluster.node.protocol.port= +nifi.cluster.node.protocol.threads=2 +# if multicast is not used, nifi.cluster.node.unicast.xxx must have same values as nifi.cluster.manager.xxx # +nifi.cluster.node.unicast.manager.address= +nifi.cluster.node.unicast.manager.protocol.port= +nifi.cluster.node.unicast.manager.authority.provider.port= + +# cluster manager properties (only configure for cluster manager) # +nifi.cluster.is.manager=false +nifi.cluster.manager.address= +nifi.cluster.manager.protocol.port= +nifi.cluster.manager.authority.provider.port= +nifi.cluster.manager.authority.provider.threads=10 +nifi.cluster.manager.node.firewall.file= +nifi.cluster.manager.node.event.history.size=10 +nifi.cluster.manager.node.api.connection.timeout=30 sec +nifi.cluster.manager.node.api.read.timeout=30 sec +nifi.cluster.manager.node.api.request.threads=10 +nifi.cluster.manager.flow.retrieval.delay=5 sec +nifi.cluster.manager.protocol.threads=10 +nifi.cluster.manager.safemode.duration=0 sec diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-properties-loader/src/test/resources/conf/nifi_with_recursive_additional_sensitive_keys.properties b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-properties-loader/src/test/resources/conf/nifi_with_recursive_additional_sensitive_keys.properties new file mode 100644 index 0000000000..1d5009c858 --- /dev/null +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-properties-loader/src/test/resources/conf/nifi_with_recursive_additional_sensitive_keys.properties @@ -0,0 +1,125 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Core Properties # +nifi.version=nifi-test 3.0.0 +nifi.flow.configuration.file=./target/flow.xml.gz +nifi.flow.configuration.archive.dir=./target/archive/ +nifi.flowcontroller.autoResumeState=true +nifi.flowcontroller.graceful.shutdown.period=10 sec +nifi.flowservice.writedelay.interval=2 sec +nifi.administrative.yield.duration=30 sec + +nifi.reporting.task.configuration.file=./target/reporting-tasks.xml +nifi.controller.service.configuration.file=./target/controller-services.xml +nifi.templates.directory=./target/templates +nifi.ui.banner.text=UI Banner Text +nifi.ui.autorefresh.interval=30 sec +nifi.nar.library.directory=./target/resources/NiFiProperties/lib/ +nifi.nar.library.directory.alt=./target/resources/NiFiProperties/lib2/ +nifi.nar.working.directory=./target/work/nar/ + +# H2 Settings +nifi.database.directory=./target/database_repository +nifi.h2.url.append=;LOCK_TIMEOUT=25000;WRITE_DELAY=0;AUTO_SERVER=FALSE + +# FlowFile Repository +nifi.flowfile.repository.directory=./target/test-repo +nifi.flowfile.repository.partitions=1 +nifi.flowfile.repository.checkpoint.interval=2 mins +nifi.queue.swap.threshold=20000 +nifi.swap.storage.directory=./target/test-repo/swap +nifi.swap.in.period=5 sec +nifi.swap.in.threads=1 +nifi.swap.out.period=5 sec +nifi.swap.out.threads=4 + +# Content Repository +nifi.content.claim.max.appendable.size=10 MB +nifi.content.claim.max.flow.files=100 +nifi.content.repository.directory.default=./target/content_repository + +# Provenance Repository Properties +nifi.provenance.repository.storage.directory=./target/provenance_repository +nifi.provenance.repository.max.storage.time=24 hours +nifi.provenance.repository.max.storage.size=1 GB +nifi.provenance.repository.rollover.time=30 secs +nifi.provenance.repository.rollover.size=100 MB + +# Site to Site properties +nifi.remote.input.socket.port=9990 +nifi.remote.input.secure=true + +# web properties # +nifi.web.war.directory=./target/lib +nifi.web.http.host= +nifi.web.http.port=8080 +nifi.web.https.host= +nifi.web.https.port= +nifi.web.jetty.working.directory=./target/work/jetty + +# security properties # +nifi.sensitive.props.key=key +nifi.sensitive.props.algorithm=PBEWITHMD5AND256BITAES-CBC-OPENSSL +nifi.sensitive.props.provider=BC +nifi.sensitive.props.additional.keys=nifi.ui.banner.text, nifi.version, nifi.sensitive.props.additional.keys + +nifi.security.keystore= +nifi.security.keystoreType= +nifi.security.keystorePasswd= +nifi.security.keyPasswd= +nifi.security.truststore= +nifi.security.truststoreType= +nifi.security.truststorePasswd= +nifi.security.needClientAuth= +nifi.security.user.authorizer= + +# cluster common properties (cluster manager and nodes must have same values) # +nifi.cluster.protocol.heartbeat.interval=5 sec +nifi.cluster.protocol.is.secure=false +nifi.cluster.protocol.socket.timeout=30 sec +nifi.cluster.protocol.connection.handshake.timeout=45 sec +# if multicast is used, then nifi.cluster.protocol.multicast.xxx properties must be configured # +nifi.cluster.protocol.use.multicast=false +nifi.cluster.protocol.multicast.address= +nifi.cluster.protocol.multicast.port= +nifi.cluster.protocol.multicast.service.broadcast.delay=500 ms +nifi.cluster.protocol.multicast.service.locator.attempts=3 +nifi.cluster.protocol.multicast.service.locator.attempts.delay=1 sec + +# cluster node properties (only configure for cluster nodes) # +nifi.cluster.is.node=false +nifi.cluster.node.address= +nifi.cluster.node.protocol.port= +nifi.cluster.node.protocol.threads=2 +# if multicast is not used, nifi.cluster.node.unicast.xxx must have same values as nifi.cluster.manager.xxx # +nifi.cluster.node.unicast.manager.address= +nifi.cluster.node.unicast.manager.protocol.port= +nifi.cluster.node.unicast.manager.authority.provider.port= + +# cluster manager properties (only configure for cluster manager) # +nifi.cluster.is.manager=false +nifi.cluster.manager.address= +nifi.cluster.manager.protocol.port= +nifi.cluster.manager.authority.provider.port= +nifi.cluster.manager.authority.provider.threads=10 +nifi.cluster.manager.node.firewall.file= +nifi.cluster.manager.node.event.history.size=10 +nifi.cluster.manager.node.api.connection.timeout=30 sec +nifi.cluster.manager.node.api.read.timeout=30 sec +nifi.cluster.manager.node.api.request.threads=10 +nifi.cluster.manager.flow.retrieval.delay=5 sec +nifi.cluster.manager.protocol.threads=10 +nifi.cluster.manager.safemode.duration=0 sec diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-properties-loader/src/test/resources/conf/nifi_with_sensitive_properties_protected_aes.properties b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-properties-loader/src/test/resources/conf/nifi_with_sensitive_properties_protected_aes.properties new file mode 100644 index 0000000000..f002cd36cf --- /dev/null +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-properties-loader/src/test/resources/conf/nifi_with_sensitive_properties_protected_aes.properties @@ -0,0 +1,128 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Core Properties # +nifi.version=nifi-test 3.0.0 +nifi.flow.configuration.file=./target/flow.xml.gz +nifi.flow.configuration.archive.dir=./target/archive/ +nifi.flowcontroller.autoResumeState=true +nifi.flowcontroller.graceful.shutdown.period=10 sec +nifi.flowservice.writedelay.interval=2 sec +nifi.administrative.yield.duration=30 sec + +nifi.reporting.task.configuration.file=./target/reporting-tasks.xml +nifi.controller.service.configuration.file=./target/controller-services.xml +nifi.templates.directory=./target/templates +nifi.ui.banner.text=UI Banner Text +nifi.ui.autorefresh.interval=30 sec +nifi.nar.library.directory=./target/resources/NiFiProperties/lib/ +nifi.nar.library.directory.alt=./target/resources/NiFiProperties/lib2/ +nifi.nar.working.directory=./target/work/nar/ + +# H2 Settings +nifi.database.directory=./target/database_repository +nifi.h2.url.append=;LOCK_TIMEOUT=25000;WRITE_DELAY=0;AUTO_SERVER=FALSE + +# FlowFile Repository +nifi.flowfile.repository.directory=./target/test-repo +nifi.flowfile.repository.partitions=1 +nifi.flowfile.repository.checkpoint.interval=2 mins +nifi.queue.swap.threshold=20000 +nifi.swap.storage.directory=./target/test-repo/swap +nifi.swap.in.period=5 sec +nifi.swap.in.threads=1 +nifi.swap.out.period=5 sec +nifi.swap.out.threads=4 + +# Content Repository +nifi.content.claim.max.appendable.size=10 MB +nifi.content.claim.max.flow.files=100 +nifi.content.repository.directory.default=./target/content_repository + +# Provenance Repository Properties +nifi.provenance.repository.storage.directory=./target/provenance_repository +nifi.provenance.repository.max.storage.time=24 hours +nifi.provenance.repository.max.storage.size=1 GB +nifi.provenance.repository.rollover.time=30 secs +nifi.provenance.repository.rollover.size=100 MB + +# Site to Site properties +nifi.remote.input.socket.port=9990 +nifi.remote.input.secure=true + +# web properties # +nifi.web.war.directory=./target/lib +nifi.web.http.host= +nifi.web.http.port= +nifi.web.https.host=nifi.nifi.apache.org +nifi.web.https.port=8443 +nifi.web.jetty.working.directory=./target/work/jetty + +# security properties # +nifi.sensitive.props.key=n2z+tTTbHuZ4V4V2||uWhdasyDXD4ZG2lMAes/vqh6u4vaz4xgL4aEbF4Y/dXevqk3ulRcOwf1vc4RDQ== +nifi.sensitive.props.key.protected=aes/gcm/256 +nifi.sensitive.props.algorithm=PBEWITHMD5AND256BITAES-CBC-OPENSSL +nifi.sensitive.props.provider=BC +nifi.sensitive.props.additional.keys=nifi.ui.banner.text, nifi.version + +nifi.security.keystore=/path/to/keystore.jks +nifi.security.keystoreType=JKS +nifi.security.keystorePasswd=oBjT92hIGRElIGOh||MZ6uYuWNBrOA6usq/Jt3DaD2e4otNirZDytac/w/KFe0HOkrJR03vcbo +nifi.security.keystorePasswd.protected=aes/gcm/256 +nifi.security.keyPasswd=ac/BaE35SL/esLiJ||+ULRvRLYdIDA2VqpE0eQXDEMjaLBMG2kbKOdOwBk/hGebDKlVg== +nifi.security.keyPasswd.protected=aes/gcm/256 +nifi.security.truststore= +nifi.security.truststoreType= +nifi.security.truststorePasswd= +nifi.security.needClientAuth= +nifi.security.user.authorizer= + +# cluster common properties (cluster manager and nodes must have same values) # +nifi.cluster.protocol.heartbeat.interval=5 sec +nifi.cluster.protocol.is.secure=false +nifi.cluster.protocol.socket.timeout=30 sec +nifi.cluster.protocol.connection.handshake.timeout=45 sec +# if multicast is used, then nifi.cluster.protocol.multicast.xxx properties must be configured # +nifi.cluster.protocol.use.multicast=false +nifi.cluster.protocol.multicast.address= +nifi.cluster.protocol.multicast.port= +nifi.cluster.protocol.multicast.service.broadcast.delay=500 ms +nifi.cluster.protocol.multicast.service.locator.attempts=3 +nifi.cluster.protocol.multicast.service.locator.attempts.delay=1 sec + +# cluster node properties (only configure for cluster nodes) # +nifi.cluster.is.node=false +nifi.cluster.node.address= +nifi.cluster.node.protocol.port= +nifi.cluster.node.protocol.threads=2 +# if multicast is not used, nifi.cluster.node.unicast.xxx must have same values as nifi.cluster.manager.xxx # +nifi.cluster.node.unicast.manager.address= +nifi.cluster.node.unicast.manager.protocol.port= +nifi.cluster.node.unicast.manager.authority.provider.port= + +# cluster manager properties (only configure for cluster manager) # +nifi.cluster.is.manager=false +nifi.cluster.manager.address= +nifi.cluster.manager.protocol.port= +nifi.cluster.manager.authority.provider.port= +nifi.cluster.manager.authority.provider.threads=10 +nifi.cluster.manager.node.firewall.file= +nifi.cluster.manager.node.event.history.size=10 +nifi.cluster.manager.node.api.connection.timeout=30 sec +nifi.cluster.manager.node.api.read.timeout=30 sec +nifi.cluster.manager.node.api.request.threads=10 +nifi.cluster.manager.flow.retrieval.delay=5 sec +nifi.cluster.manager.protocol.threads=10 +nifi.cluster.manager.safemode.duration=0 sec diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-properties-loader/src/test/resources/conf/nifi_with_sensitive_properties_protected_aes_multiple_malformed.properties b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-properties-loader/src/test/resources/conf/nifi_with_sensitive_properties_protected_aes_multiple_malformed.properties new file mode 100644 index 0000000000..51c880ea1d --- /dev/null +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-properties-loader/src/test/resources/conf/nifi_with_sensitive_properties_protected_aes_multiple_malformed.properties @@ -0,0 +1,128 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Core Properties # +nifi.version=nifi-test 3.0.0 +nifi.flow.configuration.file=./target/flow.xml.gz +nifi.flow.configuration.archive.dir=./target/archive/ +nifi.flowcontroller.autoResumeState=true +nifi.flowcontroller.graceful.shutdown.period=10 sec +nifi.flowservice.writedelay.interval=2 sec +nifi.administrative.yield.duration=30 sec + +nifi.reporting.task.configuration.file=./target/reporting-tasks.xml +nifi.controller.service.configuration.file=./target/controller-services.xml +nifi.templates.directory=./target/templates +nifi.ui.banner.text=UI Banner Text +nifi.ui.autorefresh.interval=30 sec +nifi.nar.library.directory=./target/resources/NiFiProperties/lib/ +nifi.nar.library.directory.alt=./target/resources/NiFiProperties/lib2/ +nifi.nar.working.directory=./target/work/nar/ + +# H2 Settings +nifi.database.directory=./target/database_repository +nifi.h2.url.append=;LOCK_TIMEOUT=25000;WRITE_DELAY=0;AUTO_SERVER=FALSE + +# FlowFile Repository +nifi.flowfile.repository.directory=./target/test-repo +nifi.flowfile.repository.partitions=1 +nifi.flowfile.repository.checkpoint.interval=2 mins +nifi.queue.swap.threshold=20000 +nifi.swap.storage.directory=./target/test-repo/swap +nifi.swap.in.period=5 sec +nifi.swap.in.threads=1 +nifi.swap.out.period=5 sec +nifi.swap.out.threads=4 + +# Content Repository +nifi.content.claim.max.appendable.size=10 MB +nifi.content.claim.max.flow.files=100 +nifi.content.repository.directory.default=./target/content_repository + +# Provenance Repository Properties +nifi.provenance.repository.storage.directory=./target/provenance_repository +nifi.provenance.repository.max.storage.time=24 hours +nifi.provenance.repository.max.storage.size=1 GB +nifi.provenance.repository.rollover.time=30 secs +nifi.provenance.repository.rollover.size=100 MB + +# Site to Site properties +nifi.remote.input.socket.port=9990 +nifi.remote.input.secure=true + +# web properties # +nifi.web.war.directory=./target/lib +nifi.web.http.host= +nifi.web.http.port= +nifi.web.https.host=nifi.nifi.apache.org +nifi.web.https.port=8443 +nifi.web.jetty.working.directory=./target/work/jetty + +# security properties # +nifi.sensitive.props.key=n2z+tTTbHuZ4V4V2||uWhdasyDXD4ZG2lMAes/vqh6u4vaz4xgL4aEbF4y/dXevqk3ulRcOwf1vc4RDQ== +nifi.sensitive.props.key.protected=aes/gcm/256 +nifi.sensitive.props.algorithm=PBEWITHMD5AND256BITAES-CBC-OPENSSL +nifi.sensitive.props.provider=BC +nifi.sensitive.props.additional.keys=nifi.ui.banner.text, nifi.version + +nifi.security.keystore=/path/to/keystore.jks +nifi.security.keystoreType=JKS +nifi.security.keystorePasswd=oBjT92hIGRElIGOh||MZ6uYuWNBrOA6usq/Jt3DaD2e4otNirZDytaC/w/KFe0HOkrJR03vcbo +nifi.security.keystorePasswd.protected=aes/gcm/256 +nifi.security.keyPasswd=ac/BaE35SL/esLiJ||+ULRvRLYdIDA2VqpE0eQXDEMjaLBMG2kbKOdOwBK/hGebDKlVg== +nifi.security.keyPasswd.protected=aes/gcm/256 +nifi.security.truststore= +nifi.security.truststoreType= +nifi.security.truststorePasswd= +nifi.security.needClientAuth= +nifi.security.user.authorizer= + +# cluster common properties (cluster manager and nodes must have same values) # +nifi.cluster.protocol.heartbeat.interval=5 sec +nifi.cluster.protocol.is.secure=false +nifi.cluster.protocol.socket.timeout=30 sec +nifi.cluster.protocol.connection.handshake.timeout=45 sec +# if multicast is used, then nifi.cluster.protocol.multicast.xxx properties must be configured # +nifi.cluster.protocol.use.multicast=false +nifi.cluster.protocol.multicast.address= +nifi.cluster.protocol.multicast.port= +nifi.cluster.protocol.multicast.service.broadcast.delay=500 ms +nifi.cluster.protocol.multicast.service.locator.attempts=3 +nifi.cluster.protocol.multicast.service.locator.attempts.delay=1 sec + +# cluster node properties (only configure for cluster nodes) # +nifi.cluster.is.node=false +nifi.cluster.node.address= +nifi.cluster.node.protocol.port= +nifi.cluster.node.protocol.threads=2 +# if multicast is not used, nifi.cluster.node.unicast.xxx must have same values as nifi.cluster.manager.xxx # +nifi.cluster.node.unicast.manager.address= +nifi.cluster.node.unicast.manager.protocol.port= +nifi.cluster.node.unicast.manager.authority.provider.port= + +# cluster manager properties (only configure for cluster manager) # +nifi.cluster.is.manager=false +nifi.cluster.manager.address= +nifi.cluster.manager.protocol.port= +nifi.cluster.manager.authority.provider.port= +nifi.cluster.manager.authority.provider.threads=10 +nifi.cluster.manager.node.firewall.file= +nifi.cluster.manager.node.event.history.size=10 +nifi.cluster.manager.node.api.connection.timeout=30 sec +nifi.cluster.manager.node.api.read.timeout=30 sec +nifi.cluster.manager.node.api.request.threads=10 +nifi.cluster.manager.flow.retrieval.delay=5 sec +nifi.cluster.manager.protocol.threads=10 +nifi.cluster.manager.safemode.duration=0 sec diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-properties-loader/src/test/resources/conf/nifi_with_sensitive_properties_protected_aes_single_malformed.properties b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-properties-loader/src/test/resources/conf/nifi_with_sensitive_properties_protected_aes_single_malformed.properties new file mode 100644 index 0000000000..6078bf78fd --- /dev/null +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-properties-loader/src/test/resources/conf/nifi_with_sensitive_properties_protected_aes_single_malformed.properties @@ -0,0 +1,128 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Core Properties # +nifi.version=nifi-test 3.0.0 +nifi.flow.configuration.file=./target/flow.xml.gz +nifi.flow.configuration.archive.dir=./target/archive/ +nifi.flowcontroller.autoResumeState=true +nifi.flowcontroller.graceful.shutdown.period=10 sec +nifi.flowservice.writedelay.interval=2 sec +nifi.administrative.yield.duration=30 sec + +nifi.reporting.task.configuration.file=./target/reporting-tasks.xml +nifi.controller.service.configuration.file=./target/controller-services.xml +nifi.templates.directory=./target/templates +nifi.ui.banner.text=UI Banner Text +nifi.ui.autorefresh.interval=30 sec +nifi.nar.library.directory=./target/resources/NiFiProperties/lib/ +nifi.nar.library.directory.alt=./target/resources/NiFiProperties/lib2/ +nifi.nar.working.directory=./target/work/nar/ + +# H2 Settings +nifi.database.directory=./target/database_repository +nifi.h2.url.append=;LOCK_TIMEOUT=25000;WRITE_DELAY=0;AUTO_SERVER=FALSE + +# FlowFile Repository +nifi.flowfile.repository.directory=./target/test-repo +nifi.flowfile.repository.partitions=1 +nifi.flowfile.repository.checkpoint.interval=2 mins +nifi.queue.swap.threshold=20000 +nifi.swap.storage.directory=./target/test-repo/swap +nifi.swap.in.period=5 sec +nifi.swap.in.threads=1 +nifi.swap.out.period=5 sec +nifi.swap.out.threads=4 + +# Content Repository +nifi.content.claim.max.appendable.size=10 MB +nifi.content.claim.max.flow.files=100 +nifi.content.repository.directory.default=./target/content_repository + +# Provenance Repository Properties +nifi.provenance.repository.storage.directory=./target/provenance_repository +nifi.provenance.repository.max.storage.time=24 hours +nifi.provenance.repository.max.storage.size=1 GB +nifi.provenance.repository.rollover.time=30 secs +nifi.provenance.repository.rollover.size=100 MB + +# Site to Site properties +nifi.remote.input.socket.port=9990 +nifi.remote.input.secure=true + +# web properties # +nifi.web.war.directory=./target/lib +nifi.web.http.host= +nifi.web.http.port= +nifi.web.https.host=nifi.nifi.apache.org +nifi.web.https.port=8443 +nifi.web.jetty.working.directory=./target/work/jetty + +# security properties # +nifi.sensitive.props.key=n2z+tTTbHuZ4V4V2||uWhdasyDXD4ZG2lMAes/vqh6u4vaz4xgL4aEbF4Y/dXevqk3ulRcOwf1vc4RDQ== +nifi.sensitive.props.key.protected=aes/gcm/256 +nifi.sensitive.props.algorithm=PBEWITHMD5AND256BITAES-CBC-OPENSSL +nifi.sensitive.props.provider=BC +nifi.sensitive.props.additional.keys=nifi.ui.banner.text, nifi.version + +nifi.security.keystore=/path/to/keystore.jks +nifi.security.keystoreType=JKS +nifi.security.keystorePasswd=oBjT92hIGRElIGOh||MZ6uYuWNBrOA6usq/Jt3DaD2e4otNirZDytac/w/KFe0HOkrJR03 +nifi.security.keystorePasswd.protected=aes/gcm/256 +nifi.security.keyPasswd=ac/BaE35SL/esLiJ||+ULRvRLYdIDA2VqpE0eQXDEMjaLBMG2kbKOdOwBk/hGebDKlVg== +nifi.security.keyPasswd.protected=aes/gcm/256 +nifi.security.truststore= +nifi.security.truststoreType= +nifi.security.truststorePasswd= +nifi.security.needClientAuth= +nifi.security.user.authorizer= + +# cluster common properties (cluster manager and nodes must have same values) # +nifi.cluster.protocol.heartbeat.interval=5 sec +nifi.cluster.protocol.is.secure=false +nifi.cluster.protocol.socket.timeout=30 sec +nifi.cluster.protocol.connection.handshake.timeout=45 sec +# if multicast is used, then nifi.cluster.protocol.multicast.xxx properties must be configured # +nifi.cluster.protocol.use.multicast=false +nifi.cluster.protocol.multicast.address= +nifi.cluster.protocol.multicast.port= +nifi.cluster.protocol.multicast.service.broadcast.delay=500 ms +nifi.cluster.protocol.multicast.service.locator.attempts=3 +nifi.cluster.protocol.multicast.service.locator.attempts.delay=1 sec + +# cluster node properties (only configure for cluster nodes) # +nifi.cluster.is.node=false +nifi.cluster.node.address= +nifi.cluster.node.protocol.port= +nifi.cluster.node.protocol.threads=2 +# if multicast is not used, nifi.cluster.node.unicast.xxx must have same values as nifi.cluster.manager.xxx # +nifi.cluster.node.unicast.manager.address= +nifi.cluster.node.unicast.manager.protocol.port= +nifi.cluster.node.unicast.manager.authority.provider.port= + +# cluster manager properties (only configure for cluster manager) # +nifi.cluster.is.manager=false +nifi.cluster.manager.address= +nifi.cluster.manager.protocol.port= +nifi.cluster.manager.authority.provider.port= +nifi.cluster.manager.authority.provider.threads=10 +nifi.cluster.manager.node.firewall.file= +nifi.cluster.manager.node.event.history.size=10 +nifi.cluster.manager.node.api.connection.timeout=30 sec +nifi.cluster.manager.node.api.read.timeout=30 sec +nifi.cluster.manager.node.api.request.threads=10 +nifi.cluster.manager.flow.retrieval.delay=5 sec +nifi.cluster.manager.protocol.threads=10 +nifi.cluster.manager.safemode.duration=0 sec diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-properties-loader/src/test/resources/conf/nifi_with_sensitive_properties_protected_unknown.properties b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-properties-loader/src/test/resources/conf/nifi_with_sensitive_properties_protected_unknown.properties new file mode 100644 index 0000000000..07c5ba7b0b --- /dev/null +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-properties-loader/src/test/resources/conf/nifi_with_sensitive_properties_protected_unknown.properties @@ -0,0 +1,128 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Core Properties # +nifi.version=nifi-test 3.0.0 +nifi.flow.configuration.file=./target/flow.xml.gz +nifi.flow.configuration.archive.dir=./target/archive/ +nifi.flowcontroller.autoResumeState=true +nifi.flowcontroller.graceful.shutdown.period=10 sec +nifi.flowservice.writedelay.interval=2 sec +nifi.administrative.yield.duration=30 sec + +nifi.reporting.task.configuration.file=./target/reporting-tasks.xml +nifi.controller.service.configuration.file=./target/controller-services.xml +nifi.templates.directory=./target/templates +nifi.ui.banner.text=UI Banner Text +nifi.ui.autorefresh.interval=30 sec +nifi.nar.library.directory=./target/resources/NiFiProperties/lib/ +nifi.nar.library.directory.alt=./target/resources/NiFiProperties/lib2/ +nifi.nar.working.directory=./target/work/nar/ + +# H2 Settings +nifi.database.directory=./target/database_repository +nifi.h2.url.append=;LOCK_TIMEOUT=25000;WRITE_DELAY=0;AUTO_SERVER=FALSE + +# FlowFile Repository +nifi.flowfile.repository.directory=./target/test-repo +nifi.flowfile.repository.partitions=1 +nifi.flowfile.repository.checkpoint.interval=2 mins +nifi.queue.swap.threshold=20000 +nifi.swap.storage.directory=./target/test-repo/swap +nifi.swap.in.period=5 sec +nifi.swap.in.threads=1 +nifi.swap.out.period=5 sec +nifi.swap.out.threads=4 + +# Content Repository +nifi.content.claim.max.appendable.size=10 MB +nifi.content.claim.max.flow.files=100 +nifi.content.repository.directory.default=./target/content_repository + +# Provenance Repository Properties +nifi.provenance.repository.storage.directory=./target/provenance_repository +nifi.provenance.repository.max.storage.time=24 hours +nifi.provenance.repository.max.storage.size=1 GB +nifi.provenance.repository.rollover.time=30 secs +nifi.provenance.repository.rollover.size=100 MB + +# Site to Site properties +nifi.remote.input.socket.port=9990 +nifi.remote.input.secure=true + +# web properties # +nifi.web.war.directory=./target/lib +nifi.web.http.host= +nifi.web.http.port= +nifi.web.https.host=nifi.nifi.apache.org +nifi.web.https.port=8443 +nifi.web.jetty.working.directory=./target/work/jetty + +# security properties # +nifi.sensitive.props.key=n2z+tTTbHuZ4V4V2||uWhdasyDXD4ZG2lMAes/vqh6u4vaz4xgL4aEbF4Y/dXevqk3ulRcOwf1vc4RDQ== +nifi.sensitive.props.key.protected=unknown +nifi.sensitive.props.algorithm=PBEWITHMD5AND256BITAES-CBC-OPENSSL +nifi.sensitive.props.provider=BC +nifi.sensitive.props.additional.keys=nifi.ui.banner.text, nifi.version + +nifi.security.keystore=/path/to/keystore.jks +nifi.security.keystoreType=JKS +nifi.security.keystorePasswd=oBjT92hIGRElIGOh||MZ6uYuWNBrOA6usq/Jt3DaD2e4otNirZDytac/w/KFe0HOkrJR03vcbo +nifi.security.keystorePasswd.protected=unknown +nifi.security.keyPasswd=ac/BaE35SL/esLiJ||+ULRvRLYdIDA2VqpE0eQXDEMjaLBMG2kbKOdOwBk/hGebDKlVg== +nifi.security.keyPasswd.protected=unknown +nifi.security.truststore= +nifi.security.truststoreType= +nifi.security.truststorePasswd= +nifi.security.needClientAuth= +nifi.security.user.authorizer= + +# cluster common properties (cluster manager and nodes must have same values) # +nifi.cluster.protocol.heartbeat.interval=5 sec +nifi.cluster.protocol.is.secure=false +nifi.cluster.protocol.socket.timeout=30 sec +nifi.cluster.protocol.connection.handshake.timeout=45 sec +# if multicast is used, then nifi.cluster.protocol.multicast.xxx properties must be configured # +nifi.cluster.protocol.use.multicast=false +nifi.cluster.protocol.multicast.address= +nifi.cluster.protocol.multicast.port= +nifi.cluster.protocol.multicast.service.broadcast.delay=500 ms +nifi.cluster.protocol.multicast.service.locator.attempts=3 +nifi.cluster.protocol.multicast.service.locator.attempts.delay=1 sec + +# cluster node properties (only configure for cluster nodes) # +nifi.cluster.is.node=false +nifi.cluster.node.address= +nifi.cluster.node.protocol.port= +nifi.cluster.node.protocol.threads=2 +# if multicast is not used, nifi.cluster.node.unicast.xxx must have same values as nifi.cluster.manager.xxx # +nifi.cluster.node.unicast.manager.address= +nifi.cluster.node.unicast.manager.protocol.port= +nifi.cluster.node.unicast.manager.authority.provider.port= + +# cluster manager properties (only configure for cluster manager) # +nifi.cluster.is.manager=false +nifi.cluster.manager.address= +nifi.cluster.manager.protocol.port= +nifi.cluster.manager.authority.provider.port= +nifi.cluster.manager.authority.provider.threads=10 +nifi.cluster.manager.node.firewall.file= +nifi.cluster.manager.node.event.history.size=10 +nifi.cluster.manager.node.api.connection.timeout=30 sec +nifi.cluster.manager.node.api.read.timeout=30 sec +nifi.cluster.manager.node.api.request.threads=10 +nifi.cluster.manager.flow.retrieval.delay=5 sec +nifi.cluster.manager.protocol.threads=10 +nifi.cluster.manager.safemode.duration=0 sec diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-properties-loader/src/test/resources/conf/nifi_with_sensitive_properties_unprotected.properties b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-properties-loader/src/test/resources/conf/nifi_with_sensitive_properties_unprotected.properties new file mode 100644 index 0000000000..b81c369829 --- /dev/null +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-properties-loader/src/test/resources/conf/nifi_with_sensitive_properties_unprotected.properties @@ -0,0 +1,125 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Core Properties # +nifi.version=nifi-test 3.0.0 +nifi.flow.configuration.file=./target/flow.xml.gz +nifi.flow.configuration.archive.dir=./target/archive/ +nifi.flowcontroller.autoResumeState=true +nifi.flowcontroller.graceful.shutdown.period=10 sec +nifi.flowservice.writedelay.interval=2 sec +nifi.administrative.yield.duration=30 sec + +nifi.reporting.task.configuration.file=./target/reporting-tasks.xml +nifi.controller.service.configuration.file=./target/controller-services.xml +nifi.templates.directory=./target/templates +nifi.ui.banner.text=UI Banner Text +nifi.ui.autorefresh.interval=30 sec +nifi.nar.library.directory=./target/resources/NiFiProperties/lib/ +nifi.nar.library.directory.alt=./target/resources/NiFiProperties/lib2/ +nifi.nar.working.directory=./target/work/nar/ + +# H2 Settings +nifi.database.directory=./target/database_repository +nifi.h2.url.append=;LOCK_TIMEOUT=25000;WRITE_DELAY=0;AUTO_SERVER=FALSE + +# FlowFile Repository +nifi.flowfile.repository.directory=./target/test-repo +nifi.flowfile.repository.partitions=1 +nifi.flowfile.repository.checkpoint.interval=2 mins +nifi.queue.swap.threshold=20000 +nifi.swap.storage.directory=./target/test-repo/swap +nifi.swap.in.period=5 sec +nifi.swap.in.threads=1 +nifi.swap.out.period=5 sec +nifi.swap.out.threads=4 + +# Content Repository +nifi.content.claim.max.appendable.size=10 MB +nifi.content.claim.max.flow.files=100 +nifi.content.repository.directory.default=./target/content_repository + +# Provenance Repository Properties +nifi.provenance.repository.storage.directory=./target/provenance_repository +nifi.provenance.repository.max.storage.time=24 hours +nifi.provenance.repository.max.storage.size=1 GB +nifi.provenance.repository.rollover.time=30 secs +nifi.provenance.repository.rollover.size=100 MB + +# Site to Site properties +nifi.remote.input.socket.port=9990 +nifi.remote.input.secure=true + +# web properties # +nifi.web.war.directory=./target/lib +nifi.web.http.host= +nifi.web.http.port= +nifi.web.https.host=nifi.nifi.apache.org +nifi.web.https.port=8443 +nifi.web.jetty.working.directory=./target/work/jetty + +# security properties # +nifi.sensitive.props.key=thisIsABadSensitiveKeyPassword +nifi.sensitive.props.algorithm=PBEWITHMD5AND256BITAES-CBC-OPENSSL +nifi.sensitive.props.provider=BC +nifi.sensitive.props.additional.keys=nifi.ui.banner.text, nifi.version + +nifi.security.keystore=/path/to/keystore.jks +nifi.security.keystoreType=JKS +nifi.security.keystorePasswd=thisIsABadKeystorePassword +nifi.security.keyPasswd=thisIsABadKeyPassword +nifi.security.truststore= +nifi.security.truststoreType= +nifi.security.truststorePasswd= +nifi.security.needClientAuth= +nifi.security.user.authorizer= + +# cluster common properties (cluster manager and nodes must have same values) # +nifi.cluster.protocol.heartbeat.interval=5 sec +nifi.cluster.protocol.is.secure=false +nifi.cluster.protocol.socket.timeout=30 sec +nifi.cluster.protocol.connection.handshake.timeout=45 sec +# if multicast is used, then nifi.cluster.protocol.multicast.xxx properties must be configured # +nifi.cluster.protocol.use.multicast=false +nifi.cluster.protocol.multicast.address= +nifi.cluster.protocol.multicast.port= +nifi.cluster.protocol.multicast.service.broadcast.delay=500 ms +nifi.cluster.protocol.multicast.service.locator.attempts=3 +nifi.cluster.protocol.multicast.service.locator.attempts.delay=1 sec + +# cluster node properties (only configure for cluster nodes) # +nifi.cluster.is.node=false +nifi.cluster.node.address= +nifi.cluster.node.protocol.port= +nifi.cluster.node.protocol.threads=2 +# if multicast is not used, nifi.cluster.node.unicast.xxx must have same values as nifi.cluster.manager.xxx # +nifi.cluster.node.unicast.manager.address= +nifi.cluster.node.unicast.manager.protocol.port= +nifi.cluster.node.unicast.manager.authority.provider.port= + +# cluster manager properties (only configure for cluster manager) # +nifi.cluster.is.manager=false +nifi.cluster.manager.address= +nifi.cluster.manager.protocol.port= +nifi.cluster.manager.authority.provider.port= +nifi.cluster.manager.authority.provider.threads=10 +nifi.cluster.manager.node.firewall.file= +nifi.cluster.manager.node.event.history.size=10 +nifi.cluster.manager.node.api.connection.timeout=30 sec +nifi.cluster.manager.node.api.read.timeout=30 sec +nifi.cluster.manager.node.api.request.threads=10 +nifi.cluster.manager.flow.retrieval.delay=5 sec +nifi.cluster.manager.protocol.threads=10 +nifi.cluster.manager.safemode.duration=0 sec diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-properties-loader/src/test/resources/conf/nifi_with_sensitive_properties_unprotected_extra_line.properties b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-properties-loader/src/test/resources/conf/nifi_with_sensitive_properties_unprotected_extra_line.properties new file mode 100644 index 0000000000..4ad5784866 --- /dev/null +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-properties-loader/src/test/resources/conf/nifi_with_sensitive_properties_unprotected_extra_line.properties @@ -0,0 +1,126 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Core Properties # +nifi.version=nifi-test 3.0.0 +nifi.flow.configuration.file=./target/flow.xml.gz +nifi.flow.configuration.archive.dir=./target/archive/ +nifi.flowcontroller.autoResumeState=true +nifi.flowcontroller.graceful.shutdown.period=10 sec +nifi.flowservice.writedelay.interval=2 sec +nifi.administrative.yield.duration=30 sec + +nifi.reporting.task.configuration.file=./target/reporting-tasks.xml +nifi.controller.service.configuration.file=./target/controller-services.xml +nifi.templates.directory=./target/templates +nifi.ui.banner.text=UI Banner Text +nifi.ui.autorefresh.interval=30 sec +nifi.nar.library.directory=./target/resources/NiFiProperties/lib/ +nifi.nar.library.directory.alt=./target/resources/NiFiProperties/lib2/ +nifi.nar.working.directory=./target/work/nar/ + +# H2 Settings +nifi.database.directory=./target/database_repository +nifi.h2.url.append=;LOCK_TIMEOUT=25000;WRITE_DELAY=0;AUTO_SERVER=FALSE + +# FlowFile Repository +nifi.flowfile.repository.directory=./target/test-repo +nifi.flowfile.repository.partitions=1 +nifi.flowfile.repository.checkpoint.interval=2 mins +nifi.queue.swap.threshold=20000 +nifi.swap.storage.directory=./target/test-repo/swap +nifi.swap.in.period=5 sec +nifi.swap.in.threads=1 +nifi.swap.out.period=5 sec +nifi.swap.out.threads=4 + +# Content Repository +nifi.content.claim.max.appendable.size=10 MB +nifi.content.claim.max.flow.files=100 +nifi.content.repository.directory.default=./target/content_repository + +# Provenance Repository Properties +nifi.provenance.repository.storage.directory=./target/provenance_repository +nifi.provenance.repository.max.storage.time=24 hours +nifi.provenance.repository.max.storage.size=1 GB +nifi.provenance.repository.rollover.time=30 secs +nifi.provenance.repository.rollover.size=100 MB + +# Site to Site properties +nifi.remote.input.socket.port=9990 +nifi.remote.input.secure=true + +# web properties # +nifi.web.war.directory=./target/lib +nifi.web.http.host= +nifi.web.http.port= +nifi.web.https.host=nifi.nifi.apache.org +nifi.web.https.port=8443 +nifi.web.jetty.working.directory=./target/work/jetty + +# security properties # +nifi.sensitive.props.key=thisIsABadSensitiveKeyPassword +nifi.sensitive.props.key.protected= +nifi.sensitive.props.algorithm=PBEWITHMD5AND256BITAES-CBC-OPENSSL +nifi.sensitive.props.provider=BC +nifi.sensitive.props.additional.keys=nifi.ui.banner.text, nifi.version + +nifi.security.keystore=/path/to/keystore.jks +nifi.security.keystoreType=JKS +nifi.security.keystorePasswd=thisIsABadKeystorePassword +nifi.security.keyPasswd=thisIsABadKeyPassword +nifi.security.truststore= +nifi.security.truststoreType= +nifi.security.truststorePasswd= +nifi.security.needClientAuth= +nifi.security.user.authorizer= + +# cluster common properties (cluster manager and nodes must have same values) # +nifi.cluster.protocol.heartbeat.interval=5 sec +nifi.cluster.protocol.is.secure=false +nifi.cluster.protocol.socket.timeout=30 sec +nifi.cluster.protocol.connection.handshake.timeout=45 sec +# if multicast is used, then nifi.cluster.protocol.multicast.xxx properties must be configured # +nifi.cluster.protocol.use.multicast=false +nifi.cluster.protocol.multicast.address= +nifi.cluster.protocol.multicast.port= +nifi.cluster.protocol.multicast.service.broadcast.delay=500 ms +nifi.cluster.protocol.multicast.service.locator.attempts=3 +nifi.cluster.protocol.multicast.service.locator.attempts.delay=1 sec + +# cluster node properties (only configure for cluster nodes) # +nifi.cluster.is.node=false +nifi.cluster.node.address= +nifi.cluster.node.protocol.port= +nifi.cluster.node.protocol.threads=2 +# if multicast is not used, nifi.cluster.node.unicast.xxx must have same values as nifi.cluster.manager.xxx # +nifi.cluster.node.unicast.manager.address= +nifi.cluster.node.unicast.manager.protocol.port= +nifi.cluster.node.unicast.manager.authority.provider.port= + +# cluster manager properties (only configure for cluster manager) # +nifi.cluster.is.manager=false +nifi.cluster.manager.address= +nifi.cluster.manager.protocol.port= +nifi.cluster.manager.authority.provider.port= +nifi.cluster.manager.authority.provider.threads=10 +nifi.cluster.manager.node.firewall.file= +nifi.cluster.manager.node.event.history.size=10 +nifi.cluster.manager.node.api.connection.timeout=30 sec +nifi.cluster.manager.node.api.read.timeout=30 sec +nifi.cluster.manager.node.api.request.threads=10 +nifi.cluster.manager.flow.retrieval.delay=5 sec +nifi.cluster.manager.protocol.threads=10 +nifi.cluster.manager.safemode.duration=0 sec diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-resources/pom.xml b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-resources/pom.xml index e367e2af14..5938fb276b 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-resources/pom.xml +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-resources/pom.xml @@ -81,8 +81,11 @@ ./work/nar/ ./work/docs/components + PBEWITHMD5AND256BITAES-CBC-OPENSSL BC + + ;LOCK_TIMEOUT=25000;WRITE_DELAY=0;AUTO_SERVER=FALSE 9990 diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-resources/src/main/resources/conf/bootstrap.conf b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-resources/src/main/resources/conf/bootstrap.conf index 55679c2930..f4a6c29072 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-resources/src/main/resources/conf/bootstrap.conf +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-resources/src/main/resources/conf/bootstrap.conf @@ -51,6 +51,8 @@ java.arg.13=-XX:+UseG1GC #Set headless mode by default java.arg.14=-Djava.awt.headless=true +#Set sensitive properties key +nifi.bootstrap.sensitive.key= ### # Notification Services for notifying interested parties when NiFi is stopped, started, dies diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-resources/src/main/resources/conf/nifi.properties b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-resources/src/main/resources/conf/nifi.properties index 485b60e494..bc5611e67f 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-resources/src/main/resources/conf/nifi.properties +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-resources/src/main/resources/conf/nifi.properties @@ -131,8 +131,10 @@ nifi.web.jetty.threads=${nifi.web.jetty.threads} # security properties # nifi.sensitive.props.key= +nifi.sensitive.props.key.protected=${nifi.sensitive.props.key.protected} nifi.sensitive.props.algorithm=${nifi.sensitive.props.algorithm} nifi.sensitive.props.provider=${nifi.sensitive.props.provider} +nifi.sensitive.props.additional.keys=${nifi.sensitive.props.additional.keys} nifi.security.keystore=${nifi.security.keystore} nifi.security.keystoreType=${nifi.security.keystoreType} diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-runtime/pom.xml b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-runtime/pom.xml index 8f4f9547fb..830b83e656 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-runtime/pom.xml +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-runtime/pom.xml @@ -32,6 +32,11 @@ nifi-properties compile + + org.apache.nifi + nifi-properties-loader + compile + org.apache.nifi nifi-documentation @@ -41,5 +46,10 @@ org.slf4j jul-to-slf4j + + ch.qos.logback + logback-classic + test + diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-runtime/src/main/java/org/apache/nifi/NiFi.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-runtime/src/main/java/org/apache/nifi/NiFi.java index 6d0fa97c95..44529d2ee6 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-runtime/src/main/java/org/apache/nifi/NiFi.java +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-runtime/src/main/java/org/apache/nifi/NiFi.java @@ -21,6 +21,9 @@ import java.io.IOException; import java.lang.Thread.UncaughtExceptionHandler; import java.lang.reflect.Constructor; import java.lang.reflect.InvocationTargetException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; import java.util.Timer; import java.util.TimerTask; import java.util.concurrent.Executors; @@ -30,12 +33,13 @@ import java.util.concurrent.ThreadFactory; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicLong; - import org.apache.nifi.documentation.DocGenerator; import org.apache.nifi.nar.ExtensionManager; import org.apache.nifi.nar.ExtensionMapping; import org.apache.nifi.nar.NarClassLoaders; import org.apache.nifi.nar.NarUnpacker; +import org.apache.nifi.properties.NiFiPropertiesLoader; +import org.apache.nifi.properties.SensitivePropertyProtectionException; import org.apache.nifi.util.FileUtils; import org.apache.nifi.util.NiFiProperties; import org.slf4j.Logger; @@ -45,6 +49,7 @@ import org.slf4j.bridge.SLF4JBridgeHandler; public class NiFi { private static final Logger logger = LoggerFactory.getLogger(NiFi.class); + private static final String KEY_FLAG = "-k"; private final NiFiServer nifiServer; private final BootstrapListener bootstrapListener; @@ -158,7 +163,7 @@ public class NiFi { } logger.info("Jetty web server shutdown completed (nicely or otherwise)."); } catch (final Throwable t) { - logger.warn("Problem occured ensuring Jetty web server was properly terminated due to " + t); + logger.warn("Problem occurred ensuring Jetty web server was properly terminated due to " + t); } } @@ -183,14 +188,14 @@ public class NiFi { }); final AtomicInteger occurrencesOutOfRange = new AtomicInteger(0); - final AtomicInteger occurences = new AtomicInteger(0); + final AtomicInteger occurrences = new AtomicInteger(0); final Runnable command = new Runnable() { @Override public void run() { final long curMillis = System.currentTimeMillis(); final long difference = curMillis - lastTriggerMillis.get(); final long millisOff = Math.abs(difference - 2000L); - occurences.incrementAndGet(); + occurrences.incrementAndGet(); if (millisOff > 500L) { occurrencesOutOfRange.incrementAndGet(); } @@ -206,7 +211,7 @@ public class NiFi { future.cancel(true); service.shutdownNow(); - if (occurences.get() < minRequiredOccurrences || occurrencesOutOfRange.get() > maxOccurrencesOutOfRange) { + if (occurrences.get() < minRequiredOccurrences || occurrencesOutOfRange.get() > maxOccurrencesOutOfRange) { logger.warn("NiFi has detected that this box is not responding within the expected timing interval, which may cause " + "Processors to be scheduled erratically. Please see the NiFi documentation for more information."); } @@ -224,9 +229,96 @@ public class NiFi { public static void main(String[] args) { logger.info("Launching NiFi..."); try { - new NiFi(NiFiProperties.createBasicNiFiProperties(null, null)); + NiFiProperties properties = initializeProperties(args); + new NiFi(properties); } catch (final Throwable t) { logger.error("Failure to launch NiFi due to " + t, t); } } + + private static NiFiProperties initializeProperties(String[] args) { + // Try to get key + // If key doesn't exist, instantiate without + // Load properties + // If properties are protected and key missing, throw RuntimeException + + try { + String key = loadFormattedKey(args); + // The key might be empty or null when it is passed to the loader + try { + NiFiProperties properties = NiFiPropertiesLoader.withKey(key).get(); + logger.info("Loaded {} properties", properties.size()); + return properties; + } catch (SensitivePropertyProtectionException e) { + final String msg = "There was an issue decrypting protected properties"; + logger.error(msg, e); + throw new IllegalArgumentException(msg); + } + } catch (IllegalArgumentException e) { + final String msg = "The bootstrap process did not provide a valid key and there are protected properties present in the properties file"; + logger.error(msg, e); + throw new IllegalArgumentException(msg); + } + } + + private static String loadFormattedKey(String[] args) { + String key = null; + List parsedArgs = parseArgs(args); + // Check if args contain protection key + if (parsedArgs.contains(KEY_FLAG)) { + key = getKeyFromArgs(parsedArgs); + + // Format the key (check hex validity and remove spaces) + key = formatHexKey(key); + if (!isHexKeyValid(key)) { + throw new IllegalArgumentException("The key was not provided in valid hex format and of the correct length"); + } + + return key; + } else { + // throw new IllegalStateException("No key provided from bootstrap"); + return ""; + } + } + + private static String getKeyFromArgs(List parsedArgs) { + String key; + logger.debug("The bootstrap process provided the " + KEY_FLAG + " flag"); + int i = parsedArgs.indexOf(KEY_FLAG); + if (parsedArgs.size() <= i + 1) { + logger.error("The bootstrap process passed the {} flag without a key", KEY_FLAG); + throw new IllegalArgumentException("The bootstrap process provided the " + KEY_FLAG + " flag but no key"); + } + key = parsedArgs.get(i + 1); + logger.info("Read property protection key from bootstrap process"); + return key; + } + + private static List parseArgs(String[] args) { + List parsedArgs = new ArrayList<>(Arrays.asList(args)); + for (int i = 0; i < parsedArgs.size(); i++) { + if (parsedArgs.get(i).startsWith(KEY_FLAG + " ")) { + String[] split = parsedArgs.get(i).split(" ", 2); + parsedArgs.set(i, split[0]); + parsedArgs.add(i + 1, split[1]); + break; + } + } + return parsedArgs; + } + + private static String formatHexKey(String input) { + if (input == null || input.trim().isEmpty()) { + return ""; + } + return input.replaceAll("[^0-9a-fA-F]", "").toLowerCase(); + } + + private static boolean isHexKeyValid(String key) { + if (key == null || key.trim().isEmpty()) { + return false; + } + // Key length is in "nibbles" (i.e. one hex char = 4 bits) + return Arrays.asList(128, 196, 256).contains(key.length() * 4) && key.matches("^[0-9a-fA-F]*$"); + } } diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-runtime/src/test/groovy/org/apache/nifi/NiFiGroovyTest.groovy b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-runtime/src/test/groovy/org/apache/nifi/NiFiGroovyTest.groovy new file mode 100644 index 0000000000..f192d791b7 --- /dev/null +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-runtime/src/test/groovy/org/apache/nifi/NiFiGroovyTest.groovy @@ -0,0 +1,277 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi + +import ch.qos.logback.classic.spi.LoggingEvent +import ch.qos.logback.core.AppenderBase +import org.apache.nifi.properties.AESSensitivePropertyProvider +import org.apache.nifi.properties.NiFiPropertiesLoader +import org.apache.nifi.properties.StandardNiFiProperties +import org.apache.nifi.util.NiFiProperties +import org.bouncycastle.jce.provider.BouncyCastleProvider +import org.junit.After +import org.junit.AfterClass +import org.junit.Before +import org.junit.BeforeClass +import org.junit.Test +import org.junit.runner.RunWith +import org.junit.runners.JUnit4 +import org.slf4j.Logger +import org.slf4j.LoggerFactory +import org.slf4j.bridge.SLF4JBridgeHandler + +import java.security.Security + +@RunWith(JUnit4.class) +class NiFiGroovyTest extends GroovyTestCase { + private static final Logger logger = LoggerFactory.getLogger(NiFiGroovyTest.class) + + private static String originalPropertiesPath = System.getProperty(NiFiProperties.PROPERTIES_FILE_PATH) + + private static final String TEST_RES_PATH = NiFiGroovyTest.getClassLoader().getResource(".").toURI().getPath() + private static final File workDir = new File("./target/work/jetty/") + + @BeforeClass + public static void setUpOnce() throws Exception { + Security.addProvider(new BouncyCastleProvider()) + + SLF4JBridgeHandler.install() + + logger.metaClass.methodMissing = { String name, args -> + logger.info("[${name?.toUpperCase()}] ${(args as List).join(" ")}") + } + + logger.info("Identified test resources path as ${TEST_RES_PATH}") + } + + @Before + public void setUp() throws Exception { + if (!workDir.exists()) { + workDir.mkdirs() + } + } + + @After + public void tearDown() throws Exception { + NiFiPropertiesLoader.@sensitivePropertyProviderFactory = null + TestAppender.reset() + System.setIn(System.in) + } + + @AfterClass + public static void tearDownOnce() { + if (originalPropertiesPath) { + System.setProperty(NiFiProperties.PROPERTIES_FILE_PATH, originalPropertiesPath) + } + } + + @Test + public void testInitializePropertiesShouldHandleNoBootstrapKey() throws Exception { + // Arrange + def args = [] as String[] + + String plainPropertiesPath = "${TEST_RES_PATH}/NiFiProperties/conf/nifi.properties" + System.setProperty(NiFiProperties.PROPERTIES_FILE_PATH, plainPropertiesPath) + + // Act + NiFiProperties loadedProperties = NiFi.initializeProperties(args) + + // Assert + assert loadedProperties.size() > 0 + } + + @Test + public void testMainShouldHandleNoBootstrapKeyWithProtectedProperties() throws Exception { + // Arrange + def args = [] as String[] + + System.setProperty(NiFiProperties.PROPERTIES_FILE_PATH, "${TEST_RES_PATH}/NiFiProperties/conf/nifi_with_sensitive_properties_protected_aes_different_key.properties") + + // Act + NiFi.main(args) + + // Assert + assert TestAppender.events.last().getMessage() == "Failure to launch NiFi due to java.lang.IllegalArgumentException: The bootstrap process did not provide a valid key and there are protected properties present in the properties file" + } + + @Test + public void testParseArgsShouldSplitCombinedArgs() throws Exception { + // Arrange + final String DIFFERENT_KEY = "0" * 64 + def args = ["-k ${DIFFERENT_KEY}"] as String[] + + // Act + def parsedArgs = NiFi.parseArgs(args) + + // Assert + assert parsedArgs.size() == 2 + assert parsedArgs == args.join(" ").split(" ") as List + } + + @Test + public void testMainShouldHandleBadArgs() throws Exception { + // Arrange + def args = ["-k"] as String[] + + System.setProperty(NiFiProperties.PROPERTIES_FILE_PATH, "${TEST_RES_PATH}/NiFiProperties/conf/nifi_with_sensitive_properties_protected_aes.properties") + + // Act + NiFi.main(args) + + // Assert + assert TestAppender.events.collect { + it.getFormattedMessage() + }.contains("The bootstrap process passed the -k flag without a key") + assert TestAppender.events.last().getMessage() == "Failure to launch NiFi due to java.lang.IllegalArgumentException: The bootstrap process did not provide a valid key and there are protected properties present in the properties file" + } + + @Test + public void testMainShouldHandleMalformedBootstrapKey() throws Exception { + // Arrange + def args = ["-k", "BAD KEY"] as String[] + + System.setProperty(NiFiProperties.PROPERTIES_FILE_PATH, "${TEST_RES_PATH}/NiFiProperties/conf/nifi_with_sensitive_properties_protected_aes.properties") + + // Act + NiFi.main(args) + + // Assert + assert TestAppender.events.last().getMessage() == "Failure to launch NiFi due to java.lang.IllegalArgumentException: The bootstrap process did not provide a valid key and there are protected properties present in the properties file" + } + + @Test + public void testInitializePropertiesShouldSetBootstrapKeyFromArgs() throws Exception { + // Arrange + final String DIFFERENT_KEY = "0" * 64 + def args = ["-k", DIFFERENT_KEY] as String[] + + String testPropertiesPath = "${TEST_RES_PATH}/NiFiProperties/conf/nifi_with_sensitive_properties_protected_aes_different_key.properties" + System.setProperty(NiFiProperties.PROPERTIES_FILE_PATH, testPropertiesPath) + + NiFiProperties unprocessedProperties = new NiFiPropertiesLoader().loadRaw(new File(testPropertiesPath)) + def protectedKeys = getProtectedKeys(unprocessedProperties) + logger.info("Reading from raw properties file gives protected properties: ${protectedKeys}") + + // Act + NiFiProperties properties = NiFi.initializeProperties(args) + + // Assert + + // Ensure that there were protected properties, they were encrypted using AES/GCM (128/256 bit key), and they were decrypted (raw value != retrieved value) + assert !hasProtectedKeys(properties) + def unprotectedProperties = decrypt(unprocessedProperties, DIFFERENT_KEY) + getProtectedPropertyKeys(unprocessedProperties).every { k, v -> + String rawValue = unprocessedProperties.getProperty(k) + logger.raw("${k} -> ${rawValue}") + String retrievedValue = properties.getProperty(k) + logger.decrypted("${k} -> ${retrievedValue}") + + assert v =~ "aes/gcm" + + logger.assert("${retrievedValue} != ${rawValue}") + assert retrievedValue != rawValue + + String decryptedProperty = unprotectedProperties.getProperty(k) + logger.assert("${retrievedValue} == ${decryptedProperty}") + assert retrievedValue == decryptedProperty + } + } + + private static boolean hasProtectedKeys(NiFiProperties properties) { + properties.getPropertyKeys().any { it.endsWith(".protected") } + } + + private static Map getProtectedPropertyKeys(NiFiProperties properties) { + getProtectedKeys(properties).collectEntries { String key -> + [(key): properties.getProperty(key + ".protected")] + } + } + + private static Set getProtectedKeys(NiFiProperties properties) { + properties.getPropertyKeys().findAll { it.endsWith(".protected") } + } + + private static NiFiProperties decrypt(NiFiProperties encryptedProperties, String keyHex) { + AESSensitivePropertyProvider spp = new AESSensitivePropertyProvider(keyHex) + def map = encryptedProperties.getPropertyKeys().collectEntries { String key -> + if (encryptedProperties.getProperty(key + ".protected") == spp.getIdentifierKey()) { + [(key): spp.unprotect(encryptedProperties.getProperty(key))] + } else if (!key.endsWith(".protected")) { + [(key): encryptedProperties.getProperty(key)] + } + } + new StandardNiFiProperties(map as Properties) + } + + @Test + public void testShouldValidateKeys() { + // Arrange + final List VALID_KEYS = [ + "0" * 64, // 256 bit keys + "ABCDEF01" * 8, + "0123" * 8, // 128 bit keys + "0123456789ABCDEFFEDCBA9876543210", + "0123456789ABCDEFFEDCBA9876543210".toLowerCase(), + ] + + // Act + def isValid = VALID_KEYS.collectEntries { String key -> [(key): NiFi.isHexKeyValid(key)] } + logger.info("Key validity: ${isValid}") + + // Assert + assert isValid.every { k, v -> v } + } + + @Test + public void testShouldNotValidateInvalidKeys() { + // Arrange + final List VALID_KEYS = [ + "0" * 63, + "ABCDEFG1" * 8, + "0123" * 9, + "0123456789ABCDEFFEDCBA987654321", + "0123456789ABCDEF FEDCBA9876543210".toLowerCase(), + null, + "", + " " + ] + + // Act + def isValid = VALID_KEYS.collectEntries { String key -> [(key): NiFi.isHexKeyValid(key)] } + logger.info("Key validity: ${isValid}") + + // Assert + assert isValid.every { k, v -> !v } + } +} + +public class TestAppender extends AppenderBase { + static List events = new ArrayList<>(); + + @Override + protected void append(LoggingEvent e) { + synchronized (events) { + events.add(e); + } + } + + public static void reset() { + synchronized (events) { + events.clear(); + } + } +} \ No newline at end of file diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-runtime/src/test/resources/NiFiProperties/conf/nifi.properties b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-runtime/src/test/resources/NiFiProperties/conf/nifi.properties new file mode 100644 index 0000000000..f341c01b69 --- /dev/null +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-runtime/src/test/resources/NiFiProperties/conf/nifi.properties @@ -0,0 +1,188 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Core Properties # +nifi.version=1.0.0-SNAPSHOT +nifi.flow.configuration.file=./target/conf/flow.xml.gz +nifi.flow.configuration.archive.enabled=true +nifi.flow.configuration.archive.dir=./target/conf/archive/ +nifi.flow.configuration.archive.max.time=30 days +nifi.flow.configuration.archive.max.storage=500 MB +nifi.flowcontroller.autoResumeState=true +nifi.flowcontroller.graceful.shutdown.period=10 sec +nifi.flowservice.writedelay.interval=500 ms +nifi.administrative.yield.duration=30 sec +# If a component has no work to do (is "bored"), how long should we wait before checking again for work? +nifi.bored.yield.duration=10 millis + +nifi.authorizer.configuration.file=./target/conf/authorizers.xml +nifi.login.identity.provider.configuration.file=./target/conf/login-identity-providers.xml +nifi.templates.directory=./target/conf/templates +nifi.ui.banner.text= +nifi.ui.autorefresh.interval=30 sec +nifi.nar.library.directory=./target/lib +nifi.nar.working.directory=./target/work/nar/ +nifi.documentation.working.directory=./target/work/docs/components + +#################### +# State Management # +#################### +nifi.state.management.configuration.file=./target/conf/state-management.xml +# The ID of the local state provider +nifi.state.management.provider.local=local-provider +# The ID of the cluster-wide state provider. This will be ignored if NiFi is not clustered but must be populated if running in a cluster. +nifi.state.management.provider.cluster=zk-provider +# Specifies whether or not this instance of NiFi should run an embedded ZooKeeper server +nifi.state.management.embedded.zookeeper.start=false +# Properties file that provides the ZooKeeper properties to use if is set to true +nifi.state.management.embedded.zookeeper.properties=./target/conf/zookeeper.properties + + +# H2 Settings +nifi.database.directory=./target/database_repository +nifi.h2.url.append=;LOCK_TIMEOUT=25000;WRITE_DELAY=0;AUTO_SERVER=FALSE + +# FlowFile Repository +nifi.flowfile.repository.implementation=org.apache.nifi.controller.repository.WriteAheadFlowFileRepository +nifi.flowfile.repository.directory=./target/flowfile_repository +nifi.flowfile.repository.partitions=256 +nifi.flowfile.repository.checkpoint.interval=2 mins +nifi.flowfile.repository.always.sync=false + +nifi.swap.manager.implementation=org.apache.nifi.controller.FileSystemSwapManager +nifi.queue.swap.threshold=20000 +nifi.swap.in.period=5 sec +nifi.swap.in.threads=1 +nifi.swap.out.period=5 sec +nifi.swap.out.threads=4 + +# Content Repository +nifi.content.repository.implementation=org.apache.nifi.controller.repository.FileSystemRepository +nifi.content.claim.max.appendable.size=10 MB +nifi.content.claim.max.flow.files=100 +nifi.content.repository.directory.default=./target/content_repository +nifi.content.repository.archive.max.retention.period=12 hours +nifi.content.repository.archive.max.usage.percentage=50% +nifi.content.repository.archive.enabled=true +nifi.content.repository.always.sync=false +nifi.content.viewer.url=/nifi-content-viewer/ + +# Provenance Repository Properties +nifi.provenance.repository.implementation=org.apache.nifi.provenance.PersistentProvenanceRepository + +# Persistent Provenance Repository Properties +nifi.provenance.repository.directory.default=./target/provenance_repository +nifi.provenance.repository.max.storage.time=24 hours +nifi.provenance.repository.max.storage.size=1 GB +nifi.provenance.repository.rollover.time=30 secs +nifi.provenance.repository.rollover.size=100 MB +nifi.provenance.repository.query.threads=2 +nifi.provenance.repository.index.threads=1 +nifi.provenance.repository.compress.on.rollover=true +nifi.provenance.repository.always.sync=false +nifi.provenance.repository.journal.count=16 +# Comma-separated list of fields. Fields that are not indexed will not be searchable. Valid fields are: +# EventType, FlowFileUUID, Filename, TransitURI, ProcessorID, AlternateIdentifierURI, Relationship, Details +nifi.provenance.repository.indexed.fields=EventType, FlowFileUUID, Filename, ProcessorID, Relationship +# FlowFile Attributes that should be indexed and made searchable. Some examples to consider are filename, uuid, mime.type +nifi.provenance.repository.indexed.attributes= +# Large values for the shard size will result in more Java heap usage when searching the Provenance Repository +# but should provide better performance +nifi.provenance.repository.index.shard.size=500 MB +# Indicates the maximum length that a FlowFile attribute can be when retrieving a Provenance Event from +# the repository. If the length of any attribute exceeds this value, it will be truncated when the event is retrieved. +nifi.provenance.repository.max.attribute.length=65536 + +# Volatile Provenance Respository Properties +nifi.provenance.repository.buffer.size=100000 + +# Component Status Repository +nifi.components.status.repository.implementation=org.apache.nifi.controller.status.history.VolatileComponentStatusRepository +nifi.components.status.repository.buffer.size=1440 +nifi.components.status.snapshot.frequency=1 min + +# Site to Site properties +nifi.remote.input.host= +nifi.remote.input.secure=false +nifi.remote.input.socket.port= +nifi.remote.input.http.enabled=true +nifi.remote.input.http.transaction.ttl=30 sec + +# web properties # +nifi.web.war.directory=./target/lib +nifi.web.http.host= +nifi.web.http.port=8080 +nifi.web.https.host= +nifi.web.https.port= +nifi.web.jetty.working.directory=./target/work/jetty +nifi.web.jetty.threads=200 + +# security properties # +nifi.sensitive.props.key= +nifi.sensitive.props.algorithm=PBEWITHMD5AND256BITAES-CBC-OPENSSL +nifi.sensitive.props.provider=BC +nifi.sensitive.props.additional.keys= + +nifi.security.keystore= +nifi.security.keystoreType= +nifi.security.keystorePasswd= +nifi.security.keyPasswd= +nifi.security.truststore= +nifi.security.truststoreType= +nifi.security.truststorePasswd= +nifi.security.needClientAuth= +nifi.security.user.authorizer=file-provider +nifi.security.user.login.identity.provider= +nifi.security.ocsp.responder.url= +nifi.security.ocsp.responder.certificate= + +# Identity Mapping Properties # +# These properties allow normalizing user identities such that identities coming from different identity providers +# (certificates, LDAP, Kerberos) can be treated the same internally in NiFi. The following example demonstrates normalizing +# DNs from certificates and principals from Kerberos into a common identity string: +# +# nifi.security.identity.mapping.pattern.dn=^CN=(.*?), OU=(.*?), O=(.*?), L=(.*?), ST=(.*?), C=(.*?)$ +# nifi.security.identity.mapping.value.dn=$1@$2 +# nifi.security.identity.mapping.pattern.kerb=^(.*?)/instance@(.*?)$ +# nifi.security.identity.mapping.value.kerb=$1@$2 + +# cluster common properties (all nodes must have same values) # +nifi.cluster.protocol.heartbeat.interval=5 sec +nifi.cluster.protocol.is.secure=false + +# cluster node properties (only configure for cluster nodes) # +nifi.cluster.is.node=false +nifi.cluster.node.address= +nifi.cluster.node.protocol.port= +nifi.cluster.node.protocol.threads=10 +nifi.cluster.node.event.history.size=25 +nifi.cluster.node.connection.timeout=5 sec +nifi.cluster.node.read.timeout=5 sec +nifi.cluster.firewall.file= + +# How long a request should be allowed to hold a 'lock' on a component. # +nifi.cluster.request.replication.claim.timeout=15 secs + +# zookeeper properties, used for cluster management # +nifi.zookeeper.connect.string= +nifi.zookeeper.connect.timeout=3 secs +nifi.zookeeper.session.timeout=3 secs +nifi.zookeeper.root.node=/nifi + +# kerberos # +nifi.kerberos.krb5.file= +nifi.kerberos.service.principal= +nifi.kerberos.keytab.location= +nifi.kerberos.authentication.expiration=12 hours \ No newline at end of file diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-runtime/src/test/resources/NiFiProperties/conf/nifi_with_sensitive_properties_protected_aes.properties b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-runtime/src/test/resources/NiFiProperties/conf/nifi_with_sensitive_properties_protected_aes.properties new file mode 100644 index 0000000000..cbc93828c1 --- /dev/null +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-runtime/src/test/resources/NiFiProperties/conf/nifi_with_sensitive_properties_protected_aes.properties @@ -0,0 +1,189 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Core Properties # +nifi.version=1.0.0-SNAPSHOT +nifi.flow.configuration.file=./target/conf/flow.xml.gz +nifi.flow.configuration.archive.enabled=true +nifi.flow.configuration.archive.dir=./target/conf/archive/ +nifi.flow.configuration.archive.max.time=30 days +nifi.flow.configuration.archive.max.storage=500 MB +nifi.flowcontroller.autoResumeState=true +nifi.flowcontroller.graceful.shutdown.period=10 sec +nifi.flowservice.writedelay.interval=500 ms +nifi.administrative.yield.duration=30 sec +# If a component has no work to do (is "bored"), how long should we wait before checking again for work? +nifi.bored.yield.duration=10 millis + +nifi.authorizer.configuration.file=./target/conf/authorizers.xml +nifi.login.identity.provider.configuration.file=./target/conf/login-identity-providers.xml +nifi.templates.directory=./target/conf/templates +nifi.ui.banner.text=n8hL1zgQcYpG70Vm||e1hYsrc7FLvi1E9LcHM1VYeN5atWJIGg/WCsyuxxNqN1lK1ASGEZR8040NFZNqwsnbx+ +nifi.ui.banner.text.protected=aes/gcm/256 +nifi.ui.autorefresh.interval=30 sec +nifi.nar.library.directory=./target/lib +nifi.nar.working.directory=./target/work/nar/ +nifi.documentation.working.directory=./target/work/docs/components + +#################### +# State Management # +#################### +nifi.state.management.configuration.file=./target/conf/state-management.xml +# The ID of the local state provider +nifi.state.management.provider.local=local-provider +# The ID of the cluster-wide state provider. This will be ignored if NiFi is not clustered but must be populated if running in a cluster. +nifi.state.management.provider.cluster=zk-provider +# Specifies whether or not this instance of NiFi should run an embedded ZooKeeper server +nifi.state.management.embedded.zookeeper.start=false +# Properties file that provides the ZooKeeper properties to use if is set to true +nifi.state.management.embedded.zookeeper.properties=./target/conf/zookeeper.properties + + +# H2 Settings +nifi.database.directory=./target/database_repository +nifi.h2.url.append=;LOCK_TIMEOUT=25000;WRITE_DELAY=0;AUTO_SERVER=FALSE + +# FlowFile Repository +nifi.flowfile.repository.implementation=org.apache.nifi.controller.repository.WriteAheadFlowFileRepository +nifi.flowfile.repository.directory=./target/flowfile_repository +nifi.flowfile.repository.partitions=256 +nifi.flowfile.repository.checkpoint.interval=2 mins +nifi.flowfile.repository.always.sync=false + +nifi.swap.manager.implementation=org.apache.nifi.controller.FileSystemSwapManager +nifi.queue.swap.threshold=20000 +nifi.swap.in.period=5 sec +nifi.swap.in.threads=1 +nifi.swap.out.period=5 sec +nifi.swap.out.threads=4 + +# Content Repository +nifi.content.repository.implementation=org.apache.nifi.controller.repository.FileSystemRepository +nifi.content.claim.max.appendable.size=10 MB +nifi.content.claim.max.flow.files=100 +nifi.content.repository.directory.default=./target/content_repository +nifi.content.repository.archive.max.retention.period=12 hours +nifi.content.repository.archive.max.usage.percentage=50% +nifi.content.repository.archive.enabled=true +nifi.content.repository.always.sync=false +nifi.content.viewer.url=/nifi-content-viewer/ + +# Provenance Repository Properties +nifi.provenance.repository.implementation=org.apache.nifi.provenance.PersistentProvenanceRepository + +# Persistent Provenance Repository Properties +nifi.provenance.repository.directory.default=./target/provenance_repository +nifi.provenance.repository.max.storage.time=24 hours +nifi.provenance.repository.max.storage.size=1 GB +nifi.provenance.repository.rollover.time=30 secs +nifi.provenance.repository.rollover.size=100 MB +nifi.provenance.repository.query.threads=2 +nifi.provenance.repository.index.threads=1 +nifi.provenance.repository.compress.on.rollover=true +nifi.provenance.repository.always.sync=false +nifi.provenance.repository.journal.count=16 +# Comma-separated list of fields. Fields that are not indexed will not be searchable. Valid fields are: +# EventType, FlowFileUUID, Filename, TransitURI, ProcessorID, AlternateIdentifierURI, Relationship, Details +nifi.provenance.repository.indexed.fields=EventType, FlowFileUUID, Filename, ProcessorID, Relationship +# FlowFile Attributes that should be indexed and made searchable. Some examples to consider are filename, uuid, mime.type +nifi.provenance.repository.indexed.attributes= +# Large values for the shard size will result in more Java heap usage when searching the Provenance Repository +# but should provide better performance +nifi.provenance.repository.index.shard.size=500 MB +# Indicates the maximum length that a FlowFile attribute can be when retrieving a Provenance Event from +# the repository. If the length of any attribute exceeds this value, it will be truncated when the event is retrieved. +nifi.provenance.repository.max.attribute.length=65536 + +# Volatile Provenance Respository Properties +nifi.provenance.repository.buffer.size=100000 + +# Component Status Repository +nifi.components.status.repository.implementation=org.apache.nifi.controller.status.history.VolatileComponentStatusRepository +nifi.components.status.repository.buffer.size=1440 +nifi.components.status.snapshot.frequency=1 min + +# Site to Site properties +nifi.remote.input.host= +nifi.remote.input.secure=false +nifi.remote.input.socket.port= +nifi.remote.input.http.enabled=true +nifi.remote.input.http.transaction.ttl=30 sec + +# web properties # +nifi.web.war.directory=./target/lib +nifi.web.http.host= +nifi.web.http.port=8080 +nifi.web.https.host= +nifi.web.https.port= +nifi.web.jetty.working.directory=./target/work/jetty +nifi.web.jetty.threads=200 + +# security properties # +nifi.sensitive.props.key= +nifi.sensitive.props.algorithm=PBEWITHMD5AND256BITAES-CBC-OPENSSL +nifi.sensitive.props.provider=BC +nifi.sensitive.props.additional.keys=nifi.ui.banner.text + +nifi.security.keystore= +nifi.security.keystoreType= +nifi.security.keystorePasswd= +nifi.security.keyPasswd= +nifi.security.truststore= +nifi.security.truststoreType= +nifi.security.truststorePasswd= +nifi.security.needClientAuth= +nifi.security.user.authorizer=file-provider +nifi.security.user.login.identity.provider= +nifi.security.ocsp.responder.url= +nifi.security.ocsp.responder.certificate= + +# Identity Mapping Properties # +# These properties allow normalizing user identities such that identities coming from different identity providers +# (certificates, LDAP, Kerberos) can be treated the same internally in NiFi. The following example demonstrates normalizing +# DNs from certificates and principals from Kerberos into a common identity string: +# +# nifi.security.identity.mapping.pattern.dn=^CN=(.*?), OU=(.*?), O=(.*?), L=(.*?), ST=(.*?), C=(.*?)$ +# nifi.security.identity.mapping.value.dn=$1@$2 +# nifi.security.identity.mapping.pattern.kerb=^(.*?)/instance@(.*?)$ +# nifi.security.identity.mapping.value.kerb=$1@$2 + +# cluster common properties (all nodes must have same values) # +nifi.cluster.protocol.heartbeat.interval=5 sec +nifi.cluster.protocol.is.secure=false + +# cluster node properties (only configure for cluster nodes) # +nifi.cluster.is.node=false +nifi.cluster.node.address= +nifi.cluster.node.protocol.port= +nifi.cluster.node.protocol.threads=10 +nifi.cluster.node.event.history.size=25 +nifi.cluster.node.connection.timeout=5 sec +nifi.cluster.node.read.timeout=5 sec +nifi.cluster.firewall.file= + +# How long a request should be allowed to hold a 'lock' on a component. # +nifi.cluster.request.replication.claim.timeout=15 secs + +# zookeeper properties, used for cluster management # +nifi.zookeeper.connect.string= +nifi.zookeeper.connect.timeout=3 secs +nifi.zookeeper.session.timeout=3 secs +nifi.zookeeper.root.node=/nifi + +# kerberos # +nifi.kerberos.krb5.file= +nifi.kerberos.service.principal= +nifi.kerberos.keytab.location= +nifi.kerberos.authentication.expiration=12 hours \ No newline at end of file diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-runtime/src/test/resources/NiFiProperties/conf/nifi_with_sensitive_properties_protected_aes_different_key.properties b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-runtime/src/test/resources/NiFiProperties/conf/nifi_with_sensitive_properties_protected_aes_different_key.properties new file mode 100644 index 0000000000..f170d7e25d --- /dev/null +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-runtime/src/test/resources/NiFiProperties/conf/nifi_with_sensitive_properties_protected_aes_different_key.properties @@ -0,0 +1,192 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Core Properties # +nifi.version=1.0.0-SNAPSHOT +nifi.flow.configuration.file=./target/conf/flow.xml.gz +nifi.flow.configuration.archive.enabled=true +nifi.flow.configuration.archive.dir=./target/conf/archive/ +nifi.flow.configuration.archive.max.time=30 days +nifi.flow.configuration.archive.max.storage=500 MB +nifi.flowcontroller.autoResumeState=true +nifi.flowcontroller.graceful.shutdown.period=10 sec +nifi.flowservice.writedelay.interval=500 ms +nifi.administrative.yield.duration=30 sec +# If a component has no work to do (is "bored"), how long should we wait before checking again for work? +nifi.bored.yield.duration=10 millis + +nifi.authorizer.configuration.file=./target/conf/authorizers.xml +nifi.login.identity.provider.configuration.file=./target/conf/login-identity-providers.xml +nifi.templates.directory=./target/conf/templates +nifi.ui.banner.text=dXwnu9mLyPETJrq1||n9e5dk5+HSTBCGOA/Sy6VYzwPw3baeRNvglalA1Pr1PcToyc4/qT6md24YOP4xVz14jd +nifi.ui.banner.text.protected=aes/gcm/256 +nifi.ui.autorefresh.interval=30 sec +nifi.nar.library.directory=./target/lib +nifi.nar.working.directory=./target/work/nar/ +nifi.documentation.working.directory=./target/work/docs/components + +#################### +# State Management # +#################### +nifi.state.management.configuration.file=./target/conf/state-management.xml +# The ID of the local state provider +nifi.state.management.provider.local=local-provider +# The ID of the cluster-wide state provider. This will be ignored if NiFi is not clustered but must be populated if running in a cluster. +nifi.state.management.provider.cluster=zk-provider +# Specifies whether or not this instance of NiFi should run an embedded ZooKeeper server +nifi.state.management.embedded.zookeeper.start=false +# Properties file that provides the ZooKeeper properties to use if is set to true +nifi.state.management.embedded.zookeeper.properties=./target/conf/zookeeper.properties + + +# H2 Settings +nifi.database.directory=./target/database_repository +nifi.h2.url.append=;LOCK_TIMEOUT=25000;WRITE_DELAY=0;AUTO_SERVER=FALSE + +# FlowFile Repository +nifi.flowfile.repository.implementation=org.apache.nifi.controller.repository.WriteAheadFlowFileRepository +nifi.flowfile.repository.directory=./target/flowfile_repository +nifi.flowfile.repository.partitions=256 +nifi.flowfile.repository.checkpoint.interval=2 mins +nifi.flowfile.repository.always.sync=false + +nifi.swap.manager.implementation=org.apache.nifi.controller.FileSystemSwapManager +nifi.queue.swap.threshold=20000 +nifi.swap.in.period=5 sec +nifi.swap.in.threads=1 +nifi.swap.out.period=5 sec +nifi.swap.out.threads=4 + +# Content Repository +nifi.content.repository.implementation=org.apache.nifi.controller.repository.FileSystemRepository +nifi.content.claim.max.appendable.size=10 MB +nifi.content.claim.max.flow.files=100 +nifi.content.repository.directory.default=./target/content_repository +nifi.content.repository.archive.max.retention.period=12 hours +nifi.content.repository.archive.max.usage.percentage=50% +nifi.content.repository.archive.enabled=true +nifi.content.repository.always.sync=false +nifi.content.viewer.url=/nifi-content-viewer/ + +# Provenance Repository Properties +nifi.provenance.repository.implementation=org.apache.nifi.provenance.PersistentProvenanceRepository + +# Persistent Provenance Repository Properties +nifi.provenance.repository.directory.default=./target/provenance_repository +nifi.provenance.repository.max.storage.time=24 hours +nifi.provenance.repository.max.storage.size=1 GB +nifi.provenance.repository.rollover.time=30 secs +nifi.provenance.repository.rollover.size=100 MB +nifi.provenance.repository.query.threads=2 +nifi.provenance.repository.index.threads=1 +nifi.provenance.repository.compress.on.rollover=true +nifi.provenance.repository.always.sync=false +nifi.provenance.repository.journal.count=16 +# Comma-separated list of fields. Fields that are not indexed will not be searchable. Valid fields are: +# EventType, FlowFileUUID, Filename, TransitURI, ProcessorID, AlternateIdentifierURI, Relationship, Details +nifi.provenance.repository.indexed.fields=EventType, FlowFileUUID, Filename, ProcessorID, Relationship +# FlowFile Attributes that should be indexed and made searchable. Some examples to consider are filename, uuid, mime.type +nifi.provenance.repository.indexed.attributes= +# Large values for the shard size will result in more Java heap usage when searching the Provenance Repository +# but should provide better performance +nifi.provenance.repository.index.shard.size=500 MB +# Indicates the maximum length that a FlowFile attribute can be when retrieving a Provenance Event from +# the repository. If the length of any attribute exceeds this value, it will be truncated when the event is retrieved. +nifi.provenance.repository.max.attribute.length=65536 + +# Volatile Provenance Respository Properties +nifi.provenance.repository.buffer.size=100000 + +# Component Status Repository +nifi.components.status.repository.implementation=org.apache.nifi.controller.status.history.VolatileComponentStatusRepository +nifi.components.status.repository.buffer.size=1440 +nifi.components.status.snapshot.frequency=1 min + +# Site to Site properties +nifi.remote.input.host= +nifi.remote.input.secure=false +nifi.remote.input.socket.port= +nifi.remote.input.http.enabled=true +nifi.remote.input.http.transaction.ttl=30 sec + +# web properties # +nifi.web.war.directory=./target/lib +nifi.web.http.host= +nifi.web.http.port=8080 +nifi.web.https.host= +nifi.web.https.port= +nifi.web.jetty.working.directory=./target/work/jetty +nifi.web.jetty.threads=200 + +# security properties # +nifi.sensitive.props.key=dQU402Mz4J+t+e18||6+ictR0Nssq3/rR/d8fq5CFAKmpakr9jCyPIJYxG7n6D86gxsu2TRp4M48ugUw== +nifi.sensitive.props.key.protected=aes/gcm/256 +nifi.sensitive.props.algorithm=PBEWITHMD5AND256BITAES-CBC-OPENSSL +nifi.sensitive.props.provider=BC +nifi.sensitive.props.additional.keys= + +nifi.security.keystore=/path/to/keystore.jks +nifi.security.keystoreType=JKS +nifi.security.keystorePasswd=Q8T3wv+Xl2ie98GV||qsuY9wa/Rt27cqFXs8ebX25E1iSbFAEFcD0cjCwrl3Tw6HghQjBIaCzQ +nifi.security.keystorePasswd.protected=aes/gcm/256 +nifi.security.keyPasswd=1S0XmoiAr379B8rg||PPZzjdw9BAJSon9g4xm9uscFhCCyk734FTjXtRnBXUy819zsoQ== +nifi.security.keyPasswd.protected=aes/gcm/256 +nifi.security.truststore= +nifi.security.truststoreType= +nifi.security.truststorePasswd= +nifi.security.needClientAuth= +nifi.security.user.authorizer=file-provider +nifi.security.user.login.identity.provider= +nifi.security.ocsp.responder.url= +nifi.security.ocsp.responder.certificate= + +# Identity Mapping Properties # +# These properties allow normalizing user identities such that identities coming from different identity providers +# (certificates, LDAP, Kerberos) can be treated the same internally in NiFi. The following example demonstrates normalizing +# DNs from certificates and principals from Kerberos into a common identity string: +# +# nifi.security.identity.mapping.pattern.dn=^CN=(.*?), OU=(.*?), O=(.*?), L=(.*?), ST=(.*?), C=(.*?)$ +# nifi.security.identity.mapping.value.dn=$1@$2 +# nifi.security.identity.mapping.pattern.kerb=^(.*?)/instance@(.*?)$ +# nifi.security.identity.mapping.value.kerb=$1@$2 + +# cluster common properties (all nodes must have same values) # +nifi.cluster.protocol.heartbeat.interval=5 sec +nifi.cluster.protocol.is.secure=false + +# cluster node properties (only configure for cluster nodes) # +nifi.cluster.is.node=false +nifi.cluster.node.address= +nifi.cluster.node.protocol.port= +nifi.cluster.node.protocol.threads=10 +nifi.cluster.node.event.history.size=25 +nifi.cluster.node.connection.timeout=5 sec +nifi.cluster.node.read.timeout=5 sec +nifi.cluster.firewall.file= + +# How long a request should be allowed to hold a 'lock' on a component. # +nifi.cluster.request.replication.claim.timeout=15 secs + +# zookeeper properties, used for cluster management # +nifi.zookeeper.connect.string= +nifi.zookeeper.connect.timeout=3 secs +nifi.zookeeper.session.timeout=3 secs +nifi.zookeeper.root.node=/nifi + +# kerberos # +nifi.kerberos.krb5.file= +nifi.kerberos.service.principal= +nifi.kerberos.keytab.location= +nifi.kerberos.authentication.expiration=12 hours \ No newline at end of file diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-runtime/src/test/resources/logback-test.xml b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-runtime/src/test/resources/logback-test.xml new file mode 100644 index 0000000000..7f0a03ba1e --- /dev/null +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-runtime/src/test/resources/logback-test.xml @@ -0,0 +1,34 @@ + + + + + + + %-4r [%t] %-5p %c - %m%n + + + + + %-4r [%t] %-5p %c - %m%n + + + + + + + + + diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-site-to-site/src/main/java/org/apache/nifi/remote/protocol/http/StandardHttpFlowFileServerProtocol.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-site-to-site/src/main/java/org/apache/nifi/remote/protocol/http/StandardHttpFlowFileServerProtocol.java index e71ac1d952..0795b5d0a3 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-site-to-site/src/main/java/org/apache/nifi/remote/protocol/http/StandardHttpFlowFileServerProtocol.java +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-site-to-site/src/main/java/org/apache/nifi/remote/protocol/http/StandardHttpFlowFileServerProtocol.java @@ -93,7 +93,7 @@ public class StandardHttpFlowFileServerProtocol extends AbstractFlowFileServerPr HttpServerCommunicationsSession commSession = (HttpServerCommunicationsSession) commsSession; commSession.setResponseCode(response); - if (isTransfer) { + if(isTransfer){ switch (response) { case NO_MORE_DATA: logger.debug("{} There's no data to send.", this); @@ -138,8 +138,8 @@ public class StandardHttpFlowFileServerProtocol extends AbstractFlowFileServerPr ByteArrayOutputStream bos = new ByteArrayOutputStream(); Transaction.TransactionState currentStatus = commSession.getStatus(); - if (isTransfer) { - switch (currentStatus) { + if(isTransfer){ + switch (currentStatus){ case DATA_EXCHANGED: String clientChecksum = commSession.getChecksum(); logger.debug("readTransactionResponse. clientChecksum={}", clientChecksum); @@ -151,7 +151,7 @@ public class StandardHttpFlowFileServerProtocol extends AbstractFlowFileServerPr break; } } else { - switch (currentStatus) { + switch (currentStatus){ case TRANSACTION_STARTED: logger.debug("readTransactionResponse. returning CONTINUE_TRANSACTION."); // We don't know if there's more data to receive, so just continue it. @@ -161,7 +161,7 @@ public class StandardHttpFlowFileServerProtocol extends AbstractFlowFileServerPr // Checksum was successfully validated at client side, or BAD_CHECKSUM is returned. ResponseCode responseCode = commSession.getResponseCode(); logger.debug("readTransactionResponse. responseCode={}", responseCode); - if (responseCode.containsMessage()) { + if(responseCode.containsMessage()){ responseCode.writeResponse(new DataOutputStream(bos), ""); } else { responseCode.writeResponse(new DataOutputStream(bos)); @@ -228,8 +228,8 @@ public class StandardHttpFlowFileServerProtocol extends AbstractFlowFileServerPr } @Override - public void sendPeerList(Peer peer, Optional clusterNodeInfo, String remoteInputHost, - int remoteInputPort, int remoteInputHttpPort, boolean isSiteToSiteSecure) throws IOException { + public void sendPeerList(Peer peer, Optional clusterNodeInfo, String remoteInputHost, int remoteInputPort, int remoteInputHttpPort, + boolean isSiteToSiteSecure) throws IOException { } @Override diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-site-to-site/src/main/java/org/apache/nifi/remote/protocol/socket/SocketFlowFileServerProtocol.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-site-to-site/src/main/java/org/apache/nifi/remote/protocol/socket/SocketFlowFileServerProtocol.java index e965cf47e8..8ecf96c83f 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-site-to-site/src/main/java/org/apache/nifi/remote/protocol/socket/SocketFlowFileServerProtocol.java +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-site-to-site/src/main/java/org/apache/nifi/remote/protocol/socket/SocketFlowFileServerProtocol.java @@ -16,6 +16,16 @@ */ package org.apache.nifi.remote.protocol.socket; +import java.io.DataInputStream; +import java.io.DataOutputStream; +import java.io.IOException; +import java.net.InetAddress; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; import org.apache.nifi.remote.Peer; import org.apache.nifi.remote.RemoteResourceFactory; import org.apache.nifi.remote.StandardVersionNegotiator; @@ -31,17 +41,6 @@ import org.apache.nifi.remote.protocol.HandshakeProperties; import org.apache.nifi.remote.protocol.RequestType; import org.apache.nifi.remote.protocol.ResponseCode; -import java.io.DataInputStream; -import java.io.DataOutputStream; -import java.io.IOException; -import java.net.InetAddress; -import java.util.ArrayList; -import java.util.Collections; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Optional; - public class SocketFlowFileServerProtocol extends AbstractFlowFileServerProtocol { public static final String RESOURCE_NAME = "SocketFlowFileProtocol"; @@ -83,7 +82,7 @@ public class SocketFlowFileServerProtocol extends AbstractFlowFileServerProtocol validateHandshakeRequest(confirmed, peer, properties); } catch (HandshakeException e) { ResponseCode handshakeResult = e.getResponseCode(); - if (handshakeResult.containsMessage()) { + if(handshakeResult.containsMessage()){ handshakeResult.writeResponse(dos, e.getMessage()); } else { handshakeResult.writeResponse(dos); @@ -135,6 +134,7 @@ public class SocketFlowFileServerProtocol extends AbstractFlowFileServerProtocol } } + @Override public RequestType getRequestType(final Peer peer) throws IOException { if (!handshakeCompleted) { @@ -181,7 +181,7 @@ public class SocketFlowFileServerProtocol extends AbstractFlowFileServerProtocol nodeInfos = new ArrayList<>(clusterNodeInfo.get().getNodeInformation()); } else { final NodeInformation self = new NodeInformation(remoteInputHostVal, remoteInputPort, remoteInputHttpPort, remoteInputHttpPort, - isSiteToSiteSecure, 0); + isSiteToSiteSecure, 0); nodeInfos = Collections.singletonList(self); } @@ -214,6 +214,7 @@ public class SocketFlowFileServerProtocol extends AbstractFlowFileServerProtocol return RESOURCE_NAME; } + @Override public VersionNegotiator getVersionNegotiator() { return versionNegotiator; diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/api/DataTransferResource.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/api/DataTransferResource.java index 886f24ae20..7e0aff9457 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/api/DataTransferResource.java +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/api/DataTransferResource.java @@ -16,12 +16,47 @@ */ package org.apache.nifi.web.api; +import static org.apache.commons.lang3.StringUtils.isEmpty; +import static org.apache.nifi.remote.protocol.HandshakeProperty.BATCH_COUNT; +import static org.apache.nifi.remote.protocol.HandshakeProperty.BATCH_DURATION; +import static org.apache.nifi.remote.protocol.HandshakeProperty.BATCH_SIZE; +import static org.apache.nifi.remote.protocol.HandshakeProperty.REQUEST_EXPIRATION_MILLIS; +import static org.apache.nifi.remote.protocol.http.HttpHeaders.HANDSHAKE_PROPERTY_BATCH_COUNT; +import static org.apache.nifi.remote.protocol.http.HttpHeaders.HANDSHAKE_PROPERTY_BATCH_DURATION; +import static org.apache.nifi.remote.protocol.http.HttpHeaders.HANDSHAKE_PROPERTY_BATCH_SIZE; +import static org.apache.nifi.remote.protocol.http.HttpHeaders.HANDSHAKE_PROPERTY_REQUEST_EXPIRATION; +import static org.apache.nifi.remote.protocol.http.HttpHeaders.HANDSHAKE_PROPERTY_USE_COMPRESSION; + import com.wordnik.swagger.annotations.Api; import com.wordnik.swagger.annotations.ApiOperation; import com.wordnik.swagger.annotations.ApiParam; import com.wordnik.swagger.annotations.ApiResponse; import com.wordnik.swagger.annotations.ApiResponses; import com.wordnik.swagger.annotations.Authorization; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.util.HashMap; +import java.util.Map; +import javax.servlet.ServletContext; +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; +import javax.ws.rs.Consumes; +import javax.ws.rs.DELETE; +import javax.ws.rs.DefaultValue; +import javax.ws.rs.GET; +import javax.ws.rs.POST; +import javax.ws.rs.PUT; +import javax.ws.rs.Path; +import javax.ws.rs.PathParam; +import javax.ws.rs.Produces; +import javax.ws.rs.QueryParam; +import javax.ws.rs.WebApplicationException; +import javax.ws.rs.core.Context; +import javax.ws.rs.core.MediaType; +import javax.ws.rs.core.Response; +import javax.ws.rs.core.StreamingOutput; +import javax.ws.rs.core.UriInfo; import org.apache.commons.lang3.StringUtils; import org.apache.nifi.authorization.AccessDeniedException; import org.apache.nifi.authorization.AuthorizationRequest; @@ -53,47 +88,11 @@ import org.apache.nifi.remote.protocol.ResponseCode; import org.apache.nifi.remote.protocol.http.HttpFlowFileServerProtocol; import org.apache.nifi.remote.protocol.http.StandardHttpFlowFileServerProtocol; import org.apache.nifi.stream.io.ByteArrayOutputStream; +import org.apache.nifi.util.NiFiProperties; import org.apache.nifi.web.api.entity.TransactionResultEntity; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import javax.servlet.ServletContext; -import javax.servlet.http.HttpServletRequest; -import javax.servlet.http.HttpServletResponse; -import javax.ws.rs.Consumes; -import javax.ws.rs.DELETE; -import javax.ws.rs.DefaultValue; -import javax.ws.rs.GET; -import javax.ws.rs.POST; -import javax.ws.rs.PUT; -import javax.ws.rs.Path; -import javax.ws.rs.PathParam; -import javax.ws.rs.Produces; -import javax.ws.rs.QueryParam; -import javax.ws.rs.WebApplicationException; -import javax.ws.rs.core.Context; -import javax.ws.rs.core.MediaType; -import javax.ws.rs.core.Response; -import javax.ws.rs.core.StreamingOutput; -import javax.ws.rs.core.UriInfo; -import java.io.IOException; -import java.io.InputStream; -import java.io.OutputStream; -import java.util.HashMap; -import java.util.Map; - -import static org.apache.commons.lang3.StringUtils.isEmpty; -import static org.apache.nifi.remote.protocol.HandshakeProperty.BATCH_COUNT; -import static org.apache.nifi.remote.protocol.HandshakeProperty.BATCH_DURATION; -import static org.apache.nifi.remote.protocol.HandshakeProperty.BATCH_SIZE; -import static org.apache.nifi.remote.protocol.HandshakeProperty.REQUEST_EXPIRATION_MILLIS; -import static org.apache.nifi.remote.protocol.http.HttpHeaders.HANDSHAKE_PROPERTY_BATCH_COUNT; -import static org.apache.nifi.remote.protocol.http.HttpHeaders.HANDSHAKE_PROPERTY_BATCH_DURATION; -import static org.apache.nifi.remote.protocol.http.HttpHeaders.HANDSHAKE_PROPERTY_BATCH_SIZE; -import static org.apache.nifi.remote.protocol.http.HttpHeaders.HANDSHAKE_PROPERTY_REQUEST_EXPIRATION; -import static org.apache.nifi.remote.protocol.http.HttpHeaders.HANDSHAKE_PROPERTY_USE_COMPRESSION; -import org.apache.nifi.util.NiFiProperties; - /** * RESTful endpoint for managing a SiteToSite connection. */ @@ -109,6 +108,7 @@ public class DataTransferResource extends ApplicationResource { public static final String CHECK_SUM = "checksum"; public static final String RESPONSE_CODE = "responseCode"; + private static final String PORT_TYPE_INPUT = "input-ports"; private static final String PORT_TYPE_OUTPUT = "output-ports"; @@ -116,8 +116,10 @@ public class DataTransferResource extends ApplicationResource { private final ResponseCreator responseCreator = new ResponseCreator(); private final VersionNegotiator transportProtocolVersionNegotiator = new TransportProtocolVersionNegotiator(1); private final HttpRemoteSiteListener transactionManager; + private final NiFiProperties nifiProperties; - public DataTransferResource(final NiFiProperties nifiProperties) { + public DataTransferResource(final NiFiProperties nifiProperties){ + this.nifiProperties = nifiProperties; transactionManager = HttpRemoteSiteListener.getInstance(nifiProperties); } @@ -165,17 +167,18 @@ public class DataTransferResource extends ApplicationResource { value = "Create a transaction to the specified output port or input port", response = TransactionResultEntity.class, authorizations = { - @Authorization(value = "Write - /data-transfer/{component-type}/{uuid}", type = "") + @Authorization(value = "Write - /data-transfer/{component-type}/{uuid}", type = "") } ) @ApiResponses( value = { - @ApiResponse(code = 400, message = "NiFi was unable to complete the request because it was invalid. The request should not be retried without modification."), - @ApiResponse(code = 401, message = "Client could not be authenticated."), - @ApiResponse(code = 403, message = "Client is not authorized to make this request."), - @ApiResponse(code = 404, message = "The specified resource could not be found."), - @ApiResponse(code = 409, message = "The request was valid but NiFi was not in the appropriate state to process it. Retrying the same request later may be successful."), - @ApiResponse(code = 503, message = "NiFi instance is not ready for serving request, or temporarily overloaded. Retrying the same request later may be successful"),} + @ApiResponse(code = 400, message = "NiFi was unable to complete the request because it was invalid. The request should not be retried without modification."), + @ApiResponse(code = 401, message = "Client could not be authenticated."), + @ApiResponse(code = 403, message = "Client is not authorized to make this request."), + @ApiResponse(code = 404, message = "The specified resource could not be found."), + @ApiResponse(code = 409, message = "The request was valid but NiFi was not in the appropriate state to process it. Retrying the same request later may be successful."), + @ApiResponse(code = 503, message = "NiFi instance is not ready for serving request, or temporarily overloaded. Retrying the same request later may be successful"), + } ) public Response createPortTransaction( @ApiParam( @@ -190,6 +193,7 @@ public class DataTransferResource extends ApplicationResource { @Context UriInfo uriInfo, InputStream inputStream) { + if (!PORT_TYPE_INPUT.equals(portType) && !PORT_TYPE_OUTPUT.equals(portType)) { return responseCreator.wrongPortTypeResponse(portType, portId); } @@ -237,17 +241,18 @@ public class DataTransferResource extends ApplicationResource { value = "Transfer flow files to the input port", response = String.class, authorizations = { - @Authorization(value = "Write - /data-transfer/input-ports/{uuid}", type = "") + @Authorization(value = "Write - /data-transfer/input-ports/{uuid}", type = "") } ) @ApiResponses( value = { - @ApiResponse(code = 400, message = "NiFi was unable to complete the request because it was invalid. The request should not be retried without modification."), - @ApiResponse(code = 401, message = "Client could not be authenticated."), - @ApiResponse(code = 403, message = "Client is not authorized to make this request."), - @ApiResponse(code = 404, message = "The specified resource could not be found."), - @ApiResponse(code = 409, message = "The request was valid but NiFi was not in the appropriate state to process it. Retrying the same request later may be successful."), - @ApiResponse(code = 503, message = "NiFi instance is not ready for serving request, or temporarily overloaded. Retrying the same request later may be successful"),} + @ApiResponse(code = 400, message = "NiFi was unable to complete the request because it was invalid. The request should not be retried without modification."), + @ApiResponse(code = 401, message = "Client could not be authenticated."), + @ApiResponse(code = 403, message = "Client is not authorized to make this request."), + @ApiResponse(code = 404, message = "The specified resource could not be found."), + @ApiResponse(code = 409, message = "The request was valid but NiFi was not in the appropriate state to process it. Retrying the same request later may be successful."), + @ApiResponse(code = 503, message = "NiFi instance is not ready for serving request, or temporarily overloaded. Retrying the same request later may be successful"), + } ) public Response receiveFlowFiles( @ApiParam( @@ -300,26 +305,26 @@ public class DataTransferResource extends ApplicationResource { } private HttpFlowFileServerProtocol initiateServerProtocol(final HttpServletRequest req, final Peer peer, - final Integer transportProtocolVersion) throws IOException { + final Integer transportProtocolVersion) throws IOException { // Switch transaction protocol version based on transport protocol version. TransportProtocolVersionNegotiator negotiatedTransportProtocolVersion = new TransportProtocolVersionNegotiator(transportProtocolVersion); VersionNegotiator versionNegotiator = new StandardVersionNegotiator(negotiatedTransportProtocolVersion.getTransactionProtocolVersion()); final String dataTransferUrl = req.getRequestURL().toString(); - ((HttpCommunicationsSession) peer.getCommunicationsSession()).setDataTransferUrl(dataTransferUrl); + ((HttpCommunicationsSession)peer.getCommunicationsSession()).setDataTransferUrl(dataTransferUrl); HttpFlowFileServerProtocol serverProtocol = getHttpFlowFileServerProtocol(versionNegotiator); - HttpRemoteSiteListener.getInstance(getProperties()).setupServerProtocol(serverProtocol); + HttpRemoteSiteListener.getInstance(nifiProperties).setupServerProtocol(serverProtocol); serverProtocol.handshake(peer); return serverProtocol; } HttpFlowFileServerProtocol getHttpFlowFileServerProtocol(final VersionNegotiator versionNegotiator) { - return new StandardHttpFlowFileServerProtocol(versionNegotiator, getProperties()); + return new StandardHttpFlowFileServerProtocol(versionNegotiator, nifiProperties); } private Peer constructPeer(final HttpServletRequest req, final InputStream inputStream, - final OutputStream outputStream, final String portId, final String transactionId) { + final OutputStream outputStream, final String portId, final String transactionId) { final String clientHostName = req.getRemoteHost(); final int clientPort = req.getRemotePort(); @@ -377,17 +382,18 @@ public class DataTransferResource extends ApplicationResource { value = "Commit or cancel the specified transaction", response = TransactionResultEntity.class, authorizations = { - @Authorization(value = "Write - /data-transfer/output-ports/{uuid}", type = "") + @Authorization(value = "Write - /data-transfer/output-ports/{uuid}", type = "") } ) @ApiResponses( value = { - @ApiResponse(code = 400, message = "NiFi was unable to complete the request because it was invalid. The request should not be retried without modification."), - @ApiResponse(code = 401, message = "Client could not be authenticated."), - @ApiResponse(code = 403, message = "Client is not authorized to make this request."), - @ApiResponse(code = 404, message = "The specified resource could not be found."), - @ApiResponse(code = 409, message = "The request was valid but NiFi was not in the appropriate state to process it. Retrying the same request later may be successful."), - @ApiResponse(code = 503, message = "NiFi instance is not ready for serving request, or temporarily overloaded. Retrying the same request later may be successful"),} + @ApiResponse(code = 400, message = "NiFi was unable to complete the request because it was invalid. The request should not be retried without modification."), + @ApiResponse(code = 401, message = "Client could not be authenticated."), + @ApiResponse(code = 403, message = "Client is not authorized to make this request."), + @ApiResponse(code = 404, message = "The specified resource could not be found."), + @ApiResponse(code = 409, message = "The request was valid but NiFi was not in the appropriate state to process it. Retrying the same request later may be successful."), + @ApiResponse(code = 503, message = "NiFi instance is not ready for serving request, or temporarily overloaded. Retrying the same request later may be successful"), + } ) public Response commitOutputPortTransaction( @ApiParam( @@ -474,6 +480,7 @@ public class DataTransferResource extends ApplicationResource { return clusterContext(noCache(setCommonHeaders(Response.ok(entity), transportProtocolVersion, transactionManager))).build(); } + @DELETE @Consumes(MediaType.APPLICATION_OCTET_STREAM) @Produces(MediaType.APPLICATION_JSON) @@ -482,17 +489,18 @@ public class DataTransferResource extends ApplicationResource { value = "Commit or cancel the specified transaction", response = TransactionResultEntity.class, authorizations = { - @Authorization(value = "Write - /data-transfer/input-ports/{uuid}", type = "") + @Authorization(value = "Write - /data-transfer/input-ports/{uuid}", type = "") } ) @ApiResponses( value = { - @ApiResponse(code = 400, message = "NiFi was unable to complete the request because it was invalid. The request should not be retried without modification."), - @ApiResponse(code = 401, message = "Client could not be authenticated."), - @ApiResponse(code = 403, message = "Client is not authorized to make this request."), - @ApiResponse(code = 404, message = "The specified resource could not be found."), - @ApiResponse(code = 409, message = "The request was valid but NiFi was not in the appropriate state to process it. Retrying the same request later may be successful."), - @ApiResponse(code = 503, message = "NiFi instance is not ready for serving request, or temporarily overloaded. Retrying the same request later may be successful"),} + @ApiResponse(code = 400, message = "NiFi was unable to complete the request because it was invalid. The request should not be retried without modification."), + @ApiResponse(code = 401, message = "Client could not be authenticated."), + @ApiResponse(code = 403, message = "Client is not authorized to make this request."), + @ApiResponse(code = 404, message = "The specified resource could not be found."), + @ApiResponse(code = 409, message = "The request was valid but NiFi was not in the appropriate state to process it. Retrying the same request later may be successful."), + @ApiResponse(code = 503, message = "NiFi instance is not ready for serving request, or temporarily overloaded. Retrying the same request later may be successful"), + } ) public Response commitInputPortTransaction( @ApiParam( @@ -590,6 +598,7 @@ public class DataTransferResource extends ApplicationResource { return Response.ok(entity).build(); } + @GET @Consumes(MediaType.WILDCARD) @Produces(MediaType.APPLICATION_OCTET_STREAM) @@ -598,18 +607,19 @@ public class DataTransferResource extends ApplicationResource { value = "Transfer flow files from the output port", response = StreamingOutput.class, authorizations = { - @Authorization(value = "Write - /data-transfer/output-ports/{uuid}", type = "") + @Authorization(value = "Write - /data-transfer/output-ports/{uuid}", type = "") } ) @ApiResponses( value = { - @ApiResponse(code = 200, message = "There is no flow file to return."), - @ApiResponse(code = 400, message = "NiFi was unable to complete the request because it was invalid. The request should not be retried without modification."), - @ApiResponse(code = 401, message = "Client could not be authenticated."), - @ApiResponse(code = 403, message = "Client is not authorized to make this request."), - @ApiResponse(code = 404, message = "The specified resource could not be found."), - @ApiResponse(code = 409, message = "The request was valid but NiFi was not in the appropriate state to process it. Retrying the same request later may be successful."), - @ApiResponse(code = 503, message = "NiFi instance is not ready for serving request, or temporarily overloaded. Retrying the same request later may be successful"),} + @ApiResponse(code = 200, message = "There is no flow file to return."), + @ApiResponse(code = 400, message = "NiFi was unable to complete the request because it was invalid. The request should not be retried without modification."), + @ApiResponse(code = 401, message = "Client could not be authenticated."), + @ApiResponse(code = 403, message = "Client is not authorized to make this request."), + @ApiResponse(code = 404, message = "The specified resource could not be found."), + @ApiResponse(code = 409, message = "The request was valid but NiFi was not in the appropriate state to process it. Retrying the same request later may be successful."), + @ApiResponse(code = 503, message = "NiFi instance is not ready for serving request, or temporarily overloaded. Retrying the same request later may be successful"), + } ) public Response transferFlowFiles( @ApiParam( @@ -681,16 +691,16 @@ public class DataTransferResource extends ApplicationResource { value = "Extend transaction TTL", response = TransactionResultEntity.class, authorizations = { - @Authorization(value = "Write - /data-transfer/input-ports/{uuid}", type = "") + @Authorization(value = "Write - /data-transfer/input-ports/{uuid}", type = "") } ) @ApiResponses( value = { - @ApiResponse(code = 400, message = "NiFi was unable to complete the request because it was invalid. The request should not be retried without modification."), - @ApiResponse(code = 401, message = "Client could not be authenticated."), - @ApiResponse(code = 403, message = "Client is not authorized to make this request."), - @ApiResponse(code = 404, message = "The specified resource could not be found."), - @ApiResponse(code = 409, message = "The request was valid but NiFi was not in the appropriate state to process it. Retrying the same request later may be successful.") + @ApiResponse(code = 400, message = "NiFi was unable to complete the request because it was invalid. The request should not be retried without modification."), + @ApiResponse(code = 401, message = "Client could not be authenticated."), + @ApiResponse(code = 403, message = "Client is not authorized to make this request."), + @ApiResponse(code = 404, message = "The specified resource could not be found."), + @ApiResponse(code = 409, message = "The request was valid but NiFi was not in the appropriate state to process it. Retrying the same request later may be successful.") } ) public Response extendInputPortTransactionTTL( @@ -716,17 +726,18 @@ public class DataTransferResource extends ApplicationResource { value = "Extend transaction TTL", response = TransactionResultEntity.class, authorizations = { - @Authorization(value = "Write - /data-transfer/output-ports/{uuid}", type = "") + @Authorization(value = "Write - /data-transfer/output-ports/{uuid}", type = "") } ) @ApiResponses( value = { - @ApiResponse(code = 400, message = "NiFi was unable to complete the request because it was invalid. The request should not be retried without modification."), - @ApiResponse(code = 401, message = "Client could not be authenticated."), - @ApiResponse(code = 403, message = "Client is not authorized to make this request."), - @ApiResponse(code = 404, message = "The specified resource could not be found."), - @ApiResponse(code = 409, message = "The request was valid but NiFi was not in the appropriate state to process it. Retrying the same request later may be successful."), - @ApiResponse(code = 503, message = "NiFi instance is not ready for serving request, or temporarily overloaded. Retrying the same request later may be successful"),} + @ApiResponse(code = 400, message = "NiFi was unable to complete the request because it was invalid. The request should not be retried without modification."), + @ApiResponse(code = 401, message = "Client could not be authenticated."), + @ApiResponse(code = 403, message = "Client is not authorized to make this request."), + @ApiResponse(code = 404, message = "The specified resource could not be found."), + @ApiResponse(code = 409, message = "The request was valid but NiFi was not in the appropriate state to process it. Retrying the same request later may be successful."), + @ApiResponse(code = 503, message = "NiFi instance is not ready for serving request, or temporarily overloaded. Retrying the same request later may be successful"), + } ) public Response extendOutputPortTransactionTTL( @PathParam("portId") String portId, @@ -789,7 +800,6 @@ public class DataTransferResource extends ApplicationResource { } private class ValidateRequestResult { - private Integer transportProtocolVersion; private Response errResponse; } @@ -820,7 +830,9 @@ public class DataTransferResource extends ApplicationResource { return result; } + // setters + public void setAuthorizer(Authorizer authorizer) { this.authorizer = authorizer; } diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/api/SiteToSiteResource.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/api/SiteToSiteResource.java index 036ac2ac2d..88bdeb65b8 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/api/SiteToSiteResource.java +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/api/SiteToSiteResource.java @@ -16,6 +16,10 @@ */ package org.apache.nifi.web.api; + +import java.net.InetAddress; +import java.net.UnknownHostException; + import com.wordnik.swagger.annotations.Api; import com.wordnik.swagger.annotations.ApiOperation; import com.wordnik.swagger.annotations.ApiResponse; @@ -56,8 +60,6 @@ import javax.ws.rs.Produces; import javax.ws.rs.core.Context; import javax.ws.rs.core.MediaType; import javax.ws.rs.core.Response; -import java.net.InetAddress; -import java.net.UnknownHostException; import java.util.ArrayList; import java.util.List; import java.util.Set; @@ -171,6 +173,7 @@ public class SiteToSiteResource extends ApplicationResource { @Path("/peers") @Consumes(MediaType.WILDCARD) @Produces({MediaType.APPLICATION_JSON, MediaType.APPLICATION_XML}) + // TODO: @PreAuthorize("hasRole('ROLE_NIFI')") @ApiOperation( value = "Returns the available Peers and its status of this NiFi", response = PeersEntity.class, diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/pom.xml b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/pom.xml index ed7492282d..9a27b13111 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/pom.xml +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/pom.xml @@ -40,6 +40,7 @@ nifi-resources nifi-documentation nifi-authorizer + nifi-properties-loader diff --git a/nifi-nar-bundles/nifi-framework-bundle/pom.xml b/nifi-nar-bundles/nifi-framework-bundle/pom.xml index 14593de925..d62ef60e63 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/pom.xml +++ b/nifi-nar-bundles/nifi-framework-bundle/pom.xml @@ -73,6 +73,11 @@ nifi-framework-core 1.0.0-SNAPSHOT + + org.apache.nifi + nifi-properties-loader + 1.0.0-SNAPSHOT + org.apache.nifi nifi-framework-authorization diff --git a/nifi-nar-bundles/nifi-provenance-repository-bundle/nifi-volatile-provenance-repository/src/test/java/org/apache/nifi/provenance/TestVolatileProvenanceRepository.java b/nifi-nar-bundles/nifi-provenance-repository-bundle/nifi-volatile-provenance-repository/src/test/java/org/apache/nifi/provenance/TestVolatileProvenanceRepository.java index 4190ebba08..942fea4994 100644 --- a/nifi-nar-bundles/nifi-provenance-repository-bundle/nifi-volatile-provenance-repository/src/test/java/org/apache/nifi/provenance/TestVolatileProvenanceRepository.java +++ b/nifi-nar-bundles/nifi-provenance-repository-bundle/nifi-volatile-provenance-repository/src/test/java/org/apache/nifi/provenance/TestVolatileProvenanceRepository.java @@ -44,7 +44,6 @@ public class TestVolatileProvenanceRepository { @Test public void testAddAndGet() throws IOException, InterruptedException { - repo = new VolatileProvenanceRepository(NiFiProperties.createBasicNiFiProperties(null, null)); final Map attributes = new HashMap<>(); diff --git a/nifi-toolkit/nifi-toolkit-assembly/NOTICE b/nifi-toolkit/nifi-toolkit-assembly/NOTICE index 653ba8389d..5ce896bd51 100644 --- a/nifi-toolkit/nifi-toolkit-assembly/NOTICE +++ b/nifi-toolkit/nifi-toolkit-assembly/NOTICE @@ -90,5 +90,15 @@ The following binary components are provided under the Apache Software License v (ASLv2) Jetty The following NOTICE information applies: - Jetty Web Container - Copyright 1995-2015 Mort Bay Consulting Pty Ltd. + Jetty Web Container + Copyright 1995-2015 Mort Bay Consulting Pty Ltd. + + (ASLv2) Groovy (org.codehaus.groovy:groovy-all:jar:2.4.5 - http://www.groovy-lang.org) + The following NOTICE information applies: + Groovy Language + Copyright 2003-2015 The respective authors and developers + Developers and Contributors are listed in the project POM file + and Gradle build + + This product includes software developed by + The Groovy community (http://groovy.codehaus.org/). \ No newline at end of file diff --git a/nifi-toolkit/nifi-toolkit-assembly/pom.xml b/nifi-toolkit/nifi-toolkit-assembly/pom.xml index 4439d5b3fe..cd2ece2532 100644 --- a/nifi-toolkit/nifi-toolkit-assembly/pom.xml +++ b/nifi-toolkit/nifi-toolkit-assembly/pom.xml @@ -18,7 +18,7 @@ language governing permissions and limitations under the License. --> nifi-toolkit-assembly pom - This is the assembly Apache NiFi Toolkit + This is the assembly for the Apache NiFi Toolkit @@ -64,6 +64,10 @@ language governing permissions and limitations under the License. --> org.apache.nifi nifi-toolkit-tls + + org.apache.nifi + nifi-toolkit-encrypt-config + org.slf4j slf4j-api diff --git a/nifi-toolkit/nifi-toolkit-assembly/src/main/assembly/dependencies.xml b/nifi-toolkit/nifi-toolkit-assembly/src/main/assembly/dependencies.xml index 626287a122..1ade1820df 100644 --- a/nifi-toolkit/nifi-toolkit-assembly/src/main/assembly/dependencies.xml +++ b/nifi-toolkit/nifi-toolkit-assembly/src/main/assembly/dependencies.xml @@ -45,6 +45,15 @@ conf/ 0600 + + + + + + + + + ${project.basedir}/src/main/resources/classpath classpath/ diff --git a/nifi-toolkit/nifi-toolkit-assembly/src/main/resources/bin/encrypt-config.bat b/nifi-toolkit/nifi-toolkit-assembly/src/main/resources/bin/encrypt-config.bat new file mode 100644 index 0000000000..de3fbf70f7 --- /dev/null +++ b/nifi-toolkit/nifi-toolkit-assembly/src/main/resources/bin/encrypt-config.bat @@ -0,0 +1,40 @@ +@echo off +rem +rem Licensed to the Apache Software Foundation (ASF) under one or more +rem contributor license agreements. See the NOTICE file distributed with +rem this work for additional information regarding copyright ownership. +rem The ASF licenses this file to You under the Apache License, Version 2.0 +rem (the "License"); you may not use this file except in compliance with +rem the License. You may obtain a copy of the License at +rem +rem http://www.apache.org/licenses/LICENSE-2.0 +rem +rem Unless required by applicable law or agreed to in writing, software +rem distributed under the License is distributed on an "AS IS" BASIS, +rem WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +rem See the License for the specific language governing permissions and +rem limitations under the License. +rem + +rem Use JAVA_HOME if it's set; otherwise, just use java + +if "%JAVA_HOME%" == "" goto noJavaHome +if not exist "%JAVA_HOME%\bin\java.exe" goto noJavaHome +set JAVA_EXE=%JAVA_HOME%\bin\java.exe +goto startConfig + +:noJavaHome +echo The JAVA_HOME environment variable is not defined correctly. +echo Instead the PATH will be used to find the java executable. +echo. +set JAVA_EXE=java +goto startConfig + +:startConfig +set LIB_DIR=%~dp0..\classpath;%~dp0..\lib + +SET JAVA_PARAMS=-cp %LIB_DIR%\* -Xms12m -Xmx24m %JAVA_ARGS% org.apache.nifi.util.config.ConfigEncryptionTool + +cmd.exe /C "%JAVA_EXE%" %JAVA_PARAMS% %* + +popd diff --git a/nifi-toolkit/nifi-toolkit-assembly/src/main/resources/bin/encrypt-config.sh b/nifi-toolkit/nifi-toolkit-assembly/src/main/resources/bin/encrypt-config.sh new file mode 100644 index 0000000000..b6b5f45f4f --- /dev/null +++ b/nifi-toolkit/nifi-toolkit-assembly/src/main/resources/bin/encrypt-config.sh @@ -0,0 +1,120 @@ +#!/bin/sh +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# + +# Script structure inspired from Apache Karaf and other Apache projects with similar startup approaches + +SCRIPT_DIR=$(dirname "$0") +SCRIPT_NAME=$(basename "$0") +NIFI_TOOLKIT_HOME=$(cd "${SCRIPT_DIR}" && cd .. && pwd) +PROGNAME=$(basename "$0") + + +warn() { + echo "${PROGNAME}: $*" +} + +die() { + warn "$*" + exit 1 +} + +detectOS() { + # OS specific support (must be 'true' or 'false'). + cygwin=false; + aix=false; + os400=false; + darwin=false; + case "$(uname)" in + CYGWIN*) + cygwin=true + ;; + AIX*) + aix=true + ;; + OS400*) + os400=true + ;; + Darwin) + darwin=true + ;; + esac + # For AIX, set an environment variable + if ${aix}; then + export LDR_CNTRL=MAXDATA=0xB0000000@DSA + echo ${LDR_CNTRL} + fi +} + +locateJava() { + # Setup the Java Virtual Machine + if $cygwin ; then + [ -n "${JAVA}" ] && JAVA=$(cygpath --unix "${JAVA}") + [ -n "${JAVA_HOME}" ] && JAVA_HOME=$(cygpath --unix "${JAVA_HOME}") + fi + + if [ "x${JAVA}" = "x" ] && [ -r /etc/gentoo-release ] ; then + JAVA_HOME=$(java-config --jre-home) + fi + if [ "x${JAVA}" = "x" ]; then + if [ "x${JAVA_HOME}" != "x" ]; then + if [ ! -d "${JAVA_HOME}" ]; then + die "JAVA_HOME is not valid: ${JAVA_HOME}" + fi + JAVA="${JAVA_HOME}/bin/java" + else + warn "JAVA_HOME not set; results may vary" + JAVA=$(type java) + JAVA=$(expr "${JAVA}" : '.* \(/.*\)$') + if [ "x${JAVA}" = "x" ]; then + die "java command not found" + fi + fi + fi +} + +init() { + # Determine if there is special OS handling we must perform + detectOS + + # Locate the Java VM to execute + locateJava +} + +run() { + LIBS="${NIFI_TOOLKIT_HOME}/lib/*" + + sudo_cmd_prefix="" + if $cygwin; then + NIFI_TOOLKIT_HOME=$(cygpath --path --windows "${NIFI_TOOLKIT_HOME}") + CLASSPATH="$NIFI_TOOLKIT_HOME/classpath";$(cygpath --path --windows "${LIBS}") + else + CLASSPATH="$NIFI_TOOLKIT_HOME/classpath:${LIBS}" + fi + + export JAVA_HOME="$JAVA_HOME" + export NIFI_TOOLKIT_HOME="$NIFI_TOOLKIT_HOME" + + umask 0077 + "${JAVA}" -cp "${CLASSPATH}" -Xms128m -Xmx256m org.apache.nifi.properties.ConfigEncryptionTool "$@" + return $? +} + + +init +run "$@" \ No newline at end of file diff --git a/nifi-toolkit/nifi-toolkit-encrypt-config/LICENSE b/nifi-toolkit/nifi-toolkit-encrypt-config/LICENSE new file mode 100644 index 0000000000..e1b4124db2 --- /dev/null +++ b/nifi-toolkit/nifi-toolkit-encrypt-config/LICENSE @@ -0,0 +1,225 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + + +This product bundles source from 'AbstractingTheJavaConsole'. The source is available under an MIT LICENSE. + + Copyright (C) 2010 McDowell + + Permission is hereby granted, free of charge, to any person obtaining a copy + of this software and associated documentation files (the "Software"), to deal + in the Software without restriction, including without limitation the rights + to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + copies of the Software, and to permit persons to whom the Software is + furnished to do so, subject to the following conditions: + + The above copyright notice and this permission notice shall be included in + all copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + THE SOFTWARE. diff --git a/nifi-toolkit/nifi-toolkit-encrypt-config/pom.xml b/nifi-toolkit/nifi-toolkit-encrypt-config/pom.xml new file mode 100644 index 0000000000..c146e9cc30 --- /dev/null +++ b/nifi-toolkit/nifi-toolkit-encrypt-config/pom.xml @@ -0,0 +1,162 @@ + + + + + org.apache.nifi + nifi-toolkit + 1.0.0-SNAPSHOT + + 4.0.0 + + nifi-toolkit-encrypt-config + Tool to encrypt sensitive configuration values + + + org.apache.nifi + nifi-properties + + + org.apache.nifi + nifi-properties-loader + + + org.apache.nifi + nifi-toolkit-tls + + + commons-logging + commons-logging + + + + + commons-cli + commons-cli + + + + + + + + ch.qos.logback + logback-classic + test + + + com.github.stefanbirkner + system-rules + 1.16.0 + test + + + + + + + org.apache.maven.plugins + maven-compiler-plugin + + + + compile + testCompile + + + groovy-eclipse-compiler + + + + + 1.8 + 1.8 + + + + org.codehaus.groovy + groovy-eclipse-compiler + 2.9.2-01 + + + org.codehaus.groovy + groovy-eclipse-batch + 2.4.3-01 + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + org.codehaus.mojo + build-helper-maven-plugin + 1.5 + + + add-source + generate-sources + + add-source + + + + src/main/groovy + + + + + add-test-source + generate-test-sources + + add-test-source + + + + src/test/groovy + + + + + + + + + \ No newline at end of file diff --git a/nifi-toolkit/nifi-toolkit-encrypt-config/src/main/groovy/org/apache/nifi/properties/ConfigEncryptionTool.groovy b/nifi-toolkit/nifi-toolkit-encrypt-config/src/main/groovy/org/apache/nifi/properties/ConfigEncryptionTool.groovy new file mode 100644 index 0000000000..8a275bef20 --- /dev/null +++ b/nifi-toolkit/nifi-toolkit-encrypt-config/src/main/groovy/org/apache/nifi/properties/ConfigEncryptionTool.groovy @@ -0,0 +1,578 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.properties + +import groovy.io.GroovyPrintWriter +import org.apache.commons.cli.CommandLine +import org.apache.commons.cli.CommandLineParser +import org.apache.commons.cli.DefaultParser +import org.apache.commons.cli.HelpFormatter +import org.apache.commons.cli.Options +import org.apache.commons.cli.ParseException +import org.apache.commons.codec.binary.Hex +import org.apache.nifi.toolkit.tls.commandLine.CommandLineParseException +import org.apache.nifi.toolkit.tls.commandLine.ExitCode +import org.apache.nifi.util.NiFiProperties +import org.apache.nifi.util.console.TextDevice +import org.apache.nifi.util.console.TextDevices +import org.bouncycastle.crypto.generators.SCrypt +import org.bouncycastle.jce.provider.BouncyCastleProvider +import org.slf4j.Logger +import org.slf4j.LoggerFactory + +import javax.crypto.Cipher +import java.nio.charset.StandardCharsets +import java.security.KeyException +import java.security.Security + +class ConfigEncryptionTool { + private static final Logger logger = LoggerFactory.getLogger(ConfigEncryptionTool.class) + + public String bootstrapConfPath + public String niFiPropertiesPath + public String outputNiFiPropertiesPath + public String loginIdentityProvidersPath + + private String keyHex + private String password + private NiFiProperties niFiProperties + + private boolean usingPassword = true + private boolean isVerbose = false + + private static final String HELP_ARG = "help" + private static final String VERBOSE_ARG = "verbose" + private static final String BOOTSTRAP_CONF_ARG = "bootstrapConf" + private static final String NIFI_PROPERTIES_ARG = "niFiProperties" + private static final String OUTPUT_NIFI_PROPERTIES_ARG = "outputNiFiProperties" + private static final String KEY_ARG = "key" + private static final String PASSWORD_ARG = "password" + private static final String USE_KEY_ARG = "useRawKey" + + private static final int MIN_PASSWORD_LENGTH = 12 + + // Strong parameters as of 12 Aug 2016 + private static final int SCRYPT_N = 2**16 + private static final int SCRYPT_R = 8 + private static final int SCRYPT_P = 1 + + private static + final String BOOTSTRAP_KEY_COMMENT = "# Master key in hexadecimal format for encrypted sensitive configuration values" + private static final String BOOTSTRAP_KEY_PREFIX = "nifi.bootstrap.sensitive.key=" + private static final String JAVA_HOME = "JAVA_HOME" + private static final String NIFI_TOOLKIT_HOME = "NIFI_TOOLKIT_HOME" + private static final String SEP = System.lineSeparator() + + private static final String FOOTER = buildFooter() + + private static + final String DEFAULT_DESCRIPTION = "This tool reads from a nifi.properties file with plain sensitive configuration values, prompts the user for a master key, and encrypts each value. It will replace the plain value with the protected value in the same file (or write to a new nifi.properties file if specified)." + + private static String buildHeader(String description = DEFAULT_DESCRIPTION) { + "${SEP}${description}${SEP * 2}" + } + + private static String buildFooter() { + "${SEP}Java home: ${System.getenv(JAVA_HOME)}${SEP}NiFi Toolkit home: ${System.getenv(NIFI_TOOLKIT_HOME)}" + } + + private final Options options; + private final String header; + + + public ConfigEncryptionTool() { + this(DEFAULT_DESCRIPTION) + } + + public ConfigEncryptionTool(String description) { + this.header = buildHeader(description) + this.options = new Options() + options.addOption("h", HELP_ARG, false, "Prints this usage message") + options.addOption("v", VERBOSE_ARG, false, "Sets verbose mode (default false)") + options.addOption("n", NIFI_PROPERTIES_ARG, true, "The nifi.properties file containing unprotected config values (will be overwritten)") + options.addOption("b", BOOTSTRAP_CONF_ARG, true, "The bootstrap.conf file to persist master key") + options.addOption("o", OUTPUT_NIFI_PROPERTIES_ARG, true, "The destination nifi.properties file containing protected config values (will not modify input nifi.properties)") + options.addOption("k", KEY_ARG, true, "The raw hexadecimal key to use to encrypt the sensitive properties") + options.addOption("p", PASSWORD_ARG, true, "The password from which to derive the key to use to encrypt the sensitive properties") + options.addOption("r", USE_KEY_ARG, false, "If provided, the secure console will prompt for the raw key value in hexadecimal form") + } + + /** + * Prints the usage message and available arguments for this tool (along with a specific error message if provided). + * + * @param errorMessage the optional error message + */ + public void printUsage(String errorMessage) { + if (errorMessage) { + System.out.println(errorMessage) + System.out.println() + } + HelpFormatter helpFormatter = new HelpFormatter() + helpFormatter.setWidth(160) + helpFormatter.printHelp(ConfigEncryptionTool.class.getCanonicalName(), header, options, FOOTER, true) + } + + protected void printUsageAndThrow(String errorMessage, ExitCode exitCode) throws CommandLineParseException { + printUsage(errorMessage); + throw new CommandLineParseException(errorMessage, exitCode); + } + + protected CommandLine parse(String[] args) throws CommandLineParseException { + CommandLineParser parser = new DefaultParser() + CommandLine commandLine + try { + commandLine = parser.parse(options, args) + if (commandLine.hasOption(HELP_ARG)) { + printUsageAndThrow(null, ExitCode.HELP) + } + + isVerbose = commandLine.hasOption(VERBOSE_ARG) + + bootstrapConfPath = commandLine.getOptionValue(BOOTSTRAP_CONF_ARG, determineDefaultBootstrapConfPath()) + niFiPropertiesPath = commandLine.getOptionValue(NIFI_PROPERTIES_ARG, determineDefaultNiFiPropertiesPath()) + outputNiFiPropertiesPath = commandLine.getOptionValue(OUTPUT_NIFI_PROPERTIES_ARG, niFiPropertiesPath) + + if (niFiPropertiesPath == outputNiFiPropertiesPath) { + // TODO: Add confirmation pause and provide -y flag to offer no-interaction mode? + logger.warn("The source nifi.properties and destination nifi.properties are identical [${outputNiFiPropertiesPath}] so the original will be overwritten") + } + + if (commandLine.hasOption(PASSWORD_ARG)) { + usingPassword = true + if (commandLine.hasOption(KEY_ARG)) { + printUsageAndThrow("Only one of ${PASSWORD_ARG} and ${KEY_ARG} can be used", ExitCode.INVALID_ARGS) + } else { + password = commandLine.getOptionValue(PASSWORD_ARG) + } + } else { + keyHex = commandLine.getOptionValue(KEY_ARG) + usingPassword = !keyHex + } + + if (commandLine.hasOption(USE_KEY_ARG)) { + if (keyHex || password) { + logger.warn("If the key or password is provided in the arguments, '-r'/'--${USE_KEY_ARG}' is ignored") + } else { + usingPassword = false + } + } + } catch (ParseException e) { + if (isVerbose) { + logger.error("Encountered an error", e) + } + printUsageAndThrow("Error parsing command line. (" + e.getMessage() + ")", ExitCode.ERROR_PARSING_COMMAND_LINE) + } + return commandLine + } + + private String getKey(TextDevice device = TextDevices.defaultTextDevice()) { + if (usingPassword) { + if (!password) { + password = readPasswordFromConsole(device) + } + keyHex = deriveKeyFromPassword(password) + password = null + usingPassword = false + + return keyHex + } else { + if (!keyHex) { + keyHex = readKeyFromConsole(device) + } + + return keyHex + } + } + + private static String readKeyFromConsole(TextDevice textDevice) { + textDevice.printf("Enter the master key in hexadecimal format (spaces acceptable): ") + new String(textDevice.readPassword()) + } + + private static String readPasswordFromConsole(TextDevice textDevice) { + textDevice.printf("Enter the password: ") + new String(textDevice.readPassword()) + } + + /** + * Returns the key in uppercase hexadecimal format with delimiters (spaces, '-', etc.) removed. All non-hex chars are removed. If the result is not a valid length (32, 48, 64 chars depending on the JCE), an exception is thrown. + * + * @param rawKey the unprocessed key input + * @return the formatted hex string in uppercase + * @throws KeyException if the key is not a valid length after parsing + */ + private static String parseKey(String rawKey) throws KeyException { + String hexKey = rawKey.replaceAll("[^0-9a-fA-F]", "") + def validKeyLengths = getValidKeyLengths() + if (!validKeyLengths.contains(hexKey.size() * 4)) { + throw new KeyException("The key (${hexKey.size()} hex chars) must be of length ${validKeyLengths} bits (${validKeyLengths.collect { it / 4 }} hex characters)") + } + hexKey.toUpperCase() + } + + /** + * Returns the list of acceptable key lengths in bits based on the current JCE policies. + * + * @return 128 , [192, 256] + */ + public static List getValidKeyLengths() { + Cipher.getMaxAllowedKeyLength("AES") > 128 ? [128, 192, 256] : [128] + } + + /** + * Loads the {@link NiFiProperties} instance from the provided file path (restoring the original value of the System property {@code nifi.properties.file.path} after loading this instance). + * + * @return the NiFiProperties instance + * @throw IOException if the nifi.properties file cannot be read + */ + private NiFiProperties loadNiFiProperties() throws IOException { + File niFiPropertiesFile + if (niFiPropertiesPath && (niFiPropertiesFile = new File(niFiPropertiesPath)).exists()) { + String oldNiFiPropertiesPath = System.getProperty(NiFiProperties.PROPERTIES_FILE_PATH) + logger.debug("Saving existing NiFiProperties file path ${oldNiFiPropertiesPath}") + + System.setProperty(NiFiProperties.PROPERTIES_FILE_PATH, niFiPropertiesFile.absolutePath) + logger.debug("Temporarily set NiFiProperties file path to ${niFiPropertiesFile.absolutePath}") + + NiFiProperties properties + try { + properties = NiFiPropertiesLoader.withKey(keyHex).load(niFiPropertiesFile) + logger.info("Loaded NiFiProperties instance with ${properties.size()} properties") + return properties + } catch (RuntimeException e) { + if (isVerbose) { + logger.error("Encountered an error", e) + } + throw new IOException("Cannot load NiFiProperties from [${niFiPropertiesPath}]", e) + } finally { + // Can't set a system property to null + if (oldNiFiPropertiesPath) { + System.setProperty(NiFiProperties.PROPERTIES_FILE_PATH, oldNiFiPropertiesPath) + } else { + System.clearProperty(NiFiProperties.PROPERTIES_FILE_PATH) + } + logger.debug("Restored system variable ${NiFiProperties.PROPERTIES_FILE_PATH} to ${oldNiFiPropertiesPath}") + } + } else { + printUsageAndThrow("Cannot load NiFiProperties from [${niFiPropertiesPath}]", ExitCode.ERROR_READING_NIFI_PROPERTIES) + } + } + + /** + * Accepts a {@link NiFiProperties} instance, iterates over all non-empty sensitive properties which are not already marked as protected, encrypts them using the master key, and updates the property with the protected value. Additionally, adds a new sibling property {@code x.y.z.protected=aes/gcm/{128,256}} for each indicating the encryption scheme used. + * + * @param plainProperties the NiFiProperties instance containing the raw values + * @return the NiFiProperties containing protected values + */ + private NiFiProperties encryptSensitiveProperties(NiFiProperties plainProperties) { + if (!plainProperties) { + throw new IllegalArgumentException("Cannot encrypt empty NiFiProperties") + } + + ProtectedNiFiProperties protectedWrapper = new ProtectedNiFiProperties(plainProperties) + + List sensitivePropertyKeys = protectedWrapper.getSensitivePropertyKeys() + if (sensitivePropertyKeys.isEmpty()) { + logger.info("No sensitive properties to encrypt") + return plainProperties + } + + // Holder for encrypted properties and protection schemes + Properties encryptedProperties = new Properties() + + AESSensitivePropertyProvider spp = new AESSensitivePropertyProvider(keyHex) + protectedWrapper.addSensitivePropertyProvider(spp) + + List keysToSkip = [] + + // Iterate over each -- encrypt and add .protected if populated + sensitivePropertyKeys.each { String key -> + if (!plainProperties.getProperty(key)) { + logger.debug("Skipping encryption of ${key} because it is empty") + } else { + String protectedValue = spp.protect(plainProperties.getProperty(key)) + + // Add the encrypted value + encryptedProperties.setProperty(key, protectedValue) + logger.info("Protected ${key} with ${spp.getIdentifierKey()} -> \t${protectedValue}") + + // Add the protection key ("x.y.z.protected" -> "aes/gcm/{128,256}") + String protectionKey = protectedWrapper.getProtectionKey(key) + encryptedProperties.setProperty(protectionKey, spp.getIdentifierKey()) + logger.info("Updated protection key ${protectionKey}") + + keysToSkip << key << protectionKey + } + } + + // Combine the original raw NiFiProperties and the newly-encrypted properties + // Memory-wasteful but NiFiProperties are immutable -- no setter available (unless we monkey-patch...) + Set nonSensitiveKeys = plainProperties.getPropertyKeys() - keysToSkip + nonSensitiveKeys.each { String key -> + encryptedProperties.setProperty(key, plainProperties.getProperty(key)) + } + NiFiProperties mergedProperties = new StandardNiFiProperties(encryptedProperties) + logger.info("Final result: ${mergedProperties.size()} keys including ${ProtectedNiFiProperties.countProtectedProperties(mergedProperties)} protected keys") + + mergedProperties + } + + /** + * Reads the existing {@code bootstrap.conf} file, updates it to contain the master key, and persists it back to the same location. + * + * @throw IOException if there is a problem reading or writing the bootstrap.conf file + */ + private void writeKeyToBootstrapConf() throws IOException { + File bootstrapConfFile + if (bootstrapConfPath && (bootstrapConfFile = new File(bootstrapConfPath)).exists() && bootstrapConfFile.canRead() && bootstrapConfFile.canWrite()) { + try { + List lines = bootstrapConfFile.readLines() + + updateBootstrapContentsWithKey(lines) + + // Write the updated values back to the file + bootstrapConfFile.text = lines.join("\n") + } catch (IOException e) { + def msg = "Encountered an exception updating the bootstrap.conf file with the master key" + logger.error(msg, e) + throw e + } + } else { + throw new IOException("The bootstrap.conf file at ${bootstrapConfPath} must exist and be readable and writable by the user running this tool") + } + } + + /** + * Accepts the lines of the {@code bootstrap.conf} file as a {@code List } and updates or adds the key property (and associated comment). + * + * @param lines the lines of the bootstrap file + * @return the updated lines + */ + private List updateBootstrapContentsWithKey(List lines) { + String keyLine = "${BOOTSTRAP_KEY_PREFIX}${keyHex}" + // Try to locate the key property line + int keyLineIndex = lines.findIndexOf { it.startsWith(BOOTSTRAP_KEY_PREFIX) } + + // If it was found, update inline + if (keyLineIndex != -1) { + logger.debug("The key property was detected in bootstrap.conf") + lines[keyLineIndex] = keyLine + logger.debug("The bootstrap key value was updated") + + // Ensure the comment explaining the property immediately precedes it (check for edge case where key is first line) + int keyCommentLineIndex = keyLineIndex > 0 ? keyLineIndex - 1 : 0 + if (lines[keyCommentLineIndex] != BOOTSTRAP_KEY_COMMENT) { + lines.add(keyCommentLineIndex, BOOTSTRAP_KEY_COMMENT) + logger.debug("A comment explaining the bootstrap key property was added") + } + } else { + // If it wasn't present originally, add the comment and key property + lines.addAll(["\n", BOOTSTRAP_KEY_COMMENT, keyLine]) + logger.debug("The key property was not detected in bootstrap.conf so it was added along with a comment explaining it") + } + + lines + } + + /** + * Writes the contents of the {@link NiFiProperties} instance with encrypted values to the output {@code nifi.properties} file. + * + * @throw IOException if there is a problem reading or writing the nifi.properties file + */ + private void writeNiFiProperties() throws IOException { + if (!outputNiFiPropertiesPath) { + throw new IllegalArgumentException("Cannot write encrypted properties to empty nifi.properties path") + } + + File outputNiFiPropertiesFile = new File(outputNiFiPropertiesPath) + + if (isSafeToWrite(outputNiFiPropertiesFile)) { + try { + List linesToPersist + File niFiPropertiesFile = new File(niFiPropertiesPath) + if (niFiPropertiesFile.exists() && niFiPropertiesFile.canRead()) { + // Instead of just writing the NiFiProperties instance to a properties file, this method attempts to maintain the structure of the original file and preserves comments + linesToPersist = serializeNiFiPropertiesAndPreserveFormat(niFiProperties, niFiPropertiesFile) + } else { + linesToPersist = serializeNiFiProperties(niFiProperties) + } + + // Write the updated values back to the file + outputNiFiPropertiesFile.text = linesToPersist.join("\n") + } catch (IOException e) { + def msg = "Encountered an exception updating the nifi.properties file with the encrypted values" + logger.error(msg, e) + throw e + } + } else { + throw new IOException("The nifi.properties file at ${outputNiFiPropertiesPath} must be writable by the user running this tool") + } + } + + private + static List serializeNiFiPropertiesAndPreserveFormat(NiFiProperties niFiProperties, File originalPropertiesFile) { + List lines = originalPropertiesFile.readLines() + + ProtectedNiFiProperties protectedNiFiProperties = new ProtectedNiFiProperties(niFiProperties) + // Only need to replace the keys that have been protected + Map protectedKeys = protectedNiFiProperties.getProtectedPropertyKeys() + + protectedKeys.each { String key, String protectionScheme -> + int l = lines.findIndexOf { it.startsWith(key) } + if (l != -1) { + lines[l] = "${key}=${protectedNiFiProperties.getProperty(key)}" + } + // Get the index of the following line (or cap at max) + int p = l + 1 > lines.size() ? lines.size() : l + 1 + String protectionLine = "${protectedNiFiProperties.getProtectionKey(key)}=${protectionScheme}" + if (p < lines.size() && lines.get(p).startsWith("${protectedNiFiProperties.getProtectionKey(key)}=")) { + lines.set(p, protectionLine) + } else { + lines.add(p, protectionLine) + } + } + + lines + } + + private static List serializeNiFiProperties(NiFiProperties nifiProperties) { + OutputStream out = new ByteArrayOutputStream() + Writer writer = new GroovyPrintWriter(out) + + // Again, waste of memory, but respecting the interface + Properties properties = new Properties() + nifiProperties.getPropertyKeys().each { String key -> + properties.setProperty(key, nifiProperties.getProperty(key)) + } + + properties.store(writer, null) + writer.flush() + out.toString().split("\n") + } + + /** + * Helper method which returns true if it is "safe" to write to the provided file. + * + * Conditions: + * file does not exist and the parent directory is writable + * -OR- + * file exists and is writable + * + * @param fileToWrite the proposed file to be written to + * @return true if the caller can "safely" write to this file location + */ + private static boolean isSafeToWrite(File fileToWrite) { + fileToWrite && ((!fileToWrite.exists() && fileToWrite.absoluteFile.parentFile.canWrite()) || (fileToWrite.exists() && fileToWrite.canWrite())) + } + + private static String determineDefaultBootstrapConfPath() { + String niFiToolkitPath = System.getenv(NIFI_TOOLKIT_HOME) ?: "" + "${niFiToolkitPath ? niFiToolkitPath + "/" : ""}conf/bootstrap.conf" + } + + private static String determineDefaultNiFiPropertiesPath() { + String niFiToolkitPath = System.getenv(NIFI_TOOLKIT_HOME) ?: "" + "${niFiToolkitPath ? niFiToolkitPath + "/" : ""}conf/nifi.properties" + } + + private static String deriveKeyFromPassword(String password) { + password = password?.trim() + if (!password || password.length() < MIN_PASSWORD_LENGTH) { + throw new KeyException("Cannot derive key from empty/short password -- password must be at least ${MIN_PASSWORD_LENGTH} characters") + } + + // Generate a 128 bit salt + byte[] salt = generateScryptSalt() + int keyLengthInBytes = getValidKeyLengths().max() / 8 + byte[] derivedKeyBytes = SCrypt.generate(password.getBytes(StandardCharsets.UTF_8), salt, SCRYPT_N, SCRYPT_R, SCRYPT_P, keyLengthInBytes) + Hex.encodeHexString(derivedKeyBytes).toUpperCase() + } + + private static byte[] generateScryptSalt() { +// byte[] salt = new byte[16] +// new SecureRandom().nextBytes(salt) +// salt + /* It is not ideal to use a static salt, but the KDF operation must be deterministic + for a given password, and storing and retrieving the salt in bootstrap.conf causes + compatibility concerns + */ + "NIFI_SCRYPT_SALT".getBytes(StandardCharsets.UTF_8) + } + + /** + * Runs main tool logic (parsing arguments, reading files, protecting properties, and writing key and properties out to destination files). + * + * @param args the command-line arguments + */ + public static void main(String[] args) { + Security.addProvider(new BouncyCastleProvider()) + + ConfigEncryptionTool tool = new ConfigEncryptionTool() + + try { + try { + tool.parse(args) + + tool.keyHex = tool.getKey() + + if (!tool.keyHex) { + tool.printUsageAndThrow("Hex key must be provided", ExitCode.INVALID_ARGS) + } + + try { + // Validate the length and format + tool.keyHex = parseKey(tool.keyHex) + } catch (KeyException e) { + if (tool.isVerbose) { + logger.error("Encountered an error", e) + } + tool.printUsageAndThrow(e.getMessage(), ExitCode.INVALID_ARGS) + } + + tool.niFiProperties = tool.loadNiFiProperties() + tool.niFiProperties = tool.encryptSensitiveProperties(tool.niFiProperties) + } catch (CommandLineParseException e) { + if (e.exitCode == ExitCode.HELP) { + System.exit(ExitCode.HELP.ordinal()) + } + throw e + } catch (Exception e) { + if (tool.isVerbose) { + logger.error("Encountered an error", e) + } + tool.printUsageAndThrow(e.message, ExitCode.ERROR_PARSING_COMMAND_LINE) + } + + try { + // Do this as part of a transaction? + synchronized (this) { + tool.writeKeyToBootstrapConf() + tool.writeNiFiProperties() + } + } catch (Exception e) { + if (tool.isVerbose) { + logger.error("Encountered an error", e) + } + tool.printUsageAndThrow("Encountered an error writing the master key to the bootstrap.conf file and the encrypted properties to nifi.properties", ExitCode.ERROR_GENERATING_CONFIG) + } + } catch (CommandLineParseException e) { + System.exit(e.exitCode.ordinal()) + } + + System.exit(ExitCode.SUCCESS.ordinal()) + } +} diff --git a/nifi-toolkit/nifi-toolkit-encrypt-config/src/main/java/org/apache/nifi/properties/JavaMain.java b/nifi-toolkit/nifi-toolkit-encrypt-config/src/main/java/org/apache/nifi/properties/JavaMain.java new file mode 100644 index 0000000000..2109ca19d3 --- /dev/null +++ b/nifi-toolkit/nifi-toolkit-encrypt-config/src/main/java/org/apache/nifi/properties/JavaMain.java @@ -0,0 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.properties; + +public class JavaMain { + public static void main(String[] args) { + System.out.println("The Java class #main ran successfully"); + } +} diff --git a/nifi-toolkit/nifi-toolkit-encrypt-config/src/main/java/org/apache/nifi/util/console/CharacterDevice.java b/nifi-toolkit/nifi-toolkit-encrypt-config/src/main/java/org/apache/nifi/util/console/CharacterDevice.java new file mode 100644 index 0000000000..ed254d9be9 --- /dev/null +++ b/nifi-toolkit/nifi-toolkit-encrypt-config/src/main/java/org/apache/nifi/util/console/CharacterDevice.java @@ -0,0 +1,73 @@ +/* +Copyright (c) 2010 McDowell + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. + */ +package org.apache.nifi.util.console; + +import java.io.BufferedReader; +import java.io.IOException; +import java.io.PrintWriter; +import java.io.Reader; + +/** + * @{link TextDevice} implementation wrapping character streams. + * + * @author McDowell + */ +class CharacterDevice extends TextDevice { + private final BufferedReader reader; + private final PrintWriter writer; + + public CharacterDevice(BufferedReader reader, PrintWriter writer) { + this.reader = reader; + this.writer = writer; + } + + @Override + public CharacterDevice printf(String fmt, Object... params) + throws ConsoleException { + writer.printf(fmt, params); + return this; + } + + @Override + public String readLine() throws ConsoleException { + try { + return reader.readLine(); + } catch (IOException e) { + throw new IllegalStateException(); + } + } + + @Override + public char[] readPassword() throws ConsoleException { + return readLine().toCharArray(); + } + + @Override + public Reader reader() throws ConsoleException { + return reader; + } + + @Override + public PrintWriter writer() throws ConsoleException { + return writer; + } +} \ No newline at end of file diff --git a/nifi-toolkit/nifi-toolkit-encrypt-config/src/main/java/org/apache/nifi/util/console/ConsoleDevice.java b/nifi-toolkit/nifi-toolkit-encrypt-config/src/main/java/org/apache/nifi/util/console/ConsoleDevice.java new file mode 100644 index 0000000000..3086d66add --- /dev/null +++ b/nifi-toolkit/nifi-toolkit-encrypt-config/src/main/java/org/apache/nifi/util/console/ConsoleDevice.java @@ -0,0 +1,66 @@ +/* +Copyright (c) 2010 McDowell + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. + */ +package org.apache.nifi.util.console; + +import java.io.Console; +import java.io.PrintWriter; +import java.io.Reader; + +/** + * {@link TextDevice} implementation wrapping a {@link Console}. + * + * @author McDowell + */ +class ConsoleDevice extends TextDevice { + private final Console console; + + public ConsoleDevice(Console console) { + this.console = console; + } + + @Override + public TextDevice printf(String fmt, Object... params) + throws ConsoleException { + console.format(fmt, params); + return this; + } + + @Override + public Reader reader() throws ConsoleException { + return console.reader(); + } + + @Override + public String readLine() throws ConsoleException { + return console.readLine(); + } + + @Override + public char[] readPassword() throws ConsoleException { + return console.readPassword(); + } + + @Override + public PrintWriter writer() throws ConsoleException { + return console.writer(); + } +} \ No newline at end of file diff --git a/nifi-toolkit/nifi-toolkit-encrypt-config/src/main/java/org/apache/nifi/util/console/ConsoleException.java b/nifi-toolkit/nifi-toolkit-encrypt-config/src/main/java/org/apache/nifi/util/console/ConsoleException.java new file mode 100644 index 0000000000..c6033ff874 --- /dev/null +++ b/nifi-toolkit/nifi-toolkit-encrypt-config/src/main/java/org/apache/nifi/util/console/ConsoleException.java @@ -0,0 +1,35 @@ +/* +Copyright (c) 2010 McDowell + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. + */ +package org.apache.nifi.util.console; + +/** + * Runtime exception for handling console errors. + * + * @author McDowell + */ +public class ConsoleException extends RuntimeException { + private static final long serialVersionUID = 1L; + + public ConsoleException(Throwable t) { + super(t); + } +} \ No newline at end of file diff --git a/nifi-toolkit/nifi-toolkit-encrypt-config/src/main/java/org/apache/nifi/util/console/TextDevice.java b/nifi-toolkit/nifi-toolkit-encrypt-config/src/main/java/org/apache/nifi/util/console/TextDevice.java new file mode 100644 index 0000000000..7f8af8f1af --- /dev/null +++ b/nifi-toolkit/nifi-toolkit-encrypt-config/src/main/java/org/apache/nifi/util/console/TextDevice.java @@ -0,0 +1,43 @@ +/* +Copyright (c) 2010 McDowell + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. + */ +package org.apache.nifi.util.console; + +import java.io.PrintWriter; +import java.io.Reader; + +/** + * Abstraction representing a text input/output device. + * + * @author McDowell + */ +public abstract class TextDevice { + public abstract TextDevice printf(String fmt, Object... params) + throws ConsoleException; + + public abstract String readLine() throws ConsoleException; + + public abstract char[] readPassword() throws ConsoleException; + + public abstract Reader reader() throws ConsoleException; + + public abstract PrintWriter writer() throws ConsoleException; +} \ No newline at end of file diff --git a/nifi-toolkit/nifi-toolkit-encrypt-config/src/main/java/org/apache/nifi/util/console/TextDevices.java b/nifi-toolkit/nifi-toolkit-encrypt-config/src/main/java/org/apache/nifi/util/console/TextDevices.java new file mode 100644 index 0000000000..986111855b --- /dev/null +++ b/nifi-toolkit/nifi-toolkit-encrypt-config/src/main/java/org/apache/nifi/util/console/TextDevices.java @@ -0,0 +1,80 @@ +/* +Copyright (c) 2010 McDowell + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. + */ +package org.apache.nifi.util.console; + +import java.io.BufferedReader; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.io.OutputStream; +import java.io.PrintWriter; + +/** + * Convenience class for providing {@link TextDevice} implementations. + * + * @author McDowell + */ +public final class TextDevices { + private TextDevices() {} + + private static TextDevice DEFAULT = (System.console() == null) ? streamDevice( + System.in, System.out) + : new ConsoleDevice(System.console()); + + /** + * The default system text I/O device. + * + * @return the default device + */ + public static TextDevice defaultTextDevice() { + return DEFAULT; + } + + /** + * Returns a text I/O device wrapping the given streams. The default system + * encoding is used to decode/encode data. + * + * @param in + * an input source + * @param out + * an output target + * @return a new device + */ + public static TextDevice streamDevice(InputStream in, OutputStream out) { + BufferedReader reader = new BufferedReader(new InputStreamReader(in)); + PrintWriter writer = new PrintWriter(out, true); + return new CharacterDevice(reader, writer); + } + + /** + * Returns a text I/O device wrapping the given streams. + * + * @param reader + * an input source + * @param writer + * an output target + * @return a new device + */ + public static TextDevice characterDevice(BufferedReader reader, + PrintWriter writer) { + return new CharacterDevice(reader, writer); + } +} \ No newline at end of file diff --git a/nifi-toolkit/nifi-toolkit-encrypt-config/src/main/resources/log4j.properties b/nifi-toolkit/nifi-toolkit-encrypt-config/src/main/resources/log4j.properties new file mode 100644 index 0000000000..fc2aaf12c4 --- /dev/null +++ b/nifi-toolkit/nifi-toolkit-encrypt-config/src/main/resources/log4j.properties @@ -0,0 +1,22 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +log4j.rootLogger=INFO,console + +log4j.appender.console=org.apache.log4j.ConsoleAppender +log4j.appender.console.layout=org.apache.log4j.PatternLayout +log4j.appender.console.layout.ConversionPattern=%d{yy/MM/dd HH:mm:ss} %p %c{2}: %m%n \ No newline at end of file diff --git a/nifi-toolkit/nifi-toolkit-encrypt-config/src/main/resources/logback.xml b/nifi-toolkit/nifi-toolkit-encrypt-config/src/main/resources/logback.xml new file mode 100644 index 0000000000..fb7e9614a2 --- /dev/null +++ b/nifi-toolkit/nifi-toolkit-encrypt-config/src/main/resources/logback.xml @@ -0,0 +1,34 @@ + + + + + + + %date %level [%thread] %logger{40} %msg%n + + + + + + + + + + + + + + diff --git a/nifi-toolkit/nifi-toolkit-encrypt-config/src/test/groovy/org/apache/nifi/properties/ConfigEncryptionToolTest.groovy b/nifi-toolkit/nifi-toolkit-encrypt-config/src/test/groovy/org/apache/nifi/properties/ConfigEncryptionToolTest.groovy new file mode 100644 index 0000000000..b4ca22dccc --- /dev/null +++ b/nifi-toolkit/nifi-toolkit-encrypt-config/src/test/groovy/org/apache/nifi/properties/ConfigEncryptionToolTest.groovy @@ -0,0 +1,1458 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.properties + +import ch.qos.logback.classic.spi.LoggingEvent +import ch.qos.logback.core.AppenderBase +import org.apache.commons.codec.binary.Hex +import org.apache.nifi.toolkit.tls.commandLine.CommandLineParseException +import org.apache.nifi.util.NiFiProperties +import org.apache.nifi.util.console.TextDevice +import org.apache.nifi.util.console.TextDevices +import org.bouncycastle.crypto.generators.SCrypt +import org.bouncycastle.jce.provider.BouncyCastleProvider +import org.junit.After +import org.junit.Before +import org.junit.BeforeClass +import org.junit.Rule +import org.junit.Test +import org.junit.contrib.java.lang.system.Assertion +import org.junit.contrib.java.lang.system.ExpectedSystemExit +import org.junit.contrib.java.lang.system.SystemOutRule +import org.junit.runner.RunWith +import org.junit.runners.JUnit4 +import org.slf4j.Logger +import org.slf4j.LoggerFactory + +import javax.crypto.Cipher +import java.nio.file.Files +import java.nio.file.attribute.PosixFilePermission +import java.security.KeyException +import java.security.Security + +@RunWith(JUnit4.class) +class ConfigEncryptionToolTest extends GroovyTestCase { + private static final Logger logger = LoggerFactory.getLogger(ConfigEncryptionToolTest.class) + + @Rule + public final ExpectedSystemExit exit = ExpectedSystemExit.none() + + @Rule + public final SystemOutRule systemOutRule = new SystemOutRule().enableLog() + + private static final String KEY_HEX_128 = "0123456789ABCDEFFEDCBA9876543210" + private static final String KEY_HEX_256 = KEY_HEX_128 * 2 + public static final String KEY_HEX = isUnlimitedStrengthCryptoAvailable() ? KEY_HEX_256 : KEY_HEX_128 + private static final String PASSWORD = "thisIsABadPassword" + + @BeforeClass + public static void setUpOnce() throws Exception { + Security.addProvider(new BouncyCastleProvider()) + + logger.metaClass.methodMissing = { String name, args -> + logger.info("[${name?.toUpperCase()}] ${(args as List).join(" ")}") + } + } + + @Before + public void setUp() throws Exception { + + } + + @After + public void tearDown() throws Exception { + TestAppender.reset() + } + + private static boolean isUnlimitedStrengthCryptoAvailable() { + Cipher.getMaxAllowedKeyLength("AES") > 128 + } + + private static void printProperties(NiFiProperties properties) { + if (!(properties instanceof ProtectedNiFiProperties)) { + properties = new ProtectedNiFiProperties(properties) + } + + (properties as ProtectedNiFiProperties).getPropertyKeysIncludingProtectionSchemes().sort().each { String key -> + logger.info("${key}\t\t${properties.getProperty(key)}") + } + } + + @Test + void testShouldPrintHelpMessage() { + // Arrange + def flags = ["-h", "--help"] + ConfigEncryptionTool tool = new ConfigEncryptionTool() + + // Act + flags.each { String arg -> + def msg = shouldFail(CommandLineParseException) { + tool.parse([arg] as String[]) + } + + // Assert + assert msg == null + assert systemOutRule.getLog().contains("usage: org.apache.nifi.properties.ConfigEncryptionTool [") + } + } + + @Test + void testShouldParseBootstrapConfArgument() { + // Arrange + def flags = ["-b", "--bootstrapConf"] + String bootstrapPath = "src/test/resources/bootstrap.conf" + ConfigEncryptionTool tool = new ConfigEncryptionTool() + + // Act + flags.each { String arg -> + tool.parse([arg, bootstrapPath] as String[]) + logger.info("Parsed bootstrap.conf location: ${tool.bootstrapConfPath}") + + // Assert + assert tool.bootstrapConfPath == bootstrapPath + } + } + + @Test + void testParseShouldPopulateDefaultBootstrapConfArgument() { + // Arrange + String bootstrapPath = "conf/bootstrap.conf" + ConfigEncryptionTool tool = new ConfigEncryptionTool() + + // Act + tool.parse([] as String[]) + logger.info("Parsed bootstrap.conf location: ${tool.bootstrapConfPath}") + + // Assert + assert new File(tool.bootstrapConfPath).getPath() == new File(bootstrapPath).getPath() + } + + @Test + void testShouldParseNiFiPropertiesArgument() { + // Arrange + def flags = ["-n", "--niFiProperties"] + String niFiPropertiesPath = "src/test/resources/nifi.properties" + ConfigEncryptionTool tool = new ConfigEncryptionTool() + + // Act + flags.each { String arg -> + tool.parse([arg, niFiPropertiesPath] as String[]) + logger.info("Parsed nifi.properties location: ${tool.niFiPropertiesPath}") + + // Assert + assert tool.niFiPropertiesPath == niFiPropertiesPath + } + } + + @Test + void testParseShouldPopulateDefaultNiFiPropertiesArgument() { + // Arrange + String niFiPropertiesPath = "conf/nifi.properties" + ConfigEncryptionTool tool = new ConfigEncryptionTool() + + // Act + tool.parse([] as String[]) + logger.info("Parsed nifi.properties location: ${tool.niFiPropertiesPath}") + + // Assert + assert new File(tool.niFiPropertiesPath).getPath() == new File(niFiPropertiesPath).getPath() + } + + @Test + void testShouldParseOutputNiFiPropertiesArgument() { + // Arrange + def flags = ["-o", "--outputNiFiProperties"] + String niFiPropertiesPath = "src/test/resources/nifi.properties" + ConfigEncryptionTool tool = new ConfigEncryptionTool() + + // Act + flags.each { String arg -> + tool.parse([arg, niFiPropertiesPath] as String[]) + logger.info("Parsed output nifi.properties location: ${tool.outputNiFiPropertiesPath}") + + // Assert + assert tool.outputNiFiPropertiesPath == niFiPropertiesPath + } + } + + @Test + void testParseShouldPopulateDefaultOutputNiFiPropertiesArgument() { + // Arrange + String niFiPropertiesPath = "conf/nifi.properties" + ConfigEncryptionTool tool = new ConfigEncryptionTool() + + // Act + tool.parse([] as String[]) + logger.info("Parsed output nifi.properties location: ${tool.outputNiFiPropertiesPath}") + + // Assert + assert new File(tool.outputNiFiPropertiesPath).getPath() == new File(niFiPropertiesPath).getPath() + } + + @Test + void testParseShouldWarnIfNiFiPropertiesWillBeOverwritten() { + // Arrange + String niFiPropertiesPath = "conf/nifi.properties" + ConfigEncryptionTool tool = new ConfigEncryptionTool() + + // Act + tool.parse("-n ${niFiPropertiesPath} -o ${niFiPropertiesPath}".split(" ") as String[]) + logger.info("Parsed nifi.properties location: ${tool.niFiPropertiesPath}") + logger.info("Parsed output nifi.properties location: ${tool.outputNiFiPropertiesPath}") + + // Assert + assert !TestAppender.events.isEmpty() + assert TestAppender.events.first().message =~ "The source nifi.properties and destination nifi.properties are identical \\[.*\\] so the original will be overwritten" + } + + @Test + void testShouldParseKeyArgument() { + // Arrange + def flags = ["-k", "--key"] + ConfigEncryptionTool tool = new ConfigEncryptionTool() + + // Act + flags.each { String arg -> + tool.parse([arg, KEY_HEX] as String[]) + logger.info("Parsed key: ${tool.keyHex}") + + // Assert + assert tool.keyHex == KEY_HEX + } + } + + @Test + void testShouldLoadNiFiProperties() { + // Arrange + String niFiPropertiesPath = "src/test/resources/nifi_with_sensitive_properties_unprotected.properties" + ConfigEncryptionTool tool = new ConfigEncryptionTool() + String[] args = ["-n", niFiPropertiesPath] as String[] + + String oldFilePath = System.getProperty(NiFiProperties.PROPERTIES_FILE_PATH) + + tool.parse(args) + logger.info("Parsed nifi.properties location: ${tool.niFiPropertiesPath}") + + // Act + NiFiProperties properties = tool.loadNiFiProperties() + logger.info("Loaded NiFiProperties from ${tool.niFiPropertiesPath}") + + // Assert + assert properties + assert properties.size() > 0 + + // The system variable was reset to the original value + assert System.getProperty(NiFiProperties.PROPERTIES_FILE_PATH) == oldFilePath + } + + @Test + void testShouldReadKeyFromConsole() { + // Arrange + List keyValues = [ + "0123 4567", + KEY_HEX, + " ${KEY_HEX} ", + "non-hex-chars", + ] + + // Act + keyValues.each { String key -> + TextDevice mockConsoleDevice = TextDevices.streamDevice(new ByteArrayInputStream(key.bytes), new ByteArrayOutputStream()) + String readKey = ConfigEncryptionTool.readKeyFromConsole(mockConsoleDevice) + logger.info("Read key: [${readKey}]") + + // Assert + assert readKey == key + } + } + + @Test + void testShouldReadPasswordFromConsole() { + // Arrange + List passwords = [ + "0123 4567", + PASSWORD, + " ${PASSWORD} ", + "non-hex-chars", + ] + + // Act + passwords.each { String pw -> + logger.info("Using password: [${PASSWORD}]") + TextDevice mockConsoleDevice = TextDevices.streamDevice(new ByteArrayInputStream(pw.bytes), new ByteArrayOutputStream()) + String readPassword = ConfigEncryptionTool.readPasswordFromConsole(mockConsoleDevice) + logger.info("Read password: [${readPassword}]") + + // Assert + assert readPassword == pw + } + } + + @Test + void testShouldReadPasswordFromConsoleIfNoKeyPresent() { + // Arrange + def args = [] as String[] + ConfigEncryptionTool tool = new ConfigEncryptionTool() + tool.parse(args) + logger.info("Using password flag: ${tool.usingPassword}") + logger.info("Password: ${tool.password}") + logger.info("Key hex: ${tool.keyHex}") + + assert tool.usingPassword + assert !tool.password + assert !tool.keyHex + + TextDevice mockConsoleDevice = TextDevices.streamDevice(new ByteArrayInputStream(PASSWORD.bytes), new ByteArrayOutputStream()) + + // Mocked for deterministic output and performance in test -- SCrypt is not under test here + SCrypt.metaClass.'static'.generate = { byte[] pw, byte[] s, int N, int r, int p, int dkLen -> + logger.mock("Mock SCrypt.generate(${Hex.encodeHexString(pw)}, ${Hex.encodeHexString(s)}, ${N}, ${r}, ${p}, ${dkLen})") + logger.mock("Returning ${KEY_HEX[0.. keyValues = [ + (KEY_HEX) : KEY_HEX, + " ${KEY_HEX} " : KEY_HEX, + "xxx${KEY_HEX}zzz" : KEY_HEX, + ((["0123", "4567"] * 4).join("-")): "01234567" * 4, + ((["89ab", "cdef"] * 4).join(" ")): "89ABCDEF" * 4, + (KEY_HEX.toLowerCase()) : KEY_HEX, + (KEY_HEX[0..<32]) : KEY_HEX[0..<32], + ] + + if (isUnlimitedStrengthCryptoAvailable()) { + keyValues.put(KEY_HEX[0..<48], KEY_HEX[0..<48]) + } + + // Act + keyValues.each { String key, final String EXPECTED_KEY -> + logger.info("Reading key: [${key}]") + String parsedKey = ConfigEncryptionTool.parseKey(key) + logger.info("Parsed key: [${parsedKey}]") + + // Assert + assert parsedKey == EXPECTED_KEY + } + } + + @Test + void testParseKeyShouldThrowExceptionForInvalidKeys() { + // Arrange + List keyValues = [ + "0123 4567", + "non-hex-chars", + KEY_HEX[0..<-1], + "&ITD SF^FI&&%SDIF" + ] + + def validKeyLengths = ConfigEncryptionTool.getValidKeyLengths() + def bitLengths = validKeyLengths.collect { it / 4 } + String secondHalf = /\[${validKeyLengths.join(", ")}\] bits / + + /\(\[${bitLengths.join(", ")}\]/ + / hex characters\)/.toString() + + // Act + keyValues.each { String key -> + logger.info("Reading key: [${key}]") + def msg = shouldFail(KeyException) { + String parsedKey = ConfigEncryptionTool.parseKey(key) + logger.info("Parsed key: [${parsedKey}]") + } + logger.expected(msg) + int trimmedKeySize = key.replaceAll("[^0-9a-fA-F]", "").size() + + // Assert + assert msg =~ "The key \\(${trimmedKeySize} hex chars\\) must be of length ${secondHalf}" + } + } + + @Test + void testShouldDeriveKeyFromPassword() { + // Arrange + + // Mocked for deterministic output and performance in test -- SCrypt is not under test here + SCrypt.metaClass.'static'.generate = { byte[] pw, byte[] s, int N, int r, int p, int dkLen -> + logger.mock("Mock SCrypt.generate(${Hex.encodeHexString(pw)}, ${Hex.encodeHexString(s)}, ${N}, ${r}, ${p}, ${dkLen})") + logger.mock("Returning ${KEY_HEX[0.. 128 ? 64 : 32) + } + + @Test + void testDeriveKeyFromPasswordShouldTrimPassword() { + // Arrange + final String PASSWORD_SPACES = " ${PASSWORD} " + + def attemptedPasswords = [] + + // Mocked for deterministic output and performance in test -- SCrypt is not under test here + SCrypt.metaClass.'static'.generate = { byte[] pw, byte[] s, int N, int r, int p, int dkLen -> + logger.mock("Mock SCrypt.generate(${Hex.encodeHexString(pw)}, ${Hex.encodeHexString(s)}, ${N}, ${r}, ${p}, ${dkLen})") + attemptedPasswords << new String(pw) + logger.mock("Returning ${KEY_HEX[0.. + logger.info("Using password: [${password}]") + String derivedKey = ConfigEncryptionTool.deriveKeyFromPassword(password) + logger.info("Derived key: [${derivedKey}]") + } + + // Assert + assert attemptedPasswords.size() == 2 + assert attemptedPasswords.every { it == PASSWORD } + + SCrypt.metaClass.'static' = null + } + + @Test + void testDeriveKeyFromPasswordShouldThrowExceptionForInvalidPasswords() { + // Arrange + List passwords = [ + (null), + "", + " ", + "shortpass", + "shortwith " + ] + + // Act + passwords.each { String password -> + logger.info("Reading password: [${password}]") + def msg = shouldFail(KeyException) { + String derivedKey = ConfigEncryptionTool.deriveKeyFromPassword(password) + logger.info("Derived key: [${derivedKey}]") + } + logger.expected(msg) + + // Assert + assert msg == "Cannot derive key from empty/short password -- password must be at least 12 characters" + } + } + + @Test + void testShouldHandleKeyAndPasswordFlag() { + // Arrange + def args = ["-k", KEY_HEX, "-p", PASSWORD] + logger.info("Using args: ${args}") + + // Act + def msg = shouldFail(CommandLineParseException) { + new ConfigEncryptionTool().parse(args as String[]) + } + logger.expected(msg) + + // Assert + assert msg == "Only one of password and key can be used" + } + + @Test + void testShouldNotLoadMissingNiFiProperties() { + // Arrange + String niFiPropertiesPath = "src/test/resources/non_existent_nifi.properties" + ConfigEncryptionTool tool = new ConfigEncryptionTool() + String[] args = ["-n", niFiPropertiesPath] as String[] + + String oldFilePath = System.getProperty(NiFiProperties.PROPERTIES_FILE_PATH) + + tool.parse(args) + logger.info("Parsed nifi.properties location: ${tool.niFiPropertiesPath}") + + // Act + def msg = shouldFail(CommandLineParseException) { + NiFiProperties properties = tool.loadNiFiProperties() + logger.info("Loaded NiFiProperties from ${tool.niFiPropertiesPath}") + } + + // Assert + assert msg == "Cannot load NiFiProperties from [${niFiPropertiesPath}]".toString() + + // The system variable was reset to the original value + assert System.getProperty(NiFiProperties.PROPERTIES_FILE_PATH) == oldFilePath + } + + @Test + void testLoadNiFiPropertiesShouldHandleReadFailure() { + // Arrange + File inputPropertiesFile = new File("src/test/resources/nifi_with_sensitive_properties_unprotected.properties") + File workingFile = new File("tmp_nifi.properties") + workingFile.delete() + + Files.copy(inputPropertiesFile.toPath(), workingFile.toPath()) + // Empty set of permissions + Files.setPosixFilePermissions(workingFile.toPath(), [] as Set) + logger.info("Set POSIX permissions to ${Files.getPosixFilePermissions(workingFile.toPath())}") + + ConfigEncryptionTool tool = new ConfigEncryptionTool() + String[] args = ["-n", workingFile.path, "-k", KEY_HEX] + tool.parse(args) + + // Act + def msg = shouldFail(IOException) { + tool.loadNiFiProperties() + logger.info("Read nifi.properties") + } + logger.expected(msg) + + // Assert + assert msg == "Cannot load NiFiProperties from [${workingFile.path}]".toString() + + workingFile.deleteOnExit() + } + + @Test + void testShouldEncryptSensitiveProperties() { + // Arrange + String niFiPropertiesPath = "src/test/resources/nifi_with_sensitive_properties_unprotected.properties" + String newPropertiesPath = "src/test/resources/tmp_encrypted_nifi.properties" + ConfigEncryptionTool tool = new ConfigEncryptionTool() + String[] args = ["-n", niFiPropertiesPath, "-o", newPropertiesPath] as String[] + + tool.parse(args) + logger.info("Parsed nifi.properties location: ${tool.niFiPropertiesPath}") + + tool.keyHex = KEY_HEX + + NiFiProperties plainNiFiProperties = tool.loadNiFiProperties() + ProtectedNiFiProperties protectedWrapper = new ProtectedNiFiProperties(plainNiFiProperties) + assert !protectedWrapper.hasProtectedKeys() + + // Act + NiFiProperties encryptedProperties = tool.encryptSensitiveProperties(plainNiFiProperties) + logger.info("Encrypted sensitive properties") + + // Assert + ProtectedNiFiProperties protectedWrapperAroundEncrypted = new ProtectedNiFiProperties(encryptedProperties) + assert protectedWrapperAroundEncrypted.hasProtectedKeys() + + // Ensure that all non-empty sensitive properties are marked as protected + final Set EXPECTED_PROTECTED_KEYS = protectedWrapperAroundEncrypted + .getSensitivePropertyKeys().findAll { String k -> + plainNiFiProperties.getProperty(k) + } as Set + assert protectedWrapperAroundEncrypted.getProtectedPropertyKeys().keySet() == EXPECTED_PROTECTED_KEYS + } + + @Test + void testShouldUpdateBootstrapContentsWithKey() { + // Arrange + final String EXPECTED_KEY_LINE = ConfigEncryptionTool.BOOTSTRAP_KEY_PREFIX + KEY_HEX + + ConfigEncryptionTool tool = new ConfigEncryptionTool() + tool.keyHex = KEY_HEX + + List originalLines = [ + ConfigEncryptionTool.BOOTSTRAP_KEY_COMMENT, + "${ConfigEncryptionTool.BOOTSTRAP_KEY_PREFIX}=" + ] + + // Act + List updatedLines = tool.updateBootstrapContentsWithKey(originalLines) + logger.info("Updated bootstrap.conf lines: ${updatedLines}") + + // Assert + assert updatedLines.size() == originalLines.size() + assert updatedLines.first() == originalLines.first() + assert updatedLines.last() == EXPECTED_KEY_LINE + } + + @Test + void testUpdateBootstrapContentsWithKeyShouldOverwriteExistingKey() { + // Arrange + final String EXPECTED_KEY_LINE = ConfigEncryptionTool.BOOTSTRAP_KEY_PREFIX + KEY_HEX + + ConfigEncryptionTool tool = new ConfigEncryptionTool() + tool.keyHex = KEY_HEX + + List originalLines = [ + ConfigEncryptionTool.BOOTSTRAP_KEY_COMMENT, + "${ConfigEncryptionTool.BOOTSTRAP_KEY_PREFIX}=badKey" + ] + + // Act + List updatedLines = tool.updateBootstrapContentsWithKey(originalLines) + logger.info("Updated bootstrap.conf lines: ${updatedLines}") + + // Assert + assert updatedLines.size() == originalLines.size() + assert updatedLines.first() == originalLines.first() + assert updatedLines.last() == EXPECTED_KEY_LINE + } + + @Test + void testShouldUpdateBootstrapContentsWithKeyAndComment() { + // Arrange + final String EXPECTED_KEY_LINE = ConfigEncryptionTool.BOOTSTRAP_KEY_PREFIX + KEY_HEX + + ConfigEncryptionTool tool = new ConfigEncryptionTool() + tool.keyHex = KEY_HEX + + List originalLines = [ + "${ConfigEncryptionTool.BOOTSTRAP_KEY_PREFIX}=" + ] + + // Act + List updatedLines = tool.updateBootstrapContentsWithKey(originalLines.clone() as List) + logger.info("Updated bootstrap.conf lines: ${updatedLines}") + + // Assert + assert updatedLines.size() == originalLines.size() + 1 + assert updatedLines.first() == ConfigEncryptionTool.BOOTSTRAP_KEY_COMMENT + assert updatedLines.last() == EXPECTED_KEY_LINE + } + + @Test + void testUpdateBootstrapContentsWithKeyShouldAddLines() { + // Arrange + final String EXPECTED_KEY_LINE = ConfigEncryptionTool.BOOTSTRAP_KEY_PREFIX + KEY_HEX + + ConfigEncryptionTool tool = new ConfigEncryptionTool() + tool.keyHex = KEY_HEX + + List originalLines = [] + + // Act + List updatedLines = tool.updateBootstrapContentsWithKey(originalLines.clone() as List) + logger.info("Updated bootstrap.conf lines: ${updatedLines}") + + // Assert + assert updatedLines.size() == originalLines.size() + 3 + assert updatedLines.first() == "\n" + assert updatedLines[1] == ConfigEncryptionTool.BOOTSTRAP_KEY_COMMENT + assert updatedLines.last() == EXPECTED_KEY_LINE + } + + @Test + void testShouldWriteKeyToBootstrapConf() { + // Arrange + File emptyKeyFile = new File("src/test/resources/bootstrap_with_empty_master_key.conf") + File workingFile = new File("tmp_bootstrap.conf") + workingFile.delete() + + Files.copy(emptyKeyFile.toPath(), workingFile.toPath()) + final List originalLines = workingFile.readLines() + String originalKeyLine = originalLines.find { it.startsWith(ConfigEncryptionTool.BOOTSTRAP_KEY_PREFIX) } + logger.info("Original key line from bootstrap.conf: ${originalKeyLine}") + + final String EXPECTED_KEY_LINE = ConfigEncryptionTool.BOOTSTRAP_KEY_PREFIX + KEY_HEX + + ConfigEncryptionTool tool = new ConfigEncryptionTool() + String[] args = ["-b", workingFile.path, "-k", KEY_HEX] + tool.parse(args) + + // Act + tool.writeKeyToBootstrapConf() + logger.info("Updated bootstrap.conf") + + // Assert + final List updatedLines = workingFile.readLines() + String updatedLine = updatedLines.find { it.startsWith(ConfigEncryptionTool.BOOTSTRAP_KEY_PREFIX) } + logger.info("Updated key line: ${updatedLine}") + + assert updatedLine == EXPECTED_KEY_LINE + assert originalLines.size() == updatedLines.size() + + workingFile.deleteOnExit() + } + + @Test + void testWriteKeyToBootstrapConfShouldHandleReadFailure() { + // Arrange + File emptyKeyFile = new File("src/test/resources/bootstrap_with_empty_master_key.conf") + File workingFile = new File("tmp_bootstrap.conf") + workingFile.delete() + + Files.copy(emptyKeyFile.toPath(), workingFile.toPath()) + // Empty set of permissions + Files.setPosixFilePermissions(workingFile.toPath(), [] as Set) + logger.info("Set POSIX permissions to ${Files.getPosixFilePermissions(workingFile.toPath())}") + + ConfigEncryptionTool tool = new ConfigEncryptionTool() + String[] args = ["-b", workingFile.path, "-k", KEY_HEX] + tool.parse(args) + + // Act + def msg = shouldFail(IOException) { + tool.writeKeyToBootstrapConf() + logger.info("Updated bootstrap.conf") + } + logger.expected(msg) + + // Assert + assert msg == "The bootstrap.conf file at tmp_bootstrap.conf must exist and be readable and writable by the user running this tool" + + workingFile.deleteOnExit() + } + + @Test + void testWriteKeyToBootstrapConfShouldHandleWriteFailure() { + // Arrange + File emptyKeyFile = new File("src/test/resources/bootstrap_with_empty_master_key.conf") + File workingFile = new File("tmp_bootstrap.conf") + workingFile.delete() + + Files.copy(emptyKeyFile.toPath(), workingFile.toPath()) + // Read-only set of permissions + Files.setPosixFilePermissions(workingFile.toPath(), [PosixFilePermission.OWNER_READ, PosixFilePermission.GROUP_READ, PosixFilePermission.OTHERS_READ] as Set) + logger.info("Set POSIX permissions to ${Files.getPosixFilePermissions(workingFile.toPath())}") + + ConfigEncryptionTool tool = new ConfigEncryptionTool() + String[] args = ["-b", workingFile.path, "-k", KEY_HEX] + tool.parse(args) + + // Act + def msg = shouldFail(IOException) { + tool.writeKeyToBootstrapConf() + logger.info("Updated bootstrap.conf") + } + logger.expected(msg) + + // Assert + assert msg == "The bootstrap.conf file at tmp_bootstrap.conf must exist and be readable and writable by the user running this tool" + + workingFile.deleteOnExit() + } + + @Test + void testShouldEncryptNiFiPropertiesWithEmptyProtectionScheme() { + // Arrange + String originalNiFiPropertiesPath = "src/test/resources/nifi_with_sensitive_properties_unprotected_and_empty_protection_schemes.properties" + + File originalFile = new File(originalNiFiPropertiesPath) + List originalLines = originalFile.readLines() + logger.info("Read ${originalLines.size()} lines from ${originalNiFiPropertiesPath}") + logger.info("\n" + originalLines[0..3].join("\n") + "...") + + NiFiProperties plainProperties = NiFiPropertiesLoader.withKey(KEY_HEX).load(originalNiFiPropertiesPath) + logger.info("Loaded NiFiProperties from ${originalNiFiPropertiesPath}") + + ProtectedNiFiProperties protectedWrapper = new ProtectedNiFiProperties(plainProperties) + logger.info("Loaded ${plainProperties.size()} properties") + logger.info("There are ${protectedWrapper.getSensitivePropertyKeys().size()} sensitive properties") + + SensitivePropertyProvider spp = new AESSensitivePropertyProvider(KEY_HEX) + int protectedPropertyCount = protectedWrapper.protectedPropertyKeys.size() + logger.info("Counted ${protectedPropertyCount} protected keys") + assert protectedPropertyCount < protectedWrapper.getSensitivePropertyKeys().size() + + ConfigEncryptionTool tool = new ConfigEncryptionTool(keyHex: KEY_HEX) + + // Act + NiFiProperties encryptedProperties = tool.encryptSensitiveProperties(plainProperties) + + // Assert + ProtectedNiFiProperties encryptedWrapper = new ProtectedNiFiProperties(encryptedProperties) + encryptedWrapper.getProtectedPropertyKeys().every { String key, String protectionScheme -> + logger.info("${key} is protected by ${protectionScheme}") + assert protectionScheme == spp.identifierKey + } + + printProperties(encryptedWrapper) + + assert encryptedWrapper.getProtectedPropertyKeys().size() == encryptedWrapper.getSensitivePropertyKeys().findAll { + encryptedWrapper.getProperty(it) + }.size() + } + + @Test + void testShouldSerializeNiFiProperties() { + // Arrange + Properties rawProperties = [key: "value", key2: "value2"] as Properties + NiFiProperties properties = new StandardNiFiProperties(rawProperties) + logger.info("Loaded ${properties.size()} properties") + + // Act + List lines = ConfigEncryptionTool.serializeNiFiProperties(properties) + logger.info("Serialized NiFiProperties to ${lines.size()} lines") + logger.info("\n" + lines.join("\n")) + + // Assert + + // The serialization could have occurred > 1 second ago, causing a rolling date/time mismatch, so use regex + // Format -- #Fri Aug 19 16:51:16 PDT 2016 + String datePattern = /#\w{3} \w{3} \d{2} \d{2}:\d{2}:\d{2} \w{3} \d{4}/ + + // One extra line for the date + assert lines.size() == properties.size() + 1 + assert lines.first() =~ datePattern + + rawProperties.keySet().every { String key -> + assert lines.contains("${key}=${properties.getProperty(key)}".toString()) + } + } + + @Test + void testShouldSerializeNiFiPropertiesAndPreserveFormat() { + // Arrange + String originalNiFiPropertiesPath = "src/test/resources/nifi_with_few_sensitive_properties_unprotected.properties" + + File originalFile = new File(originalNiFiPropertiesPath) + List originalLines = originalFile.readLines() + logger.info("Read ${originalLines.size()} lines from ${originalNiFiPropertiesPath}") + logger.info("\n" + originalLines[0..3].join("\n") + "...") + + NiFiProperties plainProperties = NiFiPropertiesLoader.withKey(KEY_HEX).load(originalNiFiPropertiesPath) + logger.info("Loaded NiFiProperties from ${originalNiFiPropertiesPath}") + + ProtectedNiFiProperties protectedWrapper = new ProtectedNiFiProperties(plainProperties) + logger.info("Loaded ${plainProperties.size()} properties") + logger.info("There are ${protectedWrapper.getSensitivePropertyKeys().size()} sensitive properties") + + protectedWrapper.addSensitivePropertyProvider(new AESSensitivePropertyProvider(KEY_HEX)) + NiFiProperties protectedProperties = protectedWrapper.protectPlainProperties() + int protectedPropertyCount = ProtectedNiFiProperties.countProtectedProperties(protectedProperties) + logger.info("Counted ${protectedPropertyCount} protected keys") + + // Act + List lines = ConfigEncryptionTool.serializeNiFiPropertiesAndPreserveFormat(protectedProperties, originalFile) + logger.info("Serialized NiFiProperties to ${lines.size()} lines") + lines.eachWithIndex { String entry, int i -> + logger.debug("${(i + 1).toString().padLeft(3)}: ${entry}") + } + + // Assert + + // Added n new lines for the encrypted properties + assert lines.size() == originalLines.size() + protectedPropertyCount + + protectedProperties.getPropertyKeys().every { String key -> + assert lines.contains("${key}=${protectedProperties.getProperty(key)}".toString()) + } + + logger.info("Updated nifi.properties:") + logger.info("\n" * 2 + lines.join("\n")) + } + + @Test + void testShouldSerializeNiFiPropertiesAndPreserveFormatWithExistingProtectionSchemes() { + // Arrange + String originalNiFiPropertiesPath = "src/test/resources/nifi_with_few_sensitive_properties_protected_aes.properties" + + File originalFile = new File(originalNiFiPropertiesPath) + List originalLines = originalFile.readLines() + logger.info("Read ${originalLines.size()} lines from ${originalNiFiPropertiesPath}") + logger.info("\n" + originalLines[0..3].join("\n") + "...") + + ProtectedNiFiProperties protectedProperties = NiFiPropertiesLoader.withKey(KEY_HEX).readProtectedPropertiesFromDisk(new File(originalNiFiPropertiesPath)) + logger.info("Loaded NiFiProperties from ${originalNiFiPropertiesPath}") + + logger.info("Loaded ${protectedProperties.getPropertyKeys().size()} properties") + logger.info("There are ${protectedProperties.getSensitivePropertyKeys().size()} sensitive properties") + logger.info("There are ${protectedProperties.getProtectedPropertyKeys().size()} protected properties") + int originalProtectedPropertyCount = protectedProperties.getProtectedPropertyKeys().size() + + protectedProperties.addSensitivePropertyProvider(new AESSensitivePropertyProvider(KEY_HEX)) + NiFiProperties encryptedProperties = protectedProperties.protectPlainProperties() + int protectedPropertyCount = ProtectedNiFiProperties.countProtectedProperties(encryptedProperties) + logger.info("Counted ${protectedPropertyCount} protected keys") + + int protectedCountChange = protectedPropertyCount - originalProtectedPropertyCount + logger.info("Expected line count change: ${protectedCountChange}") + + // Act + List lines = ConfigEncryptionTool.serializeNiFiPropertiesAndPreserveFormat(protectedProperties, originalFile) + logger.info("Serialized NiFiProperties to ${lines.size()} lines") + lines.eachWithIndex { String entry, int i -> + logger.debug("${(i + 1).toString().padLeft(3)}: ${entry}") + } + + // Assert + + // Added n new lines for the encrypted properties + assert lines.size() == originalLines.size() + protectedCountChange + + protectedProperties.getPropertyKeys().every { String key -> + assert lines.contains("${key}=${protectedProperties.getProperty(key)}".toString()) + } + + logger.info("Updated nifi.properties:") + logger.info("\n" * 2 + lines.join("\n")) + } + + @Test + void testShouldSerializeNiFiPropertiesAndPreserveFormatWithNewPropertyAtEOF() { + // Arrange + String originalNiFiPropertiesPath = "src/test/resources/nifi_with_few_sensitive_properties_unprotected.properties" + + File originalFile = new File(originalNiFiPropertiesPath) + List originalLines = originalFile.readLines() + logger.info("Read ${originalLines.size()} lines from ${originalNiFiPropertiesPath}") + logger.info("\n" + originalLines[0..3].join("\n") + "...") + + NiFiProperties plainProperties = NiFiPropertiesLoader.withKey(KEY_HEX).load(originalNiFiPropertiesPath) + logger.info("Loaded NiFiProperties from ${originalNiFiPropertiesPath}") + + ProtectedNiFiProperties protectedWrapper = new ProtectedNiFiProperties(plainProperties) + logger.info("Loaded ${plainProperties.size()} properties") + logger.info("There are ${protectedWrapper.getSensitivePropertyKeys().size()} sensitive properties") + + // Set a value for the sensitive property which is the last line in the file + + // Groovy access to avoid duplicating entire object to add one value + (plainProperties as StandardNiFiProperties).@rawProperties.setProperty(NiFiProperties.SECURITY_TRUSTSTORE_PASSWD, "thisIsABadTruststorePassword") + + protectedWrapper.addSensitivePropertyProvider(new AESSensitivePropertyProvider(KEY_HEX)) + NiFiProperties protectedProperties = protectedWrapper.protectPlainProperties() + int protectedPropertyCount = ProtectedNiFiProperties.countProtectedProperties(protectedProperties) + logger.info("Counted ${protectedPropertyCount} protected keys") + + // Act + List lines = ConfigEncryptionTool.serializeNiFiPropertiesAndPreserveFormat(protectedProperties, originalFile) + logger.info("Serialized NiFiProperties to ${lines.size()} lines") + lines.eachWithIndex { String entry, int i -> + logger.debug("${(i + 1).toString().padLeft(3)}: ${entry}") + } + + // Assert + + // Added n new lines for the encrypted properties + assert lines.size() == originalLines.size() + protectedPropertyCount + + protectedProperties.getPropertyKeys().every { String key -> + assert lines.contains("${key}=${protectedProperties.getProperty(key)}".toString()) + } + + logger.info("Updated nifi.properties:") + logger.info("\n" * 2 + lines.join("\n")) + } + + @Test + void testShouldWriteNiFiProperties() { + // Arrange + File inputPropertiesFile = new File("src/test/resources/nifi_with_sensitive_properties_unprotected.properties") + File workingFile = new File("tmp_nifi.properties") + workingFile.delete() + + final List originalLines = inputPropertiesFile.readLines() + + ConfigEncryptionTool tool = new ConfigEncryptionTool() + String[] args = ["-n", inputPropertiesFile.path, "-o", workingFile.path, "-k", KEY_HEX] + tool.parse(args) + NiFiProperties niFiProperties = tool.loadNiFiProperties() + tool.@niFiProperties = niFiProperties + logger.info("Loaded ${niFiProperties.size()} properties from ${inputPropertiesFile.path}") + + // Act + tool.writeNiFiProperties() + logger.info("Wrote to ${workingFile.path}") + + // Assert + final List updatedLines = workingFile.readLines() + niFiProperties.getPropertyKeys().every { String key -> + assert updatedLines.contains("${key}=${niFiProperties.getProperty(key)}".toString()) + } + + assert originalLines == updatedLines + + logger.info("Updated nifi.properties:") + logger.info("\n" * 2 + updatedLines.join("\n")) + + workingFile.deleteOnExit() + } + + @Test + void testShouldWriteNiFiPropertiesInSameLocation() { + // Arrange + File inputPropertiesFile = new File("src/test/resources/nifi_with_sensitive_properties_unprotected.properties") + File workingFile = new File("tmp_nifi.properties") + workingFile.delete() + Files.copy(inputPropertiesFile.toPath(), workingFile.toPath()) + + final List originalLines = inputPropertiesFile.readLines() + + ConfigEncryptionTool tool = new ConfigEncryptionTool() + String[] args = ["-n", workingFile.path, "-k", KEY_HEX] + tool.parse(args) + NiFiProperties niFiProperties = tool.loadNiFiProperties() + tool.@niFiProperties = niFiProperties + logger.info("Loaded ${niFiProperties.size()} properties from ${workingFile.path}") + + // Act + tool.writeNiFiProperties() + logger.info("Wrote to ${workingFile.path}") + + // Assert + final List updatedLines = workingFile.readLines() + niFiProperties.getPropertyKeys().every { String key -> + assert updatedLines.contains("${key}=${niFiProperties.getProperty(key)}".toString()) + } + + assert originalLines == updatedLines + + logger.info("Updated nifi.properties:") + logger.info("\n" * 2 + updatedLines.join("\n")) + + assert TestAppender.events.collect { + it.message + }.contains("The source nifi.properties and destination nifi.properties are identical [${workingFile.path}] so the original will be overwritten".toString()) + + workingFile.deleteOnExit() + } + + @Test + void testWriteNiFiPropertiesShouldHandleWriteFailureWhenFileExists() { + // Arrange + File inputPropertiesFile = new File("src/test/resources/nifi_with_sensitive_properties_unprotected.properties") + File workingFile = new File("tmp_nifi.properties") + workingFile.delete() + + Files.copy(inputPropertiesFile.toPath(), workingFile.toPath()) + // Read-only set of permissions + Files.setPosixFilePermissions(workingFile.toPath(), [PosixFilePermission.OWNER_READ, PosixFilePermission.GROUP_READ, PosixFilePermission.OTHERS_READ] as Set) + logger.info("Set POSIX permissions to ${Files.getPosixFilePermissions(workingFile.toPath())}") + + ConfigEncryptionTool tool = new ConfigEncryptionTool() + String[] args = ["-n", inputPropertiesFile.path, "-o", workingFile.path, "-k", KEY_HEX] + tool.parse(args) + NiFiProperties niFiProperties = tool.loadNiFiProperties() + tool.@niFiProperties = niFiProperties + logger.info("Loaded ${niFiProperties.size()} properties from ${inputPropertiesFile.path}") + + // Act + def msg = shouldFail(IOException) { + tool.writeNiFiProperties() + logger.info("Wrote to ${workingFile.path}") + } + logger.expected(msg) + + // Assert + assert msg == "The nifi.properties file at ${workingFile.path} must be writable by the user running this tool".toString() + + workingFile.deleteOnExit() + } + + @Test + void testWriteNiFiPropertiesShouldHandleWriteFailureWhenFileDoesNotExist() { + // Arrange + File inputPropertiesFile = new File("src/test/resources/nifi_with_sensitive_properties_unprotected.properties") + File tmpDir = new File("target/tmp/") + tmpDir.mkdirs() + File workingFile = new File("target/tmp/tmp_nifi.properties") + workingFile.delete() + + // Read-only set of permissions + Files.setPosixFilePermissions(tmpDir.toPath(), [PosixFilePermission.OWNER_READ, PosixFilePermission.GROUP_READ, PosixFilePermission.OTHERS_READ] as Set) + logger.info("Set POSIX permissions on parent directory to ${Files.getPosixFilePermissions(tmpDir.toPath())}") + + ConfigEncryptionTool tool = new ConfigEncryptionTool() + String[] args = ["-n", inputPropertiesFile.path, "-o", workingFile.path, "-k", KEY_HEX] + tool.parse(args) + NiFiProperties niFiProperties = tool.loadNiFiProperties() + tool.@niFiProperties = niFiProperties + logger.info("Loaded ${niFiProperties.size()} properties from ${inputPropertiesFile.path}") + + // Act + def msg = shouldFail(IOException) { + tool.writeNiFiProperties() + logger.info("Wrote to ${workingFile.path}") + } + logger.expected(msg) + + // Assert + assert msg == "The nifi.properties file at ${workingFile.path} must be writable by the user running this tool".toString() + + workingFile.deleteOnExit() + Files.setPosixFilePermissions(tmpDir.toPath(), [PosixFilePermission.OWNER_READ, PosixFilePermission.OWNER_WRITE] as Set) + tmpDir.deleteOnExit() + } + + @Test + void testShouldPerformFullOperation() { + // Arrange + exit.expectSystemExitWithStatus(0) + + File tmpDir = new File("target/tmp/") + tmpDir.mkdirs() + Files.setPosixFilePermissions(tmpDir.toPath(), [PosixFilePermission.OWNER_READ, PosixFilePermission.OWNER_WRITE, PosixFilePermission.OWNER_EXECUTE, PosixFilePermission.GROUP_READ, PosixFilePermission.GROUP_WRITE, PosixFilePermission.GROUP_EXECUTE, PosixFilePermission.OTHERS_READ, PosixFilePermission.OTHERS_WRITE, PosixFilePermission.OTHERS_EXECUTE] as Set) + + File emptyKeyFile = new File("src/test/resources/bootstrap_with_empty_master_key.conf") + File bootstrapFile = new File("target/tmp/tmp_bootstrap.conf") + bootstrapFile.delete() + + Files.copy(emptyKeyFile.toPath(), bootstrapFile.toPath()) + final List originalBootstrapLines = bootstrapFile.readLines() + String originalKeyLine = originalBootstrapLines.find { + it.startsWith(ConfigEncryptionTool.BOOTSTRAP_KEY_PREFIX) + } + logger.info("Original key line from bootstrap.conf: ${originalKeyLine}") + assert originalKeyLine == ConfigEncryptionTool.BOOTSTRAP_KEY_PREFIX + + final String EXPECTED_KEY_LINE = ConfigEncryptionTool.BOOTSTRAP_KEY_PREFIX + KEY_HEX + + File inputPropertiesFile = new File("src/test/resources/nifi_with_sensitive_properties_unprotected.properties") + File outputPropertiesFile = new File("target/tmp/tmp_nifi.properties") + outputPropertiesFile.delete() + + NiFiProperties inputProperties = new NiFiPropertiesLoader().load(inputPropertiesFile) + logger.info("Loaded ${inputProperties.size()} properties from input file") + ProtectedNiFiProperties protectedInputProperties = new ProtectedNiFiProperties(inputProperties) + def originalSensitiveValues = protectedInputProperties.getSensitivePropertyKeys().collectEntries { String key -> [(key): protectedInputProperties.getProperty(key)] } + logger.info("Original sensitive values: ${originalSensitiveValues}") + + String[] args = ["-n", inputPropertiesFile.path, "-b", bootstrapFile.path, "-o", outputPropertiesFile.path, "-k", KEY_HEX] + + exit.checkAssertionAfterwards(new Assertion() { + public void checkAssertion() { + final List updatedPropertiesLines = outputPropertiesFile.readLines() + logger.info("Updated nifi.properties:") + logger.info("\n" * 2 + updatedPropertiesLines.join("\n")) + + // Check that the output values for sensitive properties are not the same as the original (i.e. it was encrypted) + NiFiProperties updatedProperties = new NiFiPropertiesLoader().readProtectedPropertiesFromDisk(outputPropertiesFile) + assert updatedProperties.size() >= inputProperties.size() + originalSensitiveValues.every { String key, String originalValue -> + assert updatedProperties.getProperty(key) != originalValue + } + + // Check that the new NiFiProperties instance matches the output file (values still encrypted) + updatedProperties.getPropertyKeys().every { String key -> + assert updatedPropertiesLines.contains("${key}=${updatedProperties.getProperty(key)}".toString()) + } + + // Check that the key was persisted to the bootstrap.conf + final List updatedBootstrapLines = bootstrapFile.readLines() + String updatedKeyLine = updatedBootstrapLines.find { + it.startsWith(ConfigEncryptionTool.BOOTSTRAP_KEY_PREFIX) + } + logger.info("Updated key line: ${updatedKeyLine}") + + assert updatedKeyLine == EXPECTED_KEY_LINE + assert originalBootstrapLines.size() == updatedBootstrapLines.size() + + // Clean up + outputPropertiesFile.deleteOnExit() + bootstrapFile.deleteOnExit() + tmpDir.deleteOnExit() + } + }); + + // Act + ConfigEncryptionTool.main(args) + logger.info("Invoked #main with ${args.join(" ")}") + + // Assert + + // Assertions defined above + } + + @Test + void testShouldPerformFullOperationWithPassword() { + // Arrange + exit.expectSystemExitWithStatus(0) + + File tmpDir = new File("target/tmp/") + tmpDir.mkdirs() + Files.setPosixFilePermissions(tmpDir.toPath(), [PosixFilePermission.OWNER_READ, PosixFilePermission.OWNER_WRITE, PosixFilePermission.OWNER_EXECUTE, PosixFilePermission.GROUP_READ, PosixFilePermission.GROUP_WRITE, PosixFilePermission.GROUP_EXECUTE, PosixFilePermission.OTHERS_READ, PosixFilePermission.OTHERS_WRITE, PosixFilePermission.OTHERS_EXECUTE] as Set) + + File emptyKeyFile = new File("src/test/resources/bootstrap_with_empty_master_key.conf") + File bootstrapFile = new File("target/tmp/tmp_bootstrap.conf") + bootstrapFile.delete() + + Files.copy(emptyKeyFile.toPath(), bootstrapFile.toPath()) + final List originalBootstrapLines = bootstrapFile.readLines() + String originalKeyLine = originalBootstrapLines.find { + it.startsWith(ConfigEncryptionTool.BOOTSTRAP_KEY_PREFIX) + } + logger.info("Original key line from bootstrap.conf: ${originalKeyLine}") + assert originalKeyLine == ConfigEncryptionTool.BOOTSTRAP_KEY_PREFIX + + final String EXPECTED_KEY_HEX = ConfigEncryptionTool.deriveKeyFromPassword(PASSWORD) + logger.info("Derived key from password [${PASSWORD}]: ${EXPECTED_KEY_HEX}") + + final String EXPECTED_KEY_LINE = ConfigEncryptionTool.BOOTSTRAP_KEY_PREFIX + EXPECTED_KEY_HEX + + File inputPropertiesFile = new File("src/test/resources/nifi_with_sensitive_properties_unprotected.properties") + File outputPropertiesFile = new File("target/tmp/tmp_nifi.properties") + outputPropertiesFile.delete() + + NiFiProperties inputProperties = new NiFiPropertiesLoader().load(inputPropertiesFile) + logger.info("Loaded ${inputProperties.size()} properties from input file") + ProtectedNiFiProperties protectedInputProperties = new ProtectedNiFiProperties(inputProperties) + def originalSensitiveValues = protectedInputProperties.getSensitivePropertyKeys().collectEntries { String key -> [(key): protectedInputProperties.getProperty(key)] } + logger.info("Original sensitive values: ${originalSensitiveValues}") + + String[] args = ["-n", inputPropertiesFile.path, "-b", bootstrapFile.path, "-o", outputPropertiesFile.path, "-p", PASSWORD] + + exit.checkAssertionAfterwards(new Assertion() { + public void checkAssertion() { + final List updatedPropertiesLines = outputPropertiesFile.readLines() + logger.info("Updated nifi.properties:") + logger.info("\n" * 2 + updatedPropertiesLines.join("\n")) + + // Check that the output values for sensitive properties are not the same as the original (i.e. it was encrypted) + NiFiProperties updatedProperties = new NiFiPropertiesLoader().readProtectedPropertiesFromDisk(outputPropertiesFile) + assert updatedProperties.size() >= inputProperties.size() + originalSensitiveValues.every { String key, String originalValue -> + assert updatedProperties.getProperty(key) != originalValue + } + + // Check that the new NiFiProperties instance matches the output file (values still encrypted) + updatedProperties.getPropertyKeys().every { String key -> + assert updatedPropertiesLines.contains("${key}=${updatedProperties.getProperty(key)}".toString()) + } + + // Check that the key was persisted to the bootstrap.conf + final List updatedBootstrapLines = bootstrapFile.readLines() + String updatedKeyLine = updatedBootstrapLines.find { + it.startsWith(ConfigEncryptionTool.BOOTSTRAP_KEY_PREFIX) + } + logger.info("Updated key line: ${updatedKeyLine}") + + assert updatedKeyLine == EXPECTED_KEY_LINE + assert originalBootstrapLines.size() == updatedBootstrapLines.size() + + // Clean up + outputPropertiesFile.deleteOnExit() + bootstrapFile.deleteOnExit() + tmpDir.deleteOnExit() + } + }); + + // Act + ConfigEncryptionTool.main(args) + logger.info("Invoked #main with ${args.join(" ")}") + + // Assert + + // Assertions defined above + } + + @Test + void testShouldPerformFullOperationMultipleTimes() { + // Arrange + exit.expectSystemExitWithStatus(0) + + File tmpDir = new File("target/tmp/") + tmpDir.mkdirs() + Files.setPosixFilePermissions(tmpDir.toPath(), [PosixFilePermission.OWNER_READ, PosixFilePermission.OWNER_WRITE, PosixFilePermission.OWNER_EXECUTE, PosixFilePermission.GROUP_READ, PosixFilePermission.GROUP_WRITE, PosixFilePermission.GROUP_EXECUTE, PosixFilePermission.OTHERS_READ, PosixFilePermission.OTHERS_WRITE, PosixFilePermission.OTHERS_EXECUTE] as Set) + + File emptyKeyFile = new File("src/test/resources/bootstrap_with_empty_master_key.conf") + File bootstrapFile = new File("target/tmp/tmp_bootstrap.conf") + bootstrapFile.delete() + + Files.copy(emptyKeyFile.toPath(), bootstrapFile.toPath()) + final List originalBootstrapLines = bootstrapFile.readLines() + String originalKeyLine = originalBootstrapLines.find { + it.startsWith(ConfigEncryptionTool.BOOTSTRAP_KEY_PREFIX) + } + logger.info("Original key line from bootstrap.conf: ${originalKeyLine}") + assert originalKeyLine == ConfigEncryptionTool.BOOTSTRAP_KEY_PREFIX + + final String EXPECTED_KEY_HEX = ConfigEncryptionTool.deriveKeyFromPassword(PASSWORD) + logger.info("Derived key from password [${PASSWORD}]: ${EXPECTED_KEY_HEX}") + + final String EXPECTED_KEY_LINE = ConfigEncryptionTool.BOOTSTRAP_KEY_PREFIX + EXPECTED_KEY_HEX + + File inputPropertiesFile = new File("src/test/resources/nifi_with_sensitive_properties_unprotected.properties") + File outputPropertiesFile = new File("target/tmp/tmp_nifi.properties") + outputPropertiesFile.delete() + + NiFiProperties inputProperties = new NiFiPropertiesLoader().load(inputPropertiesFile) + logger.info("Loaded ${inputProperties.size()} properties from input file") + ProtectedNiFiProperties protectedInputProperties = new ProtectedNiFiProperties(inputProperties) + def originalSensitiveValues = protectedInputProperties.getSensitivePropertyKeys().collectEntries { String key -> [(key): protectedInputProperties.getProperty(key)] } + logger.info("Original sensitive values: ${originalSensitiveValues}") + + String[] args = ["-n", inputPropertiesFile.path, "-b", bootstrapFile.path, "-o", outputPropertiesFile.path, "-p", PASSWORD, "-v"] + + def msg = shouldFail { + logger.info("Invoked #main first time with ${args.join(" ")}") + ConfigEncryptionTool.main(args) + } + logger.expected(msg) + + // Act + args = ["-n", outputPropertiesFile.path, "-b", bootstrapFile.path, "-p", PASSWORD, "-v"] + + // Add a new property to be encrypted + outputPropertiesFile.text = outputPropertiesFile.text.replace("nifi.sensitive.props.additional.keys=", "nifi.sensitive.props.additional.keys=nifi.ui.banner.text") + + exit.checkAssertionAfterwards(new Assertion() { + public void checkAssertion() { + final List updatedPropertiesLines = outputPropertiesFile.readLines() + logger.info("Updated nifi.properties:") + logger.info("\n" * 2 + updatedPropertiesLines.join("\n")) + + // Check that the output values for sensitive properties are not the same as the original (i.e. it was encrypted) + NiFiProperties updatedProperties = new NiFiPropertiesLoader().readProtectedPropertiesFromDisk(outputPropertiesFile) + assert updatedProperties.size() >= inputProperties.size() + originalSensitiveValues.every { String key, String originalValue -> + assert updatedProperties.getProperty(key) != originalValue + } + + // Check that the new NiFiProperties instance matches the output file (values still encrypted) + updatedProperties.getPropertyKeys().every { String key -> + assert updatedPropertiesLines.contains("${key}=${updatedProperties.getProperty(key)}".toString()) + } + + // Check that the key was persisted to the bootstrap.conf + final List updatedBootstrapLines = bootstrapFile.readLines() + String updatedKeyLine = updatedBootstrapLines.find { + it.startsWith(ConfigEncryptionTool.BOOTSTRAP_KEY_PREFIX) + } + logger.info("Updated key line: ${updatedKeyLine}") + + assert updatedKeyLine == EXPECTED_KEY_LINE + assert originalBootstrapLines.size() == updatedBootstrapLines.size() + + // Clean up + outputPropertiesFile.deleteOnExit() + bootstrapFile.deleteOnExit() + tmpDir.deleteOnExit() + } + }); + + logger.info("Invoked #main second time with ${args.join(" ")}") + ConfigEncryptionTool.main(args) + + // Assert + + // Assertions defined above + } +} + +public class TestAppender extends AppenderBase { + static List events = new ArrayList<>(); + + @Override + protected void append(LoggingEvent e) { + synchronized (events) { + events.add(e); + } + } + + public static void reset() { + synchronized (events) { + events.clear(); + } + } +} \ No newline at end of file diff --git a/nifi-toolkit/nifi-toolkit-encrypt-config/src/test/resources/bootstrap.conf b/nifi-toolkit/nifi-toolkit-encrypt-config/src/test/resources/bootstrap.conf new file mode 100644 index 0000000000..c5bd663984 --- /dev/null +++ b/nifi-toolkit/nifi-toolkit-encrypt-config/src/test/resources/bootstrap.conf @@ -0,0 +1,72 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# Java command to use when running NiFi +java=java + +# Username to use when running NiFi. This value will be ignored on Windows. +run.as= + +# Configure where NiFi's lib and conf directories live +lib.dir=./lib +conf.dir=./conf + +# How long to wait after telling NiFi to shutdown before explicitly killing the Process +graceful.shutdown.seconds=20 + +# Disable JSR 199 so that we can use JSP's without running a JDK +java.arg.1=-Dorg.apache.jasper.compiler.disablejsr199=true + +# JVM memory settings +java.arg.2=-Xms512m +java.arg.3=-Xmx512m + +# Enable Remote Debugging +#java.arg.debug=-agentlib:jdwp=transport=dt_socket,server=y,suspend=n,address=8000 + +java.arg.4=-Djava.net.preferIPv4Stack=true + +# allowRestrictedHeaders is required for Cluster/Node communications to work properly +java.arg.5=-Dsun.net.http.allowRestrictedHeaders=true +java.arg.6=-Djava.protocol.handler.pkgs=sun.net.www.protocol + +# The G1GC is still considered experimental but has proven to be very advantageous in providing great +# performance without significant "stop-the-world" delays. +java.arg.13=-XX:+UseG1GC + +#Set headless mode by default +java.arg.14=-Djava.awt.headless=true + + +### +# Notification Services for notifying interested parties when NiFi is stopped, started, dies +### + +# XML File that contains the definitions of the notification services +notification.services.file=./conf/bootstrap-notification-services.xml + +# In the case that we are unable to send a notification for an event, how many times should we retry? +notification.max.attempts=5 + +# Comma-separated list of identifiers that are present in the notification.services.file; which services should be used to notify when NiFi is started? +#nifi.start.notification.services=email-notification + +# Comma-separated list of identifiers that are present in the notification.services.file; which services should be used to notify when NiFi is stopped? +#nifi.stop.notification.services=email-notification + +# Comma-separated list of identifiers that are present in the notification.services.file; which services should be used to notify when NiFi dies? +#nifi.dead.notification.services=email-notification \ No newline at end of file diff --git a/nifi-toolkit/nifi-toolkit-encrypt-config/src/test/resources/bootstrap_with_empty_master_key.conf b/nifi-toolkit/nifi-toolkit-encrypt-config/src/test/resources/bootstrap_with_empty_master_key.conf new file mode 100644 index 0000000000..17acc6202d --- /dev/null +++ b/nifi-toolkit/nifi-toolkit-encrypt-config/src/test/resources/bootstrap_with_empty_master_key.conf @@ -0,0 +1,74 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# Java command to use when running NiFi +java=java + +# Username to use when running NiFi. This value will be ignored on Windows. +run.as= + +# Configure where NiFi's lib and conf directories live +lib.dir=./lib +conf.dir=./conf + +# How long to wait after telling NiFi to shutdown before explicitly killing the Process +graceful.shutdown.seconds=20 + +# Disable JSR 199 so that we can use JSP's without running a JDK +java.arg.1=-Dorg.apache.jasper.compiler.disablejsr199=true + +# JVM memory settings +java.arg.2=-Xms512m +java.arg.3=-Xmx512m + +# Enable Remote Debugging +#java.arg.debug=-agentlib:jdwp=transport=dt_socket,server=y,suspend=n,address=8000 + +java.arg.4=-Djava.net.preferIPv4Stack=true + +# allowRestrictedHeaders is required for Cluster/Node communications to work properly +java.arg.5=-Dsun.net.http.allowRestrictedHeaders=true +java.arg.6=-Djava.protocol.handler.pkgs=sun.net.www.protocol + +# The G1GC is still considered experimental but has proven to be very advantageous in providing great +# performance without significant "stop-the-world" delays. +java.arg.13=-XX:+UseG1GC + +#Set headless mode by default +java.arg.14=-Djava.awt.headless=true + +# Master key in hexadecimal format for encrypted sensitive configuration values +nifi.bootstrap.sensitive.key= + +### +# Notification Services for notifying interested parties when NiFi is stopped, started, dies +### + +# XML File that contains the definitions of the notification services +notification.services.file=./conf/bootstrap-notification-services.xml + +# In the case that we are unable to send a notification for an event, how many times should we retry? +notification.max.attempts=5 + +# Comma-separated list of identifiers that are present in the notification.services.file; which services should be used to notify when NiFi is started? +#nifi.start.notification.services=email-notification + +# Comma-separated list of identifiers that are present in the notification.services.file; which services should be used to notify when NiFi is stopped? +#nifi.stop.notification.services=email-notification + +# Comma-separated list of identifiers that are present in the notification.services.file; which services should be used to notify when NiFi dies? +#nifi.dead.notification.services=email-notification \ No newline at end of file diff --git a/nifi-toolkit/nifi-toolkit-encrypt-config/src/test/resources/bootstrap_with_master_key.conf b/nifi-toolkit/nifi-toolkit-encrypt-config/src/test/resources/bootstrap_with_master_key.conf new file mode 100644 index 0000000000..9225126caa --- /dev/null +++ b/nifi-toolkit/nifi-toolkit-encrypt-config/src/test/resources/bootstrap_with_master_key.conf @@ -0,0 +1,74 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# Java command to use when running NiFi +java=java + +# Username to use when running NiFi. This value will be ignored on Windows. +run.as= + +# Configure where NiFi's lib and conf directories live +lib.dir=./lib +conf.dir=./conf + +# How long to wait after telling NiFi to shutdown before explicitly killing the Process +graceful.shutdown.seconds=20 + +# Disable JSR 199 so that we can use JSP's without running a JDK +java.arg.1=-Dorg.apache.jasper.compiler.disablejsr199=true + +# JVM memory settings +java.arg.2=-Xms512m +java.arg.3=-Xmx512m + +# Enable Remote Debugging +#java.arg.debug=-agentlib:jdwp=transport=dt_socket,server=y,suspend=n,address=8000 + +java.arg.4=-Djava.net.preferIPv4Stack=true + +# allowRestrictedHeaders is required for Cluster/Node communications to work properly +java.arg.5=-Dsun.net.http.allowRestrictedHeaders=true +java.arg.6=-Djava.protocol.handler.pkgs=sun.net.www.protocol + +# The G1GC is still considered experimental but has proven to be very advantageous in providing great +# performance without significant "stop-the-world" delays. +java.arg.13=-XX:+UseG1GC + +#Set headless mode by default +java.arg.14=-Djava.awt.headless=true + +# Master key in hexadecimal format for encrypted sensitive configuration values +nifi.bootstrap.sensitive.key=0123456789ABCDEFFEDCBA98765432100123456789ABCDEFFEDCBA9876543210 + +### +# Notification Services for notifying interested parties when NiFi is stopped, started, dies +### + +# XML File that contains the definitions of the notification services +notification.services.file=./conf/bootstrap-notification-services.xml + +# In the case that we are unable to send a notification for an event, how many times should we retry? +notification.max.attempts=5 + +# Comma-separated list of identifiers that are present in the notification.services.file; which services should be used to notify when NiFi is started? +#nifi.start.notification.services=email-notification + +# Comma-separated list of identifiers that are present in the notification.services.file; which services should be used to notify when NiFi is stopped? +#nifi.stop.notification.services=email-notification + +# Comma-separated list of identifiers that are present in the notification.services.file; which services should be used to notify when NiFi dies? +#nifi.dead.notification.services=email-notification \ No newline at end of file diff --git a/nifi-toolkit/nifi-toolkit-encrypt-config/src/test/resources/log4j.properties b/nifi-toolkit/nifi-toolkit-encrypt-config/src/test/resources/log4j.properties new file mode 100644 index 0000000000..05cd3758f1 --- /dev/null +++ b/nifi-toolkit/nifi-toolkit-encrypt-config/src/test/resources/log4j.properties @@ -0,0 +1,22 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +log4j.rootLogger=DEBUG,console + +log4j.appender.console=org.apache.log4j.ConsoleAppender +log4j.appender.console.layout=org.apache.log4j.PatternLayout +log4j.appender.console.layout.ConversionPattern=%d{yy/MM/dd HH:mm:ss} %p %c{2}: %m%n \ No newline at end of file diff --git a/nifi-toolkit/nifi-toolkit-encrypt-config/src/test/resources/logback-test.xml b/nifi-toolkit/nifi-toolkit-encrypt-config/src/test/resources/logback-test.xml new file mode 100644 index 0000000000..af8074bdc8 --- /dev/null +++ b/nifi-toolkit/nifi-toolkit-encrypt-config/src/test/resources/logback-test.xml @@ -0,0 +1,34 @@ + + + + + + + %-4r [%t] %-5p %c - %m%n + + + + + %-4r [%t] %-5p %c - %m%n + + + + + + + + + diff --git a/nifi-toolkit/nifi-toolkit-encrypt-config/src/test/resources/nifi_with_few_sensitive_properties_protected_aes.properties b/nifi-toolkit/nifi-toolkit-encrypt-config/src/test/resources/nifi_with_few_sensitive_properties_protected_aes.properties new file mode 100644 index 0000000000..f3f243d05c --- /dev/null +++ b/nifi-toolkit/nifi-toolkit-encrypt-config/src/test/resources/nifi_with_few_sensitive_properties_protected_aes.properties @@ -0,0 +1,34 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Core Properties # +nifi.version=nifi-test 3.0.0 + +# security properties # +nifi.sensitive.props.key=n2z+tTTbHuZ4V4V2||uWhdasyDXD4ZG2lMAes/vqh6u4vaz4xgL4aEbF4Y/dXevqk3ulRcOwf1vc4RDQ== +nifi.sensitive.props.key.protected=aes/gcm/256 +nifi.sensitive.props.algorithm=PBEWITHMD5AND256BITAES-CBC-OPENSSL +nifi.sensitive.props.provider=BC +nifi.sensitive.props.additional.keys= + +nifi.security.keystore=/path/to/keystore.jks +nifi.security.keystoreType=JKS +nifi.security.keystorePasswd=oBjT92hIGRElIGOh||MZ6uYuWNBrOA6usq/Jt3DaD2e4otNirZDytac/w/KFe0HOkrJR03vcbo +nifi.security.keystorePasswd.protected=aes/gcm/256 +nifi.security.keyPasswd=ac/BaE35SL/esLiJ||+ULRvRLYdIDA2VqpE0eQXDEMjaLBMG2kbKOdOwBk/hGebDKlVg== +nifi.security.keyPasswd.protected=aes/gcm/256 +nifi.security.truststore= +nifi.security.truststoreType= +nifi.security.truststorePasswd= diff --git a/nifi-toolkit/nifi-toolkit-encrypt-config/src/test/resources/nifi_with_few_sensitive_properties_unprotected.properties b/nifi-toolkit/nifi-toolkit-encrypt-config/src/test/resources/nifi_with_few_sensitive_properties_unprotected.properties new file mode 100644 index 0000000000..ee09f47153 --- /dev/null +++ b/nifi-toolkit/nifi-toolkit-encrypt-config/src/test/resources/nifi_with_few_sensitive_properties_unprotected.properties @@ -0,0 +1,31 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Core Properties # +nifi.version=nifi-test 3.0.0 + +# security properties # +nifi.sensitive.props.key=thisIsABadSensitiveKeyPassword +nifi.sensitive.props.algorithm=PBEWITHMD5AND256BITAES-CBC-OPENSSL +nifi.sensitive.props.provider=BC +nifi.sensitive.props.additional.keys= + +nifi.security.keystore=/path/to/keystore.jks +nifi.security.keystoreType=JKS +nifi.security.keystorePasswd=thisIsABadKeystorePassword +nifi.security.keyPasswd=thisIsABadKeyPassword +nifi.security.truststore= +nifi.security.truststoreType= +nifi.security.truststorePasswd= \ No newline at end of file diff --git a/nifi-toolkit/nifi-toolkit-encrypt-config/src/test/resources/nifi_with_sensitive_properties_protected_aes.properties b/nifi-toolkit/nifi-toolkit-encrypt-config/src/test/resources/nifi_with_sensitive_properties_protected_aes.properties new file mode 100644 index 0000000000..c3a7a5d4fd --- /dev/null +++ b/nifi-toolkit/nifi-toolkit-encrypt-config/src/test/resources/nifi_with_sensitive_properties_protected_aes.properties @@ -0,0 +1,128 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Core Properties # +nifi.version=nifi-test 3.0.0 +nifi.flow.configuration.file=./target/flow.xml.gz +nifi.flow.configuration.archive.dir=./target/archive/ +nifi.flowcontroller.autoResumeState=true +nifi.flowcontroller.graceful.shutdown.period=10 sec +nifi.flowservice.writedelay.interval=2 sec +nifi.administrative.yield.duration=30 sec + +nifi.reporting.task.configuration.file=./target/reporting-tasks.xml +nifi.controller.service.configuration.file=./target/controller-services.xml +nifi.templates.directory=./target/templates +nifi.ui.banner.text=UI Banner Text +nifi.ui.autorefresh.interval=30 sec +nifi.nar.library.directory=./target/resources/NiFiProperties/lib/ +nifi.nar.library.directory.alt=./target/resources/NiFiProperties/lib2/ +nifi.nar.working.directory=./target/work/nar/ + +# H2 Settings +nifi.database.directory=./target/database_repository +nifi.h2.url.append=;LOCK_TIMEOUT=25000;WRITE_DELAY=0;AUTO_SERVER=FALSE + +# FlowFile Repository +nifi.flowfile.repository.directory=./target/test-repo +nifi.flowfile.repository.partitions=1 +nifi.flowfile.repository.checkpoint.interval=2 mins +nifi.queue.swap.threshold=20000 +nifi.swap.storage.directory=./target/test-repo/swap +nifi.swap.in.period=5 sec +nifi.swap.in.threads=1 +nifi.swap.out.period=5 sec +nifi.swap.out.threads=4 + +# Content Repository +nifi.content.claim.max.appendable.size=10 MB +nifi.content.claim.max.flow.files=100 +nifi.content.repository.directory.default=./target/content_repository + +# Provenance Repository Properties +nifi.provenance.repository.storage.directory=./target/provenance_repository +nifi.provenance.repository.max.storage.time=24 hours +nifi.provenance.repository.max.storage.size=1 GB +nifi.provenance.repository.rollover.time=30 secs +nifi.provenance.repository.rollover.size=100 MB + +# Site to Site properties +nifi.remote.input.socket.port=9990 +nifi.remote.input.secure=true + +# web properties # +nifi.web.war.directory=./target/lib +nifi.web.http.host= +nifi.web.http.port= +nifi.web.https.host=nifi.nifi.apache.org +nifi.web.https.port=8443 +nifi.web.jetty.working.directory=./target/work/jetty + +# security properties # +nifi.sensitive.props.key=n2z+tTTbHuZ4V4V2||uWhdasyDXD4ZG2lMAes/vqh6u4vaz4xgL4aEbF4Y/dXevqk3ulRcOwf1vc4RDQ== +nifi.sensitive.props.key.protected=aes/gcm/256 +nifi.sensitive.props.algorithm=PBEWITHMD5AND256BITAES-CBC-OPENSSL +nifi.sensitive.props.provider=BC +nifi.sensitive.props.additional.keys= + +nifi.security.keystore=/path/to/keystore.jks +nifi.security.keystoreType=JKS +nifi.security.keystorePasswd=oBjT92hIGRElIGOh||MZ6uYuWNBrOA6usq/Jt3DaD2e4otNirZDytac/w/KFe0HOkrJR03vcbo +nifi.security.keystorePasswd.protected=aes/gcm/256 +nifi.security.keyPasswd=ac/BaE35SL/esLiJ||+ULRvRLYdIDA2VqpE0eQXDEMjaLBMG2kbKOdOwBk/hGebDKlVg== +nifi.security.keyPasswd.protected=aes/gcm/256 +nifi.security.truststore= +nifi.security.truststoreType= +nifi.security.truststorePasswd= +nifi.security.needClientAuth= +nifi.security.user.authorizer= + +# cluster common properties (cluster manager and nodes must have same values) # +nifi.cluster.protocol.heartbeat.interval=5 sec +nifi.cluster.protocol.is.secure=false +nifi.cluster.protocol.socket.timeout=30 sec +nifi.cluster.protocol.connection.handshake.timeout=45 sec +# if multicast is used, then nifi.cluster.protocol.multicast.xxx properties must be configured # +nifi.cluster.protocol.use.multicast=false +nifi.cluster.protocol.multicast.address= +nifi.cluster.protocol.multicast.port= +nifi.cluster.protocol.multicast.service.broadcast.delay=500 ms +nifi.cluster.protocol.multicast.service.locator.attempts=3 +nifi.cluster.protocol.multicast.service.locator.attempts.delay=1 sec + +# cluster node properties (only configure for cluster nodes) # +nifi.cluster.is.node=false +nifi.cluster.node.address= +nifi.cluster.node.protocol.port= +nifi.cluster.node.protocol.threads=2 +# if multicast is not used, nifi.cluster.node.unicast.xxx must have same values as nifi.cluster.manager.xxx # +nifi.cluster.node.unicast.manager.address= +nifi.cluster.node.unicast.manager.protocol.port= +nifi.cluster.node.unicast.manager.authority.provider.port= + +# cluster manager properties (only configure for cluster manager) # +nifi.cluster.is.manager=false +nifi.cluster.manager.address= +nifi.cluster.manager.protocol.port= +nifi.cluster.manager.authority.provider.port= +nifi.cluster.manager.authority.provider.threads=10 +nifi.cluster.manager.node.firewall.file= +nifi.cluster.manager.node.event.history.size=10 +nifi.cluster.manager.node.api.connection.timeout=30 sec +nifi.cluster.manager.node.api.read.timeout=30 sec +nifi.cluster.manager.node.api.request.threads=10 +nifi.cluster.manager.flow.retrieval.delay=5 sec +nifi.cluster.manager.protocol.threads=10 +nifi.cluster.manager.safemode.duration=0 sec diff --git a/nifi-toolkit/nifi-toolkit-encrypt-config/src/test/resources/nifi_with_sensitive_properties_unprotected.properties b/nifi-toolkit/nifi-toolkit-encrypt-config/src/test/resources/nifi_with_sensitive_properties_unprotected.properties new file mode 100644 index 0000000000..954c2656e2 --- /dev/null +++ b/nifi-toolkit/nifi-toolkit-encrypt-config/src/test/resources/nifi_with_sensitive_properties_unprotected.properties @@ -0,0 +1,125 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Core Properties # +nifi.version=nifi-test 3.0.0 +nifi.flow.configuration.file=./target/flow.xml.gz +nifi.flow.configuration.archive.dir=./target/archive/ +nifi.flowcontroller.autoResumeState=true +nifi.flowcontroller.graceful.shutdown.period=10 sec +nifi.flowservice.writedelay.interval=2 sec +nifi.administrative.yield.duration=30 sec + +nifi.reporting.task.configuration.file=./target/reporting-tasks.xml +nifi.controller.service.configuration.file=./target/controller-services.xml +nifi.templates.directory=./target/templates +nifi.ui.banner.text=UI Banner Text +nifi.ui.autorefresh.interval=30 sec +nifi.nar.library.directory=./target/resources/NiFiProperties/lib/ +nifi.nar.library.directory.alt=./target/resources/NiFiProperties/lib2/ +nifi.nar.working.directory=./target/work/nar/ + +# H2 Settings +nifi.database.directory=./target/database_repository +nifi.h2.url.append=;LOCK_TIMEOUT=25000;WRITE_DELAY=0;AUTO_SERVER=FALSE + +# FlowFile Repository +nifi.flowfile.repository.directory=./target/test-repo +nifi.flowfile.repository.partitions=1 +nifi.flowfile.repository.checkpoint.interval=2 mins +nifi.queue.swap.threshold=20000 +nifi.swap.storage.directory=./target/test-repo/swap +nifi.swap.in.period=5 sec +nifi.swap.in.threads=1 +nifi.swap.out.period=5 sec +nifi.swap.out.threads=4 + +# Content Repository +nifi.content.claim.max.appendable.size=10 MB +nifi.content.claim.max.flow.files=100 +nifi.content.repository.directory.default=./target/content_repository + +# Provenance Repository Properties +nifi.provenance.repository.storage.directory=./target/provenance_repository +nifi.provenance.repository.max.storage.time=24 hours +nifi.provenance.repository.max.storage.size=1 GB +nifi.provenance.repository.rollover.time=30 secs +nifi.provenance.repository.rollover.size=100 MB + +# Site to Site properties +nifi.remote.input.socket.port=9990 +nifi.remote.input.secure=true + +# web properties # +nifi.web.war.directory=./target/lib +nifi.web.http.host= +nifi.web.http.port= +nifi.web.https.host=nifi.nifi.apache.org +nifi.web.https.port=8443 +nifi.web.jetty.working.directory=./target/work/jetty + +# security properties # +nifi.sensitive.props.key=thisIsABadSensitiveKeyPassword +nifi.sensitive.props.algorithm=PBEWITHMD5AND256BITAES-CBC-OPENSSL +nifi.sensitive.props.provider=BC +nifi.sensitive.props.additional.keys= + +nifi.security.keystore=/path/to/keystore.jks +nifi.security.keystoreType=JKS +nifi.security.keystorePasswd=thisIsABadKeystorePassword +nifi.security.keyPasswd=thisIsABadKeyPassword +nifi.security.truststore= +nifi.security.truststoreType= +nifi.security.truststorePasswd= +nifi.security.needClientAuth= +nifi.security.user.authorizer= + +# cluster common properties (cluster manager and nodes must have same values) # +nifi.cluster.protocol.heartbeat.interval=5 sec +nifi.cluster.protocol.is.secure=false +nifi.cluster.protocol.socket.timeout=30 sec +nifi.cluster.protocol.connection.handshake.timeout=45 sec +# if multicast is used, then nifi.cluster.protocol.multicast.xxx properties must be configured # +nifi.cluster.protocol.use.multicast=false +nifi.cluster.protocol.multicast.address= +nifi.cluster.protocol.multicast.port= +nifi.cluster.protocol.multicast.service.broadcast.delay=500 ms +nifi.cluster.protocol.multicast.service.locator.attempts=3 +nifi.cluster.protocol.multicast.service.locator.attempts.delay=1 sec + +# cluster node properties (only configure for cluster nodes) # +nifi.cluster.is.node=false +nifi.cluster.node.address= +nifi.cluster.node.protocol.port= +nifi.cluster.node.protocol.threads=2 +# if multicast is not used, nifi.cluster.node.unicast.xxx must have same values as nifi.cluster.manager.xxx # +nifi.cluster.node.unicast.manager.address= +nifi.cluster.node.unicast.manager.protocol.port= +nifi.cluster.node.unicast.manager.authority.provider.port= + +# cluster manager properties (only configure for cluster manager) # +nifi.cluster.is.manager=false +nifi.cluster.manager.address= +nifi.cluster.manager.protocol.port= +nifi.cluster.manager.authority.provider.port= +nifi.cluster.manager.authority.provider.threads=10 +nifi.cluster.manager.node.firewall.file= +nifi.cluster.manager.node.event.history.size=10 +nifi.cluster.manager.node.api.connection.timeout=30 sec +nifi.cluster.manager.node.api.read.timeout=30 sec +nifi.cluster.manager.node.api.request.threads=10 +nifi.cluster.manager.flow.retrieval.delay=5 sec +nifi.cluster.manager.protocol.threads=10 +nifi.cluster.manager.safemode.duration=0 sec diff --git a/nifi-toolkit/nifi-toolkit-encrypt-config/src/test/resources/nifi_with_sensitive_properties_unprotected_and_empty_protection_schemes.properties b/nifi-toolkit/nifi-toolkit-encrypt-config/src/test/resources/nifi_with_sensitive_properties_unprotected_and_empty_protection_schemes.properties new file mode 100644 index 0000000000..6bf2609dee --- /dev/null +++ b/nifi-toolkit/nifi-toolkit-encrypt-config/src/test/resources/nifi_with_sensitive_properties_unprotected_and_empty_protection_schemes.properties @@ -0,0 +1,127 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Core Properties # +nifi.version=nifi-test 3.0.0 +nifi.flow.configuration.file=./target/flow.xml.gz +nifi.flow.configuration.archive.dir=./target/archive/ +nifi.flowcontroller.autoResumeState=true +nifi.flowcontroller.graceful.shutdown.period=10 sec +nifi.flowservice.writedelay.interval=2 sec +nifi.administrative.yield.duration=30 sec + +nifi.reporting.task.configuration.file=./target/reporting-tasks.xml +nifi.controller.service.configuration.file=./target/controller-services.xml +nifi.templates.directory=./target/templates +nifi.ui.banner.text=UI Banner Text +nifi.ui.banner.text.protected= +nifi.ui.autorefresh.interval=30 sec +nifi.nar.library.directory=./target/resources/NiFiProperties/lib/ +nifi.nar.library.directory.alt=./target/resources/NiFiProperties/lib2/ +nifi.nar.working.directory=./target/work/nar/ + +# H2 Settings +nifi.database.directory=./target/database_repository +nifi.h2.url.append=;LOCK_TIMEOUT=25000;WRITE_DELAY=0;AUTO_SERVER=FALSE + +# FlowFile Repository +nifi.flowfile.repository.directory=./target/test-repo +nifi.flowfile.repository.partitions=1 +nifi.flowfile.repository.checkpoint.interval=2 mins +nifi.queue.swap.threshold=20000 +nifi.swap.storage.directory=./target/test-repo/swap +nifi.swap.in.period=5 sec +nifi.swap.in.threads=1 +nifi.swap.out.period=5 sec +nifi.swap.out.threads=4 + +# Content Repository +nifi.content.claim.max.appendable.size=10 MB +nifi.content.claim.max.flow.files=100 +nifi.content.repository.directory.default=./target/content_repository + +# Provenance Repository Properties +nifi.provenance.repository.storage.directory=./target/provenance_repository +nifi.provenance.repository.max.storage.time=24 hours +nifi.provenance.repository.max.storage.size=1 GB +nifi.provenance.repository.rollover.time=30 secs +nifi.provenance.repository.rollover.size=100 MB + +# Site to Site properties +nifi.remote.input.socket.port=9990 +nifi.remote.input.secure=true + +# web properties # +nifi.web.war.directory=./target/lib +nifi.web.http.host= +nifi.web.http.port= +nifi.web.https.host=nifi.nifi.apache.org +nifi.web.https.port=8443 +nifi.web.jetty.working.directory=./target/work/jetty + +# security properties # +nifi.sensitive.props.key=thisIsABadSensitiveKeyPassword +nifi.sensitive.props.key.protected= +nifi.sensitive.props.algorithm=PBEWITHMD5AND256BITAES-CBC-OPENSSL +nifi.sensitive.props.provider=BC +nifi.sensitive.props.additional.keys= + +nifi.security.keystore=/path/to/keystore.jks +nifi.security.keystoreType=JKS +nifi.security.keystorePasswd=thisIsABadKeystorePassword +nifi.security.keyPasswd=thisIsABadKeyPassword +nifi.security.truststore= +nifi.security.truststoreType= +nifi.security.truststorePasswd= +nifi.security.needClientAuth= +nifi.security.user.authorizer= + +# cluster common properties (cluster manager and nodes must have same values) # +nifi.cluster.protocol.heartbeat.interval=5 sec +nifi.cluster.protocol.is.secure=false +nifi.cluster.protocol.socket.timeout=30 sec +nifi.cluster.protocol.connection.handshake.timeout=45 sec +# if multicast is used, then nifi.cluster.protocol.multicast.xxx properties must be configured # +nifi.cluster.protocol.use.multicast=false +nifi.cluster.protocol.multicast.address= +nifi.cluster.protocol.multicast.port= +nifi.cluster.protocol.multicast.service.broadcast.delay=500 ms +nifi.cluster.protocol.multicast.service.locator.attempts=3 +nifi.cluster.protocol.multicast.service.locator.attempts.delay=1 sec + +# cluster node properties (only configure for cluster nodes) # +nifi.cluster.is.node=false +nifi.cluster.node.address= +nifi.cluster.node.protocol.port= +nifi.cluster.node.protocol.threads=2 +# if multicast is not used, nifi.cluster.node.unicast.xxx must have same values as nifi.cluster.manager.xxx # +nifi.cluster.node.unicast.manager.address= +nifi.cluster.node.unicast.manager.protocol.port= +nifi.cluster.node.unicast.manager.authority.provider.port= + +# cluster manager properties (only configure for cluster manager) # +nifi.cluster.is.manager=false +nifi.cluster.manager.address= +nifi.cluster.manager.protocol.port= +nifi.cluster.manager.authority.provider.port= +nifi.cluster.manager.authority.provider.threads=10 +nifi.cluster.manager.node.firewall.file= +nifi.cluster.manager.node.event.history.size=10 +nifi.cluster.manager.node.api.connection.timeout=30 sec +nifi.cluster.manager.node.api.read.timeout=30 sec +nifi.cluster.manager.node.api.request.threads=10 +nifi.cluster.manager.flow.retrieval.delay=5 sec +nifi.cluster.manager.protocol.threads=10 +nifi.cluster.manager.safemode.duration=0 sec diff --git a/nifi-toolkit/nifi-toolkit-tls/pom.xml b/nifi-toolkit/nifi-toolkit-tls/pom.xml index 247ebdbf10..f2699172e3 100644 --- a/nifi-toolkit/nifi-toolkit-tls/pom.xml +++ b/nifi-toolkit/nifi-toolkit-tls/pom.xml @@ -35,10 +35,6 @@ org.slf4j slf4j-api - - org.slf4j - slf4j-log4j12 - org.bouncycastle bcpkix-jdk15on diff --git a/nifi-toolkit/pom.xml b/nifi-toolkit/pom.xml index 4f9e7d1ee7..30344736de 100644 --- a/nifi-toolkit/pom.xml +++ b/nifi-toolkit/pom.xml @@ -13,18 +13,35 @@ See the License for the specific language governing permissions and limitations under the License. --> - + 4.0.0 org.apache.nifi nifi 1.0.0-SNAPSHOT - org.apache.nifi nifi-toolkit pom nifi-toolkit-tls + nifi-toolkit-encrypt-config nifi-toolkit-assembly + + + + org.codehaus.groovy + groovy-all + 2.4.5 + compile + + + org.slf4j + slf4j-log4j12 + + + + + diff --git a/pom.xml b/pom.xml index 47a8992aea..1e816a5187 100644 --- a/pom.xml +++ b/pom.xml @@ -884,6 +884,11 @@ language governing permissions and limitations under the License. --> nifi-toolkit-tls 1.0.0-SNAPSHOT + + org.apache.nifi + nifi-toolkit-encrypt-config + 1.0.0-SNAPSHOT + org.apache.nifi nifi-resources @@ -1211,7 +1216,7 @@ language governing permissions and limitations under the License. --> 1.0.0-SNAPSHOT nar - + org.apache.nifi nifi-windows-event-log-nar 1.0.0-SNAPSHOT @@ -1228,6 +1233,11 @@ language governing permissions and limitations under the License. --> nifi-properties 1.0.0-SNAPSHOT + + org.apache.nifi + nifi-properties-loader + 1.0.0-SNAPSHOT + org.apache.nifi nifi-security-utils