From a8817e023805499491f9fc62495208d198de84f0 Mon Sep 17 00:00:00 2001 From: Kevin Doran Date: Sat, 30 Dec 2017 08:54:18 -0500 Subject: [PATCH] NIFI-4708 Add Registry support to encrypt-config. Adds support for NiFI Registry config files to the encrypt-config tool in NiFi Toolkit. Also adds decryption capability to encrypt-config tool. This closes #2376. Signed-off-by: Andy LoPresto --- .../AESSensitivePropertyProvider.java | 4 +- nifi-toolkit/nifi-toolkit-assembly/NOTICE | 10 + .../src/main/resources/bin/encrypt-config.bat | 2 +- .../src/main/resources/bin/encrypt-config.sh | 2 +- .../nifi-toolkit-encrypt-config/pom.xml | 22 + .../properties/ConfigEncryptionTool.groovy | 47 ++- .../encryptconfig/Configuration.groovy | 29 ++ .../toolkit/encryptconfig/DecryptMode.groovy | 327 +++++++++++++++ .../encryptconfig/EncryptConfigLogger.groovy | 83 ++++ .../encryptconfig/EncryptConfigMain.groovy | 138 +++++++ .../toolkit/encryptconfig/LegacyMode.groovy | 32 ++ .../NiFiRegistryDecryptMode.groovy | 124 ++++++ .../encryptconfig/NiFiRegistryMode.groovy | 382 ++++++++++++++++++ .../toolkit/encryptconfig/ToolMode.groovy | 23 ++ .../encryptconfig/util/BootstrapUtil.groovy | 132 ++++++ .../util/NiFiPropertiesEncryptor.groovy | 54 +++ ...NiFiRegistryAuthorizersXmlEncryptor.groovy | 103 +++++ ...gistryIdentityProvidersXmlEncryptor.groovy | 102 +++++ .../NiFiRegistryPropertiesEncryptor.groovy | 65 +++ .../util/PropertiesEncryptor.groovy | 269 ++++++++++++ .../encryptconfig/util/ToolUtilities.groovy | 164 ++++++++ .../encryptconfig/util/XmlEncryptor.groovy | 200 +++++++++ .../src/main/resources/log4j.properties | 3 +- .../EncryptConfigMainTest.groovy | 285 +++++++++++++ .../NiFiRegistryDecryptModeSpec.groovy | 117 ++++++ .../encryptconfig/NiFiRegistryModeSpec.groovy | 331 +++++++++++++++ .../toolkit/encryptconfig/TestUtil.groovy | 376 +++++++++++++++++ .../util/BootstrapUtilSpec.groovy | 113 ++++++ .../nifi-registry/authorizers-commented.xml | 242 +++++++++++ .../nifi-registry/authorizers-empty.xml | 240 +++++++++++ .../authorizers-populated-unprotected.xml | 246 +++++++++++ .../nifi-registry/bootstrap_default.conf | 48 +++ .../bootstrap_with_empty_master_key.conf | 48 +++ .../bootstrap_with_master_key_128.conf | 48 +++ ...rap_with_master_key_from_password_128.conf | 48 +++ .../bootstrap_without_master_key.conf | 45 +++ .../identity-providers-commented.xml | 106 +++++ .../identity-providers-empty.xml | 104 +++++ ...entity-providers-populated-unprotected.xml | 97 +++++ .../nifi-registry-commented.properties | 31 ++ .../nifi-registry-empty.properties | 31 ++ ...try-populated-protected-key-128.properties | 50 +++ ...try-populated-protected-key-256.properties | 50 +++ ...opulated-protected-password-256.properties | 52 +++ ...-registry-populated-unprotected.properties | 45 +++ 45 files changed, 5044 insertions(+), 26 deletions(-) create mode 100644 nifi-toolkit/nifi-toolkit-encrypt-config/src/main/groovy/org/apache/nifi/toolkit/encryptconfig/Configuration.groovy create mode 100644 nifi-toolkit/nifi-toolkit-encrypt-config/src/main/groovy/org/apache/nifi/toolkit/encryptconfig/DecryptMode.groovy create mode 100644 nifi-toolkit/nifi-toolkit-encrypt-config/src/main/groovy/org/apache/nifi/toolkit/encryptconfig/EncryptConfigLogger.groovy create mode 100644 nifi-toolkit/nifi-toolkit-encrypt-config/src/main/groovy/org/apache/nifi/toolkit/encryptconfig/EncryptConfigMain.groovy create mode 100644 nifi-toolkit/nifi-toolkit-encrypt-config/src/main/groovy/org/apache/nifi/toolkit/encryptconfig/LegacyMode.groovy create mode 100644 nifi-toolkit/nifi-toolkit-encrypt-config/src/main/groovy/org/apache/nifi/toolkit/encryptconfig/NiFiRegistryDecryptMode.groovy create mode 100644 nifi-toolkit/nifi-toolkit-encrypt-config/src/main/groovy/org/apache/nifi/toolkit/encryptconfig/NiFiRegistryMode.groovy create mode 100644 nifi-toolkit/nifi-toolkit-encrypt-config/src/main/groovy/org/apache/nifi/toolkit/encryptconfig/ToolMode.groovy create mode 100644 nifi-toolkit/nifi-toolkit-encrypt-config/src/main/groovy/org/apache/nifi/toolkit/encryptconfig/util/BootstrapUtil.groovy create mode 100644 nifi-toolkit/nifi-toolkit-encrypt-config/src/main/groovy/org/apache/nifi/toolkit/encryptconfig/util/NiFiPropertiesEncryptor.groovy create mode 100644 nifi-toolkit/nifi-toolkit-encrypt-config/src/main/groovy/org/apache/nifi/toolkit/encryptconfig/util/NiFiRegistryAuthorizersXmlEncryptor.groovy create mode 100644 nifi-toolkit/nifi-toolkit-encrypt-config/src/main/groovy/org/apache/nifi/toolkit/encryptconfig/util/NiFiRegistryIdentityProvidersXmlEncryptor.groovy create mode 100644 nifi-toolkit/nifi-toolkit-encrypt-config/src/main/groovy/org/apache/nifi/toolkit/encryptconfig/util/NiFiRegistryPropertiesEncryptor.groovy create mode 100644 nifi-toolkit/nifi-toolkit-encrypt-config/src/main/groovy/org/apache/nifi/toolkit/encryptconfig/util/PropertiesEncryptor.groovy create mode 100644 nifi-toolkit/nifi-toolkit-encrypt-config/src/main/groovy/org/apache/nifi/toolkit/encryptconfig/util/ToolUtilities.groovy create mode 100644 nifi-toolkit/nifi-toolkit-encrypt-config/src/main/groovy/org/apache/nifi/toolkit/encryptconfig/util/XmlEncryptor.groovy create mode 100644 nifi-toolkit/nifi-toolkit-encrypt-config/src/test/groovy/org/apache/nifi/toolkit/encryptconfig/EncryptConfigMainTest.groovy create mode 100644 nifi-toolkit/nifi-toolkit-encrypt-config/src/test/groovy/org/apache/nifi/toolkit/encryptconfig/NiFiRegistryDecryptModeSpec.groovy create mode 100644 nifi-toolkit/nifi-toolkit-encrypt-config/src/test/groovy/org/apache/nifi/toolkit/encryptconfig/NiFiRegistryModeSpec.groovy create mode 100644 nifi-toolkit/nifi-toolkit-encrypt-config/src/test/groovy/org/apache/nifi/toolkit/encryptconfig/TestUtil.groovy create mode 100644 nifi-toolkit/nifi-toolkit-encrypt-config/src/test/groovy/org/apache/nifi/toolkit/encryptconfig/util/BootstrapUtilSpec.groovy create mode 100644 nifi-toolkit/nifi-toolkit-encrypt-config/src/test/resources/nifi-registry/authorizers-commented.xml create mode 100644 nifi-toolkit/nifi-toolkit-encrypt-config/src/test/resources/nifi-registry/authorizers-empty.xml create mode 100644 nifi-toolkit/nifi-toolkit-encrypt-config/src/test/resources/nifi-registry/authorizers-populated-unprotected.xml create mode 100644 nifi-toolkit/nifi-toolkit-encrypt-config/src/test/resources/nifi-registry/bootstrap_default.conf create mode 100644 nifi-toolkit/nifi-toolkit-encrypt-config/src/test/resources/nifi-registry/bootstrap_with_empty_master_key.conf create mode 100644 nifi-toolkit/nifi-toolkit-encrypt-config/src/test/resources/nifi-registry/bootstrap_with_master_key_128.conf create mode 100644 nifi-toolkit/nifi-toolkit-encrypt-config/src/test/resources/nifi-registry/bootstrap_with_master_key_from_password_128.conf create mode 100644 nifi-toolkit/nifi-toolkit-encrypt-config/src/test/resources/nifi-registry/bootstrap_without_master_key.conf create mode 100644 nifi-toolkit/nifi-toolkit-encrypt-config/src/test/resources/nifi-registry/identity-providers-commented.xml create mode 100644 nifi-toolkit/nifi-toolkit-encrypt-config/src/test/resources/nifi-registry/identity-providers-empty.xml create mode 100644 nifi-toolkit/nifi-toolkit-encrypt-config/src/test/resources/nifi-registry/identity-providers-populated-unprotected.xml create mode 100644 nifi-toolkit/nifi-toolkit-encrypt-config/src/test/resources/nifi-registry/nifi-registry-commented.properties create mode 100644 nifi-toolkit/nifi-toolkit-encrypt-config/src/test/resources/nifi-registry/nifi-registry-empty.properties create mode 100644 nifi-toolkit/nifi-toolkit-encrypt-config/src/test/resources/nifi-registry/nifi-registry-populated-protected-key-128.properties create mode 100644 nifi-toolkit/nifi-toolkit-encrypt-config/src/test/resources/nifi-registry/nifi-registry-populated-protected-key-256.properties create mode 100644 nifi-toolkit/nifi-toolkit-encrypt-config/src/test/resources/nifi-registry/nifi-registry-populated-protected-password-256.properties create mode 100644 nifi-toolkit/nifi-toolkit-encrypt-config/src/test/resources/nifi-registry/nifi-registry-populated-unprotected.properties diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-properties-loader/src/main/java/org/apache/nifi/properties/AESSensitivePropertyProvider.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-properties-loader/src/main/java/org/apache/nifi/properties/AESSensitivePropertyProvider.java index 1df398ebda..062e35233c 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-properties-loader/src/main/java/org/apache/nifi/properties/AESSensitivePropertyProvider.java +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-properties-loader/src/main/java/org/apache/nifi/properties/AESSensitivePropertyProvider.java @@ -175,7 +175,7 @@ public class AESSensitivePropertyProvider implements SensitivePropertyProvider { byte[] plainBytes = unprotectedValue.getBytes(StandardCharsets.UTF_8); byte[] cipherBytes = cipher.doFinal(plainBytes); - logger.info(getName() + " encrypted a sensitive value successfully"); + logger.debug(getName() + " encrypted a sensitive value successfully"); return base64Encode(iv) + DELIMITER + base64Encode(cipherBytes); // return Base64.toBase64String(iv) + DELIMITER + Base64.toBase64String(cipherBytes); } catch (BadPaddingException | IllegalBlockSizeException | EncoderException | InvalidAlgorithmParameterException | InvalidKeyException e) { @@ -238,7 +238,7 @@ public class AESSensitivePropertyProvider implements SensitivePropertyProvider { cipher.init(Cipher.DECRYPT_MODE, this.key, new IvParameterSpec(iv)); byte[] plainBytes = cipher.doFinal(cipherBytes); - logger.info(getName() + " decrypted a sensitive value successfully"); + logger.debug(getName() + " decrypted a sensitive value successfully"); return new String(plainBytes, StandardCharsets.UTF_8); } catch (BadPaddingException | IllegalBlockSizeException | DecoderException | InvalidAlgorithmParameterException | InvalidKeyException e) { final String msg = "Error decrypting a protected value"; diff --git a/nifi-toolkit/nifi-toolkit-assembly/NOTICE b/nifi-toolkit/nifi-toolkit-assembly/NOTICE index ff89f5ed61..dd99d2d464 100644 --- a/nifi-toolkit/nifi-toolkit-assembly/NOTICE +++ b/nifi-toolkit/nifi-toolkit-assembly/NOTICE @@ -15,6 +15,11 @@ The following binary components are provided under the Apache Software License v Apache NiFi Copyright 2014-2016 The Apache Software Foundation + (ASLv2) Apache Commons BeanUtils + The following NOTICE information applies: + Apache Commons BeanUtils + Copyright 2000-2016 The Apache Software Foundation + (ASLv2) Apache Commons CLI The following NOTICE information applies: Apache Commons CLI @@ -37,6 +42,11 @@ The following binary components are provided under the Apache Software License v Original source copyright: Copyright (c) 2008 Alexander Beider & Stephen P. Morse. + (ASLv2) Apache Commons Configuration + The following NOTICE information applies: + Apache Commons Configuration + Copyright 2001-2017 The Apache Software Foundation + (ASLv2) Apache Commons IO The following NOTICE information applies: Apache Commons IO diff --git a/nifi-toolkit/nifi-toolkit-assembly/src/main/resources/bin/encrypt-config.bat b/nifi-toolkit/nifi-toolkit-assembly/src/main/resources/bin/encrypt-config.bat index 29e42e676d..6ed06686c2 100644 --- a/nifi-toolkit/nifi-toolkit-assembly/src/main/resources/bin/encrypt-config.bat +++ b/nifi-toolkit/nifi-toolkit-assembly/src/main/resources/bin/encrypt-config.bat @@ -35,7 +35,7 @@ set LIB_DIR=%~dp0..\classpath;%~dp0..\lib if "%JAVA_OPTS%" == "" set JAVA_OPTS=-Xms128m -Xmx256m -SET JAVA_PARAMS=-cp %LIB_DIR%\* %JAVA_OPTS% org.apache.nifi.properties.ConfigEncryptionTool +SET JAVA_PARAMS=-cp %LIB_DIR%\* %JAVA_OPTS% org.apache.nifi.toolkit.encryptconfig.EncryptConfigMain cmd.exe /C ""%JAVA_EXE%" %JAVA_PARAMS% %* "" diff --git a/nifi-toolkit/nifi-toolkit-assembly/src/main/resources/bin/encrypt-config.sh b/nifi-toolkit/nifi-toolkit-assembly/src/main/resources/bin/encrypt-config.sh index 4acaab61ce..891b5ad082 100644 --- a/nifi-toolkit/nifi-toolkit-assembly/src/main/resources/bin/encrypt-config.sh +++ b/nifi-toolkit/nifi-toolkit-assembly/src/main/resources/bin/encrypt-config.sh @@ -111,7 +111,7 @@ run() { export NIFI_TOOLKIT_HOME="$NIFI_TOOLKIT_HOME" umask 0077 - "${JAVA}" -cp "${CLASSPATH}" ${JAVA_OPTS:--Xms128m -Xmx256m} org.apache.nifi.properties.ConfigEncryptionTool "$@" + "${JAVA}" -cp "${CLASSPATH}" ${JAVA_OPTS:--Xms128m -Xmx256m} org.apache.nifi.toolkit.encryptconfig.EncryptConfigMain "$@" return $? } diff --git a/nifi-toolkit/nifi-toolkit-encrypt-config/pom.xml b/nifi-toolkit/nifi-toolkit-encrypt-config/pom.xml index f4b9c3ecf5..b20f159c1b 100644 --- a/nifi-toolkit/nifi-toolkit-encrypt-config/pom.xml +++ b/nifi-toolkit/nifi-toolkit-encrypt-config/pom.xml @@ -70,6 +70,28 @@ + + org.apache.commons + commons-configuration2 + 2.0 + + + commons-beanutils + commons-beanutils + 1.9.3 + + + org.spockframework + spock-core + 1.0-groovy-2.4 + test + + + cglib + cglib-nodep + 2.2.2 + test + diff --git a/nifi-toolkit/nifi-toolkit-encrypt-config/src/main/groovy/org/apache/nifi/properties/ConfigEncryptionTool.groovy b/nifi-toolkit/nifi-toolkit-encrypt-config/src/main/groovy/org/apache/nifi/properties/ConfigEncryptionTool.groovy index 0a1112fdad..0e507c87cf 100644 --- a/nifi-toolkit/nifi-toolkit-encrypt-config/src/main/groovy/org/apache/nifi/properties/ConfigEncryptionTool.groovy +++ b/nifi-toolkit/nifi-toolkit-encrypt-config/src/main/groovy/org/apache/nifi/properties/ConfigEncryptionTool.groovy @@ -22,6 +22,7 @@ import org.apache.commons.cli.CommandLine import org.apache.commons.cli.CommandLineParser import org.apache.commons.cli.DefaultParser import org.apache.commons.cli.HelpFormatter +import org.apache.commons.cli.Option import org.apache.commons.cli.Options import org.apache.commons.cli.ParseException import org.apache.commons.codec.binary.Hex @@ -196,27 +197,31 @@ class ConfigEncryptionTool { ConfigEncryptionTool(String description) { this.header = buildHeader(description) this.options = new Options() - options.addOption("h", HELP_ARG, false, "Prints this usage message") - options.addOption("v", VERBOSE_ARG, false, "Sets verbose mode (default false)") - options.addOption("n", NIFI_PROPERTIES_ARG, true, "The nifi.properties file containing unprotected config values (will be overwritten)") - options.addOption("l", LOGIN_IDENTITY_PROVIDERS_ARG, true, "The login-identity-providers.xml file containing unprotected config values (will be overwritten)") - options.addOption("a", AUTHORIZERS_ARG, true, "The authorizers.xml file containing unprotected config values (will be overwritten)") - options.addOption("f", FLOW_XML_ARG, true, "The flow.xml.gz file currently protected with old password (will be overwritten)") - options.addOption("b", BOOTSTRAP_CONF_ARG, true, "The bootstrap.conf file to persist master key") - options.addOption("o", OUTPUT_NIFI_PROPERTIES_ARG, true, "The destination nifi.properties file containing protected config values (will not modify input nifi.properties)") - options.addOption("i", OUTPUT_LOGIN_IDENTITY_PROVIDERS_ARG, true, "The destination login-identity-providers.xml file containing protected config values (will not modify input login-identity-providers.xml)") - options.addOption("u", OUTPUT_AUTHORIZERS_ARG, true, "The destination authorizers.xml file containing protected config values (will not modify input authorizers.xml)") - options.addOption("g", OUTPUT_FLOW_XML_ARG, true, "The destination flow.xml.gz file containing protected config values (will not modify input flow.xml.gz)") - options.addOption("k", KEY_ARG, true, "The raw hexadecimal key to use to encrypt the sensitive properties") - options.addOption("e", KEY_MIGRATION_ARG, true, "The old raw hexadecimal key to use during key migration") - options.addOption("p", PASSWORD_ARG, true, "The password from which to derive the key to use to encrypt the sensitive properties") - options.addOption("w", PASSWORD_MIGRATION_ARG, true, "The old password from which to derive the key during migration") - options.addOption("r", USE_KEY_ARG, false, "If provided, the secure console will prompt for the raw key value in hexadecimal form") - options.addOption("m", MIGRATION_ARG, false, "If provided, the nifi.properties and/or login-identity-providers.xml sensitive properties will be re-encrypted with a new key") - options.addOption("x", DO_NOT_ENCRYPT_NIFI_PROPERTIES_ARG, false, "If provided, the properties in flow.xml.gz will be re-encrypted with a new key but the nifi.properties and/or login-identity-providers.xml files will not be modified") - options.addOption("s", PROPS_KEY_ARG, true, "The password or key to use to encrypt the sensitive processor properties in flow.xml.gz") - options.addOption("A", NEW_FLOW_ALGORITHM_ARG, true, "The algorithm to use to encrypt the sensitive processor properties in flow.xml.gz") - options.addOption("P", NEW_FLOW_PROVIDER_ARG, true, "The security provider to use to encrypt the sensitive processor properties in flow.xml.gz") + options.addOption(Option.builder("h").longOpt(HELP_ARG).hasArg(false).desc("Show usage information (this message)").build()) + options.addOption(Option.builder("v").longOpt(VERBOSE_ARG).hasArg(false).desc("Sets verbose mode (default false)").build()) + options.addOption(Option.builder("n").longOpt(NIFI_PROPERTIES_ARG).hasArg(true).argName("file").desc("The nifi.properties file containing unprotected config values (will be overwritten unless -o is specified)").build()) + options.addOption(Option.builder("o").longOpt(OUTPUT_NIFI_PROPERTIES_ARG).hasArg(true).argName("file").desc("The destination nifi.properties file containing protected config values (will not modify input nifi.properties)").build()) + options.addOption(Option.builder("l").longOpt(LOGIN_IDENTITY_PROVIDERS_ARG).hasArg(true).argName("file").desc("The login-identity-providers.xml file containing unprotected config values (will be overwritten unless -i is specified)").build()) + options.addOption(Option.builder("i").longOpt(OUTPUT_LOGIN_IDENTITY_PROVIDERS_ARG).hasArg(true).argName("file").desc("The destination login-identity-providers.xml file containing protected config values (will not modify input login-identity-providers.xml)").build()) + options.addOption(Option.builder("a").longOpt(AUTHORIZERS_ARG).hasArg(true).argName("file").desc("The authorizers.xml file containing unprotected config values (will be overwritten unless -u is specified)").build()) + options.addOption(Option.builder("u").longOpt(OUTPUT_AUTHORIZERS_ARG).hasArg(true).argName("file").desc("The destination authorizers.xml file containing protected config values (will not modify input authorizers.xml)").build()) + options.addOption(Option.builder("f").longOpt(FLOW_XML_ARG).hasArg(true).argName("file").desc("The flow.xml.gz file currently protected with old password (will be overwritten unless -g is specified)").build()) + options.addOption(Option.builder("g").longOpt(OUTPUT_FLOW_XML_ARG).hasArg(true).argName("file").desc("The destination flow.xml.gz file containing protected config values (will not modify input flow.xml.gz)").build()) + options.addOption(Option.builder("b").longOpt(BOOTSTRAP_CONF_ARG).hasArg(true).argName("file").desc("The bootstrap.conf file to persist master key").build()) + options.addOption(Option.builder("k").longOpt(KEY_ARG).hasArg(true).argName("keyhex").desc("The raw hexadecimal key to use to encrypt the sensitive properties").build()) + options.addOption(Option.builder("e").longOpt(KEY_MIGRATION_ARG).hasArg(true).argName("keyhex").desc("The old raw hexadecimal key to use during key migration").build()) + options.addOption(Option.builder("p").longOpt(PASSWORD_ARG).hasArg(true).argName("password").desc("The password from which to derive the key to use to encrypt the sensitive properties").build()) + options.addOption(Option.builder("w").longOpt(PASSWORD_MIGRATION_ARG).hasArg(true).argName("password").desc("The old password from which to derive the key during migration").build()) + options.addOption(Option.builder("r").longOpt(USE_KEY_ARG).hasArg(false).desc("If provided, the secure console will prompt for the raw key value in hexadecimal form").build()) + options.addOption(Option.builder("m").longOpt(MIGRATION_ARG).hasArg(false).desc("If provided, the nifi.properties and/or login-identity-providers.xml sensitive properties will be re-encrypted with a new key").build()) + options.addOption(Option.builder("x").longOpt(DO_NOT_ENCRYPT_NIFI_PROPERTIES_ARG).hasArg(false).desc("If provided, the properties in flow.xml.gz will be re-encrypted with a new key but the nifi.properties and/or login-identity-providers.xml files will not be modified").build()) + options.addOption(Option.builder("s").longOpt(PROPS_KEY_ARG).hasArg(true).argName("password|keyhex").desc("The password or key to use to encrypt the sensitive processor properties in flow.xml.gz").build()) + options.addOption(Option.builder("A").longOpt(NEW_FLOW_ALGORITHM_ARG).hasArg(true).argName("algorithm").desc("The algorithm to use to encrypt the sensitive processor properties in flow.xml.gz").build()) + options.addOption(Option.builder("P").longOpt(NEW_FLOW_PROVIDER_ARG).hasArg(true).argName("algorithm").desc("The security provider to use to encrypt the sensitive processor properties in flow.xml.gz").build()) + } + + static Options getCliOptions() { + return new ConfigEncryptionTool().options } /** diff --git a/nifi-toolkit/nifi-toolkit-encrypt-config/src/main/groovy/org/apache/nifi/toolkit/encryptconfig/Configuration.groovy b/nifi-toolkit/nifi-toolkit-encrypt-config/src/main/groovy/org/apache/nifi/toolkit/encryptconfig/Configuration.groovy new file mode 100644 index 0000000000..e5a8b23182 --- /dev/null +++ b/nifi-toolkit/nifi-toolkit-encrypt-config/src/main/groovy/org/apache/nifi/toolkit/encryptconfig/Configuration.groovy @@ -0,0 +1,29 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.toolkit.encryptconfig + +interface Configuration { + + enum KeySource { + PASSWORD, + KEY_HEX, + BOOTSTRAP_FILE + } + + // Future enhancement: configuration field accessors that are common to multiple (action, domain) combinations can go here + +} diff --git a/nifi-toolkit/nifi-toolkit-encrypt-config/src/main/groovy/org/apache/nifi/toolkit/encryptconfig/DecryptMode.groovy b/nifi-toolkit/nifi-toolkit-encrypt-config/src/main/groovy/org/apache/nifi/toolkit/encryptconfig/DecryptMode.groovy new file mode 100644 index 0000000000..4dbef47cd2 --- /dev/null +++ b/nifi-toolkit/nifi-toolkit-encrypt-config/src/main/groovy/org/apache/nifi/toolkit/encryptconfig/DecryptMode.groovy @@ -0,0 +1,327 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.toolkit.encryptconfig + +import org.apache.commons.cli.HelpFormatter +import org.apache.nifi.properties.AESSensitivePropertyProvider +import org.apache.nifi.properties.SensitivePropertyProvider +import org.apache.nifi.toolkit.encryptconfig.util.BootstrapUtil +import org.apache.nifi.toolkit.encryptconfig.util.PropertiesEncryptor +import org.apache.nifi.toolkit.encryptconfig.util.ToolUtilities +import org.apache.nifi.toolkit.encryptconfig.util.XmlEncryptor +import org.apache.nifi.util.console.TextDevices +import org.slf4j.Logger +import org.slf4j.LoggerFactory + +class DecryptMode implements ToolMode { + + private static final Logger logger = LoggerFactory.getLogger(DecryptMode.class) + + static enum FileType { + properties, + xml + } + + CliBuilder cli + boolean verboseEnabled + + DecryptMode() { + cli = cliBuilder() + verboseEnabled = false + } + + void printUsage(String message = "") { + if (message) { + System.out.println(message) + System.out.println() + } + cli.usage() + } + + void printUsageAndExit(String message = "", int exitStatusCode) { + printUsage(message) + System.exit(exitStatusCode) + } + + @Override + void run(String[] args) { + try { + + def options = cli.parse(args) + + if (!options || options.h) { + printUsageAndExit("", EncryptConfigMain.EXIT_STATUS_OTHER) + } + + if (options.v) { + verboseEnabled = true + } + EncryptConfigLogger.configureLogger(verboseEnabled) + + DecryptConfiguration config = new DecryptConfiguration(options) + + run(config) + + } catch (Exception e) { + if (verboseEnabled) { + logger.error("Encountered an error: ${e.getMessage()}", e) + } + printUsageAndExit(e.getMessage(), EncryptConfigMain.EXIT_STATUS_FAILURE) + } + } + + void run(DecryptConfiguration config) throws Exception { + + if (!config.fileType) { + + // Try to load the input file to auto-detect the file type + boolean isPropertiesFile = PropertiesEncryptor.supportsFile(config.inputFilePath) + + boolean isXmlFile = XmlEncryptor.supportsFile(config.inputFilePath) + + if (ToolUtilities.isExactlyOneTrue(isPropertiesFile, isXmlFile)) { + if (isPropertiesFile) { + config.fileType = FileType.properties + logger.debug("Auto-detection of input file type determined the type to be: ${FileType.properties}") + } + if (isXmlFile) { + config.fileType = FileType.xml + logger.debug("Auto-detection of input file type determined the type to be: ${FileType.xml}") + } + } + + // Could we successfully auto-detect? + if (!config.fileType) { + throw new RuntimeException("Auto-detection of input file type failed. Please re-run the tool specifying the file type with the -t/--fileType flag.") + } + } + + String decryptedSerializedContent = null + switch (config.fileType) { + + case FileType.properties: + PropertiesEncryptor propertiesEncryptor = new PropertiesEncryptor(null, config.decryptionProvider) + Properties properties = propertiesEncryptor.loadFile(config.inputFilePath) + properties = propertiesEncryptor.decrypt(properties) + decryptedSerializedContent = propertiesEncryptor.serializePropertiesAndPreserveFormatIfPossible(properties, config.inputFilePath) + break + + case FileType.xml: + XmlEncryptor xmlEncryptor = new XmlEncryptor(null, config.decryptionProvider) { + @Override + List serializeXmlContentAndPreserveFormat(String updatedXmlContent, String originalXmlContent) { + // For decrypting unknown, generic XML, this tool will not support preserving the format + return updatedXmlContent.split("\n") + } + } + + String xmlContent = xmlEncryptor.loadXmlFile(config.inputFilePath) + xmlContent = xmlEncryptor.decrypt(xmlContent) + decryptedSerializedContent = xmlEncryptor.serializeXmlContentAndPreserveFormatIfPossible(xmlContent, config.inputFilePath) + break + + default: + throw new RuntimeException("Unsupported file type '${config.fileType}'") + } + + if (!decryptedSerializedContent) { + throw new RuntimeException("Failed to load and decrypt input file.") + } + + if (config.outputToFile) { + try { + File outputFile = new File(config.outputFilePath) + if (ToolUtilities.isSafeToWrite(outputFile)) { + outputFile.text = decryptedSerializedContent + logger.info("Wrote decrypted file contents to '${config.outputFilePath}'") + } + } catch (IOException e) { + throw new RuntimeException("Encountered an exception writing the decrypted content to '${config.outputFilePath}': ${e.getMessage()}", e) + } + } else { + System.out.println(decryptedSerializedContent) + } + + } + + private CliBuilder cliBuilder() { + + String usage = "${EncryptConfigMain.class.getCanonicalName()} decrypt [options] file" + + int formatWidth = EncryptConfigMain.HELP_FORMAT_WIDTH + HelpFormatter formatter = new HelpFormatter() + formatter.setWidth(formatWidth) + formatter.setOptionComparator(null) // preserve order of options below in help text + + CliBuilder cli = new CliBuilder( + usage: usage, + width: formatWidth, + formatter: formatter, + stopAtNonOption: false) + + cli.h(longOpt: 'help', 'Show usage information (this message)') + cli.v(longOpt: 'verbose', 'Enables verbose mode (off by default)') + + // Options for the password or key or bootstrap.conf + cli.p(longOpt: 'password', + args: 1, + argName: 'password', + optionalArg: true, + 'Use a password to derive the key to decrypt the input file. If an argument is not provided to this flag, interactive mode will be triggered to prompt the user to enter the password.') + cli.k(longOpt: 'key', + args: 1, + argName: 'keyhex', + optionalArg: true, + 'Use a raw hexadecimal key to decrypt the input file. If an argument is not provided to this flag, interactive mode will be triggered to prompt the user to enter the key.') + cli.b(longOpt: 'bootstrapConf', + args: 1, + argName: 'file', + 'Use a bootstrap.conf file containing the master key to decrypt the input file (as an alternative to -p or -k)') + + cli.o(longOpt: 'output', + args: 1, + argName: 'file', + 'Specify an output file. If omitted, Standard Out is used. Output file can be set to the input file to decrypt the file in-place.') + + return cli + + } + + static class DecryptConfiguration implements Configuration { + + OptionAccessor rawOptions + + Configuration.KeySource keySource + String key + SensitivePropertyProvider decryptionProvider + String inputBootstrapPath + + FileType fileType + + String inputFilePath + + boolean outputToFile = false + String outputFilePath + + DecryptConfiguration() { + } + + DecryptConfiguration(OptionAccessor options) { + this.rawOptions = options + + validateOptions() + determineInputFileFromRemainingArgs() + + determineKey() + if (!key) { + throw new RuntimeException("Failed to configure tool, could not determine key.") + } + decryptionProvider = new AESSensitivePropertyProvider(key) + + if (rawOptions.t) { + fileType = FileType.valueOf(rawOptions.t) + } + + if (rawOptions.o) { + outputToFile = true + outputFilePath = rawOptions.o + } + } + + private void validateOptions() { + + String validationFailedMessage = null + + if (!rawOptions.b && !rawOptions.p && !rawOptions.k) { + validationFailedMessage = "-p, -k, or -b is required in order to determine the master key to use for decryption." + } + + if (validationFailedMessage) { + throw new RuntimeException("Invalid options: " + validationFailedMessage) + } + + } + + private void determineInputFileFromRemainingArgs() { + String[] remainingArgs = this.rawOptions.getInner().getArgs() + if (remainingArgs.length == 0) { + throw new RuntimeException("Missing argument: Input file must be provided.") + } else if (remainingArgs.length > 1) { + throw new RuntimeException("Too many arguments: Please specify exactly one input file in addition to the options.") + } + this.inputFilePath = remainingArgs[0] + } + + private void determineKey() { + + boolean usingPassword = false + boolean usingRawKeyHex = false + boolean usingBootstrapKey = false + + if (rawOptions.p) { + usingPassword = true + } + if (rawOptions.k) { + usingRawKeyHex = true + } + if (rawOptions.b) { + usingBootstrapKey = true + } + + if (!ToolUtilities.isExactlyOneTrue(usingPassword, usingRawKeyHex, usingBootstrapKey)) { + throw new RuntimeException("Invalid options: Only one of [-p, -k, -b] is allowed for specifying the decryption password/key.") + } + + if (usingPassword || usingRawKeyHex) { + String password = null + String keyHex = null + if (usingPassword) { + logger.debug("Using password to derive master key for decryption") + password = rawOptions.getInner().getOptionValue("p") + keySource = Configuration.KeySource.PASSWORD + } else { + logger.debug("Using raw key hex as master key for decryption") + keyHex = rawOptions.getInner().getOptionValue("k") + keySource = Configuration.KeySource.KEY_HEX + } + key = ToolUtilities.determineKey(TextDevices.defaultTextDevice(), keyHex, password, usingPassword) + } else if (usingBootstrapKey) { + inputBootstrapPath = rawOptions.b + logger.debug("Looking in bootstrap conf file ${inputBootstrapPath} for master key for decryption.") + + // first, try to treat the bootstrap file as a NiFi bootstrap.conf + logger.debug("Checking expected NiFi bootstrap.conf format") + key = BootstrapUtil.extractKeyFromBootstrapFile(inputBootstrapPath, BootstrapUtil.NIFI_BOOTSTRAP_KEY_PROPERTY) + + // if the key is still null, try again, this time treating the bootstrap file as a NiFi Registry bootstrap.conf + if (!key) { + logger.debug("Checking expected NiFi Registry bootstrap.conf format") + key = BootstrapUtil.extractKeyFromBootstrapFile(inputBootstrapPath, BootstrapUtil.REGISTRY_BOOTSTRAP_KEY_PROPERTY) + } + + // check we have found the key after trying all bootstrap formats + if (key) { + logger.debug("Master key found in ${inputBootstrapPath}. This key will be used for decryption operations.") + keySource = Configuration.KeySource.BOOTSTRAP_FILE + } else { + logger.warn("Bootstrap Conf flag present, but master key could not be found in ${inputBootstrapPath}.") + } + } + } + + } +} diff --git a/nifi-toolkit/nifi-toolkit-encrypt-config/src/main/groovy/org/apache/nifi/toolkit/encryptconfig/EncryptConfigLogger.groovy b/nifi-toolkit/nifi-toolkit-encrypt-config/src/main/groovy/org/apache/nifi/toolkit/encryptconfig/EncryptConfigLogger.groovy new file mode 100644 index 0000000000..07c95777a9 --- /dev/null +++ b/nifi-toolkit/nifi-toolkit-encrypt-config/src/main/groovy/org/apache/nifi/toolkit/encryptconfig/EncryptConfigLogger.groovy @@ -0,0 +1,83 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.toolkit.encryptconfig + +import org.apache.log4j.LogManager +import org.apache.log4j.PropertyConfigurator +import org.slf4j.Logger +import org.slf4j.LoggerFactory + +class EncryptConfigLogger { + private static final Logger logger = LoggerFactory.getLogger(EncryptConfigLogger.class) + + /** + * Configures the logger. + * + * The nifi-toolkit module uses log4j, which will be configured to append all + * log output to the system STDERR. The log level can be specified using the verboseEnabled + * argument. A value of true will set the log level to DEBUG, a value of + * false will set the log level to INFO. + * + * @param verboseEnabled flag to indicate if verbose mode is enabled, which sets the log level to DEBUG + */ + static configureLogger(boolean verboseEnabled) { + + Properties log4jProps = null + URL log4jPropsPath = this.getClass().getResource("log4j.properties") + if (log4jPropsPath) { + try { + log4jPropsPath.withReader { reader -> + log4jProps = new Properties() + log4jProps.load(reader) + } + } catch (IOException e) { + // do nothing, we will fallback to hardcoded defaults below + } + } + + if (!log4jProps) { + log4jProps = defaultProperties() + } + + if (verboseEnabled) { + // Override the log level for this package. For this to work as intended, this class must belong + // to the same package (or a parent package) of all the encrypt-config classes + log4jProps.put("log4j.logger." + EncryptConfigLogger.class.package.name, "DEBUG") + } + + LogManager.resetConfiguration() + PropertyConfigurator.configure(log4jProps) + + if (verboseEnabled) { + logger.debug("Verbose mode is enabled (goes to stderr by default).") + } + } + + /** + * A copy of the settings in /src/main/resources/log4j.properties, in case that is not on the classpath at runtime + * @return Properties containing the default properties for Log4j + */ + static Properties defaultProperties() { + Properties defaultProperties = new Properties() + defaultProperties.setProperty("log4j.rootLogger", "INFO,console") + defaultProperties.setProperty("log4j.appender.console", "org.apache.log4j.ConsoleAppender") + defaultProperties.setProperty("log4j.appender.console.Target", "System.err") + defaultProperties.setProperty("log4j.appender.console.layout", "org.apache.log4j.PatternLayout") + defaultProperties.setProperty("log4j.appender.console.layout.ConversionPattern", "%d{yyyy-mm-dd HH:mm:ss} %p %c{1}: %m%n") + return defaultProperties + } +} diff --git a/nifi-toolkit/nifi-toolkit-encrypt-config/src/main/groovy/org/apache/nifi/toolkit/encryptconfig/EncryptConfigMain.groovy b/nifi-toolkit/nifi-toolkit-encrypt-config/src/main/groovy/org/apache/nifi/toolkit/encryptconfig/EncryptConfigMain.groovy new file mode 100644 index 0000000000..e6ce68e3cc --- /dev/null +++ b/nifi-toolkit/nifi-toolkit-encrypt-config/src/main/groovy/org/apache/nifi/toolkit/encryptconfig/EncryptConfigMain.groovy @@ -0,0 +1,138 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.toolkit.encryptconfig + +import org.apache.commons.cli.HelpFormatter +import org.apache.commons.cli.Options +import org.apache.nifi.properties.ConfigEncryptionTool +import org.bouncycastle.jce.provider.BouncyCastleProvider +import org.slf4j.Logger +import org.slf4j.LoggerFactory + +import java.security.Security + +class EncryptConfigMain { + + private static final Logger logger = LoggerFactory.getLogger(EncryptConfigMain.class) + + static final int EXIT_STATUS_SUCCESS = 0 + static final int EXIT_STATUS_FAILURE = -1 + static final int EXIT_STATUS_OTHER = 1 + + static final String NIFI_REGISTRY_OPT = "nifiRegistry" + static final String NIFI_REGISTRY_FLAG = "--${NIFI_REGISTRY_OPT}".toString() + static final String DECRYPT_OPT = "decrypt" + static final String DECRYPT_FLAG = "--${DECRYPT_OPT}".toString() + + static final int HELP_FORMAT_WIDTH = 160 + + // Access should only be through static methods + private EncryptConfigMain() { + } + + static printUsage(String message = "") { + + if (message) { + System.out.println(message) + System.out.println() + } + + String header = "\nThis tool enables easy encryption and decryption of configuration files for NiFi and its sub-projects. " + + "Unprotected files can be input to this tool to be protected by a key in a manner that is understood by NiFi. " + + "Protected files, along with a key, can be input to this tool to be unprotected, for troubleshooting or automation purposes.\n\n" + + def options = new Options() + options.addOption("h", "help", false, "Show usage information (this message)") + options.addOption(null, NIFI_REGISTRY_OPT, false, "Specifies to target NiFi Registry. When this flag is not included, NiFi is the target.") + + HelpFormatter helpFormatter = new HelpFormatter() + helpFormatter.setWidth(160) + helpFormatter.setOptionComparator(null) + helpFormatter.printHelp("${EncryptConfigMain.class.getCanonicalName()} [-h] [options]", header, options, "\n") + System.out.println() + + helpFormatter.setSyntaxPrefix("") // disable "usage: " prefix for the following outputs + + Options nifiModeOptions = ConfigEncryptionTool.getCliOptions() + helpFormatter.printHelp( + "When targeting NiFi:", + nifiModeOptions, + false) + System.out.println() + + Options nifiRegistryModeOptions = NiFiRegistryMode.getCliOptions() + nifiRegistryModeOptions.addOption(null, DECRYPT_OPT, false, "Can be used with -r to decrypt a previously encrypted NiFi Registry Properties file. Decrypted content is printed to STDOUT.") + helpFormatter.printHelp( + "When targeting NiFi Registry using the ${NIFI_REGISTRY_FLAG} flag:", + nifiRegistryModeOptions, + false) + System.out.println() + + } + + static void printUsageAndExit(String message = "", int exitStatusCode) { + printUsage(message) + System.exit(exitStatusCode) + } + + static void main(String[] args) { + Security.addProvider(new BouncyCastleProvider()) + + if (args.length < 1) { + printUsageAndExit(EXIT_STATUS_FAILURE) + } + + String firstArg = args[0] + + if (["-h", "--help"].contains(firstArg)) { + printUsageAndExit(EXIT_STATUS_OTHER) + } + + try { + List argsList = args + ToolMode toolMode = determineModeFromArgs(argsList) + if (toolMode) { + toolMode.run((String[])argsList.toArray()) + System.exit(EXIT_STATUS_SUCCESS) + } else { + printUsageAndExit(EXIT_STATUS_FAILURE) + } + } catch (Throwable t) { + logger.error("", t) + printUsageAndExit(t.getMessage(), EXIT_STATUS_FAILURE) + } + } + + static ToolMode determineModeFromArgs(List args) { + if (args.contains(NIFI_REGISTRY_FLAG)) { + args.remove(NIFI_REGISTRY_FLAG) + if (args.contains(DECRYPT_FLAG)) { + args.remove(DECRYPT_FLAG) + return new NiFiRegistryDecryptMode() + } else { + return new NiFiRegistryMode() + } + } else { + if (args.contains(DECRYPT_FLAG)) { + logger.error("The ${DECRYPT_FLAG} flag is only available when running in ${NIFI_REGISTRY_FLAG} mode and targeting nifi-registry.properties to allow for the inline TLS status check.") + return null + } else { + return new LegacyMode() + } + } + } +} diff --git a/nifi-toolkit/nifi-toolkit-encrypt-config/src/main/groovy/org/apache/nifi/toolkit/encryptconfig/LegacyMode.groovy b/nifi-toolkit/nifi-toolkit-encrypt-config/src/main/groovy/org/apache/nifi/toolkit/encryptconfig/LegacyMode.groovy new file mode 100644 index 0000000000..09fbfbd030 --- /dev/null +++ b/nifi-toolkit/nifi-toolkit-encrypt-config/src/main/groovy/org/apache/nifi/toolkit/encryptconfig/LegacyMode.groovy @@ -0,0 +1,32 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.toolkit.encryptconfig + +import org.apache.nifi.properties.ConfigEncryptionTool +import org.slf4j.Logger +import org.slf4j.LoggerFactory + +class LegacyMode extends ConfigEncryptionTool implements ToolMode { + + private static final Logger logger = LoggerFactory.getLogger(LegacyMode.class) + + @Override + void run(String[] args) { + logger.debug("Invoking NiFi Config Encryption Tool") + ConfigEncryptionTool.main(args) + } +} diff --git a/nifi-toolkit/nifi-toolkit-encrypt-config/src/main/groovy/org/apache/nifi/toolkit/encryptconfig/NiFiRegistryDecryptMode.groovy b/nifi-toolkit/nifi-toolkit-encrypt-config/src/main/groovy/org/apache/nifi/toolkit/encryptconfig/NiFiRegistryDecryptMode.groovy new file mode 100644 index 0000000000..18d773c080 --- /dev/null +++ b/nifi-toolkit/nifi-toolkit-encrypt-config/src/main/groovy/org/apache/nifi/toolkit/encryptconfig/NiFiRegistryDecryptMode.groovy @@ -0,0 +1,124 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.toolkit.encryptconfig + +import org.apache.nifi.properties.AESSensitivePropertyProvider +import org.apache.nifi.toolkit.encryptconfig.util.BootstrapUtil +import org.apache.nifi.toolkit.encryptconfig.util.ToolUtilities +import org.slf4j.Logger +import org.slf4j.LoggerFactory + +/** + * A special DecryptMode that can run using NiFiRegistry CLI Options + */ +class NiFiRegistryDecryptMode extends DecryptMode { + + private static final Logger logger = LoggerFactory.getLogger(NiFiRegistryDecryptMode.class) + + CliBuilder cli + boolean verboseEnabled + + NiFiRegistryDecryptMode() { + cli = NiFiRegistryMode.cliBuilder() + verboseEnabled = false + } + + @Override + void run(String[] args) { + try { + + def options = cli.parse(args) + + if (!options || options.h) { + EncryptConfigMain.printUsageAndExit("", EncryptConfigMain.EXIT_STATUS_OTHER) + } + + if (options.v) { + verboseEnabled = true + } + EncryptConfigLogger.configureLogger(verboseEnabled) + + DecryptConfiguration config = new DecryptConfiguration() + + /* Invalid fields when used with --decrypt: */ + def invalidDecryptOptions = ["R", "i", "I", "a", "A", "oldPassword", "oldKey"] + def presentInvalidOptions = Arrays.stream(options.getInner().getOptions()).findAll { + invalidDecryptOptions.contains(it.getOpt()) + } + if (presentInvalidOptions.size() > 0) { + throw new RuntimeException("Invalid options: ${EncryptConfigMain.DECRYPT_FLAG} cannot be used with [${presentInvalidOptions.join(", ")}]. It should only be used with -r and one of [-p, -k, -b].") + } + + /* Required fields when using --decrypt */ + // registryPropertiesFile (-r) + if (!options.r) { + throw new RuntimeException("Invalid options: Input nifiRegistryProperties (-r) is required when using --decrypt") + } + config.inputFilePath = options.r + config.fileType = FileType.properties // disables auto-detection, which is still experimental + + // one of [-p, -k, -b] + String keyHex = null + String password = null + config.keySource = null + if (options.p) { + config.keySource = Configuration.KeySource.PASSWORD + password = options.getInner().getOptionValue("p") + } + if (options.k) { + if (config.keySource != null) { + throw new RuntimeException("Invalid options: Only one of [-b, -p, -k] is allowed for specifying the decryption password/key.") + } + config.keySource = Configuration.KeySource.KEY_HEX + keyHex = options.getInner().getOptionValue("k") + } + + if (config.keySource) { + config.key = ToolUtilities.determineKey(keyHex, password, Configuration.KeySource.PASSWORD == config.keySource) + } + + if (options.b) { + if (config.keySource != null) { + throw new RuntimeException("Invalid options: Only one of [-b, -p, -k] is allowed for specifying the decryption password/key.") + } + config.keySource = Configuration.KeySource.BOOTSTRAP_FILE + config.inputBootstrapPath = options.b + + logger.debug("Checking expected NiFi Registry bootstrap.conf format") + config.key = BootstrapUtil.extractKeyFromBootstrapFile(config.inputBootstrapPath, BootstrapUtil.REGISTRY_BOOTSTRAP_KEY_PROPERTY) + + // check we have found the key + if (config.key) { + logger.debug("Master key found in ${config.inputBootstrapPath}. This key will be used for decryption operations.") + } else { + logger.warn("Bootstrap Conf flag present, but master key could not be found in ${config.inputBootstrapPath}.") + } + } + + config.decryptionProvider = new AESSensitivePropertyProvider(config.key) + + run(config) + + } catch (Exception e) { + if (verboseEnabled) { + logger.error("Encountered an error: ${e.getMessage()}", e) + } + EncryptConfigMain.printUsageAndExit(e.getMessage(), EncryptConfigMain.EXIT_STATUS_FAILURE) + } + } + +} diff --git a/nifi-toolkit/nifi-toolkit-encrypt-config/src/main/groovy/org/apache/nifi/toolkit/encryptconfig/NiFiRegistryMode.groovy b/nifi-toolkit/nifi-toolkit-encrypt-config/src/main/groovy/org/apache/nifi/toolkit/encryptconfig/NiFiRegistryMode.groovy new file mode 100644 index 0000000000..2034f002eb --- /dev/null +++ b/nifi-toolkit/nifi-toolkit-encrypt-config/src/main/groovy/org/apache/nifi/toolkit/encryptconfig/NiFiRegistryMode.groovy @@ -0,0 +1,382 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.toolkit.encryptconfig + +import org.apache.commons.cli.HelpFormatter +import org.apache.commons.cli.Options +import org.apache.http.annotation.Experimental +import org.apache.nifi.properties.AESSensitivePropertyProvider +import org.apache.nifi.properties.SensitivePropertyProvider +import org.apache.nifi.toolkit.encryptconfig.util.BootstrapUtil +import org.apache.nifi.toolkit.encryptconfig.util.NiFiRegistryAuthorizersXmlEncryptor +import org.apache.nifi.toolkit.encryptconfig.util.NiFiRegistryIdentityProvidersXmlEncryptor +import org.apache.nifi.toolkit.encryptconfig.util.NiFiRegistryPropertiesEncryptor +import org.apache.nifi.toolkit.encryptconfig.util.ToolUtilities +import org.apache.nifi.util.console.TextDevices +import org.slf4j.Logger +import org.slf4j.LoggerFactory + +class NiFiRegistryMode implements ToolMode { + + private static final Logger logger = LoggerFactory.getLogger(NiFiRegistryMode.class) + + CliBuilder cli + boolean verboseEnabled + + NiFiRegistryMode() { + cli = cliBuilder() + verboseEnabled = false + } + + @Override + void run(String[] args) { + try { + + def options = cli.parse(args) + + if (!options || options.h) { + EncryptConfigMain.printUsageAndExit("", EncryptConfigMain.EXIT_STATUS_OTHER) + } + + if (options.v) { + verboseEnabled = true + } + EncryptConfigLogger.configureLogger(verboseEnabled) + + NiFiRegistryConfiguration config = new NiFiRegistryConfiguration(options) + run(config) + + } catch (Exception e) { + if (verboseEnabled) { + logger.error("Encountered an error: ${e.getMessage()}") + } + EncryptConfigMain.printUsageAndExit(e.getMessage(), EncryptConfigMain.EXIT_STATUS_FAILURE) + } + } + + void run(NiFiRegistryConfiguration config) throws Exception { + + if (config.usingPassword) { + logger.info("Using encryption key derived from password.") + } else if (config.usingRawKeyHex) { + logger.info("Using encryption key provided.") + } else if (config.usingBootstrapKey) { + logger.info("Using encryption key from input bootstrap.conf.") + } + + logger.debug("(src) bootstrap.conf: ${config.inputBootstrapPath}") + logger.debug("(dest) bootstrap.conf: ${config.outputBootstrapPath}") + logger.debug("(src) nifi-registry.properties: ${config.inputNiFiRegistryPropertiesPath}") + logger.debug("(dest) nifi-registry.properties: ${config.outputNiFiRegistryPropertiesPath}") + logger.debug("(src) identity-providers.xml: ${config.inputIdentityProvidersPath}") + logger.debug("(dest) identity-providers.xml: ${config.outputIdentityProvidersPath}") + logger.debug("(src) authorizers.xml: ${config.inputAuthorizersPath}") + logger.debug("(dest) authorizers.xml: ${config.outputAuthorizersPath}") + + Properties niFiRegistryProperties = null + if (config.handlingNiFiRegistryProperties) { + try { + logger.debug("Encrypting NiFi Registry Properties") + niFiRegistryProperties = config.propertiesEncryptor.loadFile(config.inputNiFiRegistryPropertiesPath) + // if properties are not protected, then the call to decrypt is a no-op + niFiRegistryProperties = config.propertiesEncryptor.decrypt(niFiRegistryProperties) + niFiRegistryProperties = config.propertiesEncryptor.encrypt(niFiRegistryProperties) + } catch (Exception e) { + throw new RuntimeException("Encountered error trying to load and encrypt NiFi Registry Properties in ${config.inputNiFiRegistryPropertiesPath}: ${e.getMessage()}", e) + } + } + + String identityProvidersXml = null + if (config.handlingIdentityProviders) { + try { + logger.debug("Encrypting Identity Providers XML") + identityProvidersXml = config.identityProvidersXmlEncryptor.loadXmlFile(config.inputIdentityProvidersPath) + // if xml is not protected, then the call to decrypt is a no-op + identityProvidersXml = config.identityProvidersXmlEncryptor.decrypt(identityProvidersXml) + identityProvidersXml = config.identityProvidersXmlEncryptor.encrypt(identityProvidersXml) + } catch (Exception e) { + throw new RuntimeException("Encountered error trying to load and encrypt Identity Providers XML in ${config.inputIdentityProvidersPath}: ${e.getMessage()}", e) + } + } + + String authorizersXml = null + if (config.handlingAuthorizers) { + try { + logger.debug("Encrypting Authorizers XML") + authorizersXml = config.authorizersXmlEncryptor.loadXmlFile(config.inputAuthorizersPath) + // if xml is not protected, then the call to decrypt is a no-op + authorizersXml = config.authorizersXmlEncryptor.decrypt(authorizersXml) + authorizersXml = config.authorizersXmlEncryptor.encrypt(authorizersXml) + } catch (Exception e) { + throw new RuntimeException("Encountered error trying to load and encrypt Authorizers XML in ${config.inputAuthorizersPath}: ${e.getMessage()}", e) + } + } + + try { + // Do this as part of a transaction? + synchronized (this) { + + if (config.writingKeyToBootstrap) { + BootstrapUtil.writeKeyToBootstrapFile(config.encryptionKey, BootstrapUtil.REGISTRY_BOOTSTRAP_KEY_PROPERTY, config.outputBootstrapPath, config.inputBootstrapPath) + logger.info("Updated bootstrap config file with master key: ${config.outputBootstrapPath}") + } + + if (config.handlingNiFiRegistryProperties) { + config.propertiesEncryptor.write(niFiRegistryProperties, config.outputNiFiRegistryPropertiesPath, config.inputNiFiRegistryPropertiesPath) + logger.info("Updated NiFi Registry Properties file with protected values: ${config.outputNiFiRegistryPropertiesPath}") + } + if (config.handlingIdentityProviders) { + config.identityProvidersXmlEncryptor.writeXmlFile(identityProvidersXml, config.outputIdentityProvidersPath, config.inputIdentityProvidersPath) + logger.info("Updated Identity Providers XML file with protected values: ${config.outputIdentityProvidersPath}") + } + if (config.handlingAuthorizers) { + config.authorizersXmlEncryptor.writeXmlFile(authorizersXml, config.outputAuthorizersPath, config.inputAuthorizersPath) + logger.info("Updated Authorizers XML file with protected values: ${config.outputAuthorizersPath}") + } + } + } catch (Exception e) { + throw new RuntimeException("Encountered error while writing the output files: ${e.getMessage()}", e) + } + } + + static Options getCliOptions() { + return cliBuilder().options + } + + static CliBuilder cliBuilder() { + + String usage = "${NiFiRegistryMode.class.getCanonicalName()} [options]" + + int formatWidth = EncryptConfigMain.HELP_FORMAT_WIDTH + HelpFormatter formatter = new HelpFormatter() + formatter.setWidth(formatWidth) + formatter.setOptionComparator(null) // preserve order of options below in help text + + CliBuilder cli = new CliBuilder( + usage: usage, + width: formatWidth, + formatter: formatter) + + cli.h(longOpt: 'help', 'Show usage information (this message)') + cli.v(longOpt: 'verbose', 'Sets verbose mode (default false)') + + // Options for the new password or key + cli.p(longOpt: 'password', + args: 1, + argName: 'password', + optionalArg: true, + 'Protect the files using a password-derived key. If an argument is not provided to this flag, interactive mode will be triggered to prompt the user to enter the password.') + cli.k(longOpt: 'key', + args: 1, + argName: 'keyhex', + optionalArg: true, + 'Protect the files using a raw hexadecimal key. If an argument is not provided to this flag, interactive mode will be triggered to prompt the user to enter the key.') + + // Options for the old password or key, if running the tool to migrate keys + cli._(longOpt: 'oldPassword', + args: 1, + argName: 'password', + 'If the input files are already protected using a password-derived key, this specifies the old password so that the files can be unprotected before re-protecting.') + cli._(longOpt: 'oldKey', + args: 1, + argName: 'keyhex', + 'If the input files are already protected using a key, this specifies the raw hexadecimal key so that the files can be unprotected before re-protecting.') + + // Options for output bootstrap.conf file + cli.b(longOpt: 'bootstrapConf', + args: 1, + argName: 'file', + 'The bootstrap.conf file containing no master key or an existing master key. If a new password or key is specified (using -p or -k) and no output bootstrap.conf file is specified, then this file will be overwritten to persist the new master key.') + cli.B(longOpt: 'outputBootstrapConf', + args: 1, + argName: 'file', + 'The destination bootstrap.conf file to persist master key. If specified, the input bootstrap.conf will not be modified.') + + // Options for input/output nifi-registry.properties files + cli.r(longOpt: 'nifiRegistryProperties', + args: 1, + argName: 'file', + 'The nifi-registry.properties file containing unprotected config values, overwritten if no output file specified.') + cli.R(longOpt: 'outputNifiRegistryProperties', + args: 1, + argName: 'file', + 'The destination nifi-registry.properties file containing protected config values.') + + // Options for input/output authorizers.xml files + cli.a(longOpt: 'authorizersXml', + args: 1, + argName: 'file', + 'The authorizers.xml file containing unprotected config values, overwritten if no output file specified.') + cli.A(longOpt: 'outputAuthorizersXml', + args: 1, + argName: 'file', + 'The destination authorizers.xml file containing protected config values.') + + // Options for input/output identity-providers.xml files + cli.i(longOpt: 'identityProvidersXml', + args: 1, + argName: 'file', + 'The identity-providers.xml file containing unprotected config values, overwritten if no output file specified.') + cli.I(longOpt: 'outputIdentityProvidersXml', + args: 1, + argName: 'file', + 'The destination identity-providers.xml file containing protected config values.') + + return cli + + } + + static class NiFiRegistryConfiguration implements Configuration { + + OptionAccessor rawOptions + + boolean usingRawKeyHex + boolean usingPassword + boolean usingBootstrapKey + + String encryptionKey + String decryptionKey + + SensitivePropertyProvider encryptionProvider + SensitivePropertyProvider decryptionProvider + + boolean writingKeyToBootstrap = false + String inputBootstrapPath + String outputBootstrapPath + + boolean handlingNiFiRegistryProperties = false + String inputNiFiRegistryPropertiesPath + String outputNiFiRegistryPropertiesPath + NiFiRegistryPropertiesEncryptor propertiesEncryptor + + boolean handlingIdentityProviders = false + String inputIdentityProvidersPath + String outputIdentityProvidersPath + NiFiRegistryIdentityProvidersXmlEncryptor identityProvidersXmlEncryptor + + boolean handlingAuthorizers = false + String inputAuthorizersPath + String outputAuthorizersPath + NiFiRegistryAuthorizersXmlEncryptor authorizersXmlEncryptor + + NiFiRegistryConfiguration() { + } + + NiFiRegistryConfiguration(OptionAccessor options) { + this.rawOptions = options + + validateOptions() + + // Set input bootstrap.conf path + inputBootstrapPath = rawOptions.b + + // Determine key for encryption (required) + determineEncryptionKey() + if (!encryptionKey) { + throw new RuntimeException("Failed to configure tool, could not determine encryption key. Must provide -p, -k, or -b. If using -b, bootstrap.conf argument must already contain master key.") + } + encryptionProvider = new AESSensitivePropertyProvider(encryptionKey) + + // Determine key for decryption (if migrating) + determineDecryptionKey() + if (!decryptionKey) { + logger.debug("No decryption key specified via options, so if any input files require decryption prior to re-encryption (i.e., migration), this tool will fail.") + } + decryptionProvider = decryptionKey ? new AESSensitivePropertyProvider(decryptionKey) : null + + writingKeyToBootstrap = (usingPassword || usingRawKeyHex || rawOptions.B) + if (writingKeyToBootstrap) { + outputBootstrapPath = rawOptions.B ?: inputBootstrapPath + } + + handlingNiFiRegistryProperties = rawOptions.r + if (handlingNiFiRegistryProperties) { + inputNiFiRegistryPropertiesPath = rawOptions.r + outputNiFiRegistryPropertiesPath = rawOptions.R ?: inputNiFiRegistryPropertiesPath + propertiesEncryptor = new NiFiRegistryPropertiesEncryptor(encryptionProvider, decryptionProvider) + } + + handlingIdentityProviders = rawOptions.i + if (handlingIdentityProviders) { + inputIdentityProvidersPath = rawOptions.i + outputIdentityProvidersPath = rawOptions.I ?: inputIdentityProvidersPath + identityProvidersXmlEncryptor = new NiFiRegistryIdentityProvidersXmlEncryptor(encryptionProvider, decryptionProvider) + } + + handlingAuthorizers = rawOptions.a + if (handlingAuthorizers) { + inputAuthorizersPath = rawOptions.a + outputAuthorizersPath = rawOptions.A ?: inputAuthorizersPath + authorizersXmlEncryptor = new NiFiRegistryAuthorizersXmlEncryptor(encryptionProvider, decryptionProvider) + } + + } + + private void validateOptions() { + + String validationFailedMessage = null + + if (!rawOptions.b) { + validationFailedMessage = "-b flag for bootstrap.conf is required." + if (rawOptions.B) { + validationFailedMessage += " Input bootsrap.conf will be used as template for output bootstrap.conf" + } else if (rawOptions.p || rawOptions.k) { + validationFailedMessage = " Encryption key will be persisted to bootstrap.conf" + } + } + + if (validationFailedMessage) { + throw new RuntimeException("Invalid options: " + validationFailedMessage) + } + + } + + private void determineEncryptionKey() { + if (rawOptions.p || rawOptions.k) { + String password = null + String keyHex = null + if (rawOptions.p) { + logger.debug("Attempting to generate key from password.") + usingPassword = true + password = rawOptions.getInner().getOptionValue("p") + } else { + usingRawKeyHex = true + keyHex = rawOptions.getInner().getOptionValue("k") + } + encryptionKey = ToolUtilities.determineKey(TextDevices.defaultTextDevice(), keyHex, password, usingPassword) + } else if (rawOptions.b) { + logger.debug("Attempting to read master key from input bootstrap.conf file.") + usingBootstrapKey = true + encryptionKey = BootstrapUtil.extractKeyFromBootstrapFile(inputBootstrapPath, BootstrapUtil.REGISTRY_BOOTSTRAP_KEY_PROPERTY) + if (!encryptionKey) { + logger.warn("-b specified without -p or -k, but the input bootstrap.conf file did not contain a master key.") + } + } + } + + private String determineDecryptionKey() { + if (rawOptions.oldPassword) { + logger.debug("Attempting to generate decryption key (for migration) from old password.") + encryptionKey = ToolUtilities.determineKey(TextDevices.defaultTextDevice(), null, rawOptions.oldPassword, true) + } else if (rawOptions.oldKey) { + decryptionKey = rawOptions.oldKey + } + } + + } + +} diff --git a/nifi-toolkit/nifi-toolkit-encrypt-config/src/main/groovy/org/apache/nifi/toolkit/encryptconfig/ToolMode.groovy b/nifi-toolkit/nifi-toolkit-encrypt-config/src/main/groovy/org/apache/nifi/toolkit/encryptconfig/ToolMode.groovy new file mode 100644 index 0000000000..3b49718639 --- /dev/null +++ b/nifi-toolkit/nifi-toolkit-encrypt-config/src/main/groovy/org/apache/nifi/toolkit/encryptconfig/ToolMode.groovy @@ -0,0 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.toolkit.encryptconfig + +interface ToolMode { + + void run(String[] args) + +} diff --git a/nifi-toolkit/nifi-toolkit-encrypt-config/src/main/groovy/org/apache/nifi/toolkit/encryptconfig/util/BootstrapUtil.groovy b/nifi-toolkit/nifi-toolkit-encrypt-config/src/main/groovy/org/apache/nifi/toolkit/encryptconfig/util/BootstrapUtil.groovy new file mode 100644 index 0000000000..85f0ebd59d --- /dev/null +++ b/nifi-toolkit/nifi-toolkit-encrypt-config/src/main/groovy/org/apache/nifi/toolkit/encryptconfig/util/BootstrapUtil.groovy @@ -0,0 +1,132 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.toolkit.encryptconfig.util + +import org.slf4j.Logger +import org.slf4j.LoggerFactory + +class BootstrapUtil { + + static final String NIFI_BOOTSTRAP_KEY_PROPERTY = "nifi.bootstrap.sensitive.key"; + static final String REGISTRY_BOOTSTRAP_KEY_PROPERTY = "nifi.registry.bootstrap.sensitive.key"; + + private static final Logger logger = LoggerFactory.getLogger(BootstrapUtil.class) + + private static final String BOOTSTRAP_KEY_COMMENT = "# Master key in hexadecimal format for encrypted sensitive configuration values" + + /** + * Tries to load keyHex from input bootstrap.conf + * + * @return keyHex, if present in input bootstrap file; otherwise, null + */ + static String extractKeyFromBootstrapFile(String inputBootstrapPath, String bootstrapKeyPropertyName) throws IOException { + + File inputBootstrapConfFile = new File(inputBootstrapPath) + if (!(inputBootstrapPath && ToolUtilities.canRead(inputBootstrapConfFile))) { + throw new IOException("The bootstrap.conf file at ${inputBootstrapPath} must exist and be readable by the user running this tool") + } + + String keyValue = null + try { + List lines = inputBootstrapConfFile.readLines() + int keyLineIndex = lines.findIndexOf { it.startsWith("${bootstrapKeyPropertyName}=") } + + if (keyLineIndex != -1) { + logger.debug("The key property was detected in bootstrap.conf") + String keyLine = lines[keyLineIndex] + keyValue = keyLine.split("=", 2)[1] + if (keyValue.trim().isEmpty()) { + keyValue = null + } + } else { + logger.debug("The key property was not detected in input bootstrap.conf.") + } + + + } catch (IOException e) { + logger.error("Encountered an exception reading the master key from the input bootstrap.conf file: ${e.getMessage()}") + throw e + } + + return keyValue; + + } + + /** + * Writes key to output bootstrap.conf + * + * @param keyHex + */ + static void writeKeyToBootstrapFile(String keyHex, String bootstrapKeyPropertyName, String outputBootstrapPath, String inputBootstrapPath) throws IOException { + File inputBootstrapConfFile = new File(inputBootstrapPath) + File outputBootstrapConfFile = new File(outputBootstrapPath) + + if (!ToolUtilities.canRead(inputBootstrapConfFile)) { + throw new IOException("The bootstrap.conf file at ${inputBootstrapPath} must exist and be readable by the user running this tool") + } + + if (!ToolUtilities.isSafeToWrite(outputBootstrapConfFile)) { + throw new IOException("The bootstrap.conf file at ${outputBootstrapPath} must exist and be readable and writable by the user running this tool") + } + + try { + List lines = inputBootstrapConfFile.readLines() + + updateBootstrapContentsWithKey(lines, keyHex, bootstrapKeyPropertyName) + + // Write the updated values to the output file + outputBootstrapConfFile.text = lines.join("\n") + } catch (IOException e) { + logger.error("Encountered an exception reading the master key from the input bootstrap.conf file: ${e.getMessage()}") + throw e + } + } + + + /** + * Accepts the lines of the {@code bootstrap.conf} file as a {@code List } and updates or adds the key property (and associated comment). + * + * @param lines the lines of the bootstrap file + * @return the updated lines + */ + private static List updateBootstrapContentsWithKey(List lines, String newKeyHex, String bootstrapKeyPropertyName) { + String keyLine = "${bootstrapKeyPropertyName}=${newKeyHex}" + // Try to locate the key property line + int keyLineIndex = lines.findIndexOf { it.startsWith("${bootstrapKeyPropertyName}=") } + + // If it was found, update inline + if (keyLineIndex != -1) { + logger.debug("The key property was detected in bootstrap.conf") + lines[keyLineIndex] = keyLine + logger.debug("The bootstrap key value was updated") + + // Ensure the comment explaining the property immediately precedes it (check for edge case where key is first line) + int keyCommentLineIndex = keyLineIndex > 0 ? keyLineIndex - 1 : 0 + if (lines[keyCommentLineIndex] != BOOTSTRAP_KEY_COMMENT) { + lines.add(keyCommentLineIndex, BOOTSTRAP_KEY_COMMENT) + logger.debug("A comment explaining the bootstrap key property was added") + } + } else { + // If it wasn't present originally, add the comment and key property + lines.addAll(["\n", BOOTSTRAP_KEY_COMMENT, keyLine]) + logger.debug("The key property was not detected in bootstrap.conf so it was added along with a comment explaining it") + } + + return lines + } + +} diff --git a/nifi-toolkit/nifi-toolkit-encrypt-config/src/main/groovy/org/apache/nifi/toolkit/encryptconfig/util/NiFiPropertiesEncryptor.groovy b/nifi-toolkit/nifi-toolkit-encrypt-config/src/main/groovy/org/apache/nifi/toolkit/encryptconfig/util/NiFiPropertiesEncryptor.groovy new file mode 100644 index 0000000000..28c9ee02d7 --- /dev/null +++ b/nifi-toolkit/nifi-toolkit-encrypt-config/src/main/groovy/org/apache/nifi/toolkit/encryptconfig/util/NiFiPropertiesEncryptor.groovy @@ -0,0 +1,54 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.toolkit.encryptconfig.util + +import org.apache.nifi.properties.ProtectedNiFiProperties +import org.apache.nifi.properties.SensitivePropertyProvider +import org.slf4j.Logger +import org.slf4j.LoggerFactory + +import java.util.regex.Pattern + +class NiFiPropertiesEncryptor extends PropertiesEncryptor { + + private static final Logger logger = LoggerFactory.getLogger(NiFiPropertiesEncryptor.class) + + private static final String ADDITIONAL_SENSITIVE_PROPERTIES_KEY = ProtectedNiFiProperties.ADDITIONAL_SENSITIVE_PROPERTIES_KEY + private static final String[] DEFAULT_SENSITIVE_PROPERTIES = ProtectedNiFiProperties.DEFAULT_SENSITIVE_PROPERTIES + + NiFiPropertiesEncryptor(SensitivePropertyProvider encryptionProvider, SensitivePropertyProvider decryptionProvider) { + super(encryptionProvider, decryptionProvider) + } + + @Override + Properties encrypt(Properties properties) { + Set propertiesToEncrypt = new HashSet<>() + propertiesToEncrypt.addAll(DEFAULT_SENSITIVE_PROPERTIES) + propertiesToEncrypt.addAll(getAdditionalSensitivePropertyKeys(properties)) + + return encrypt(properties, propertiesToEncrypt) + } + + private static String[] getAdditionalSensitivePropertyKeys(Properties properties) { + String rawAdditionalSensitivePropertyKeys = properties.getProperty(ADDITIONAL_SENSITIVE_PROPERTIES_KEY) + if (!rawAdditionalSensitivePropertyKeys) { + return [] + } + return rawAdditionalSensitivePropertyKeys.split(Pattern.quote(",")) + } + +} diff --git a/nifi-toolkit/nifi-toolkit-encrypt-config/src/main/groovy/org/apache/nifi/toolkit/encryptconfig/util/NiFiRegistryAuthorizersXmlEncryptor.groovy b/nifi-toolkit/nifi-toolkit-encrypt-config/src/main/groovy/org/apache/nifi/toolkit/encryptconfig/util/NiFiRegistryAuthorizersXmlEncryptor.groovy new file mode 100644 index 0000000000..102ad9e1ee --- /dev/null +++ b/nifi-toolkit/nifi-toolkit-encrypt-config/src/main/groovy/org/apache/nifi/toolkit/encryptconfig/util/NiFiRegistryAuthorizersXmlEncryptor.groovy @@ -0,0 +1,103 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.toolkit.encryptconfig.util + +import groovy.xml.XmlUtil +import org.apache.nifi.properties.SensitivePropertyProvider +import org.slf4j.Logger +import org.slf4j.LoggerFactory +import org.xml.sax.SAXException + +class NiFiRegistryAuthorizersXmlEncryptor extends XmlEncryptor { + + private static final Logger logger = LoggerFactory.getLogger(NiFiRegistryAuthorizersXmlEncryptor.class) + + static final String LDAP_USER_GROUP_PROVIDER_CLASS = "org.apache.nifi.registry.security.ldap.tenants.LdapUserGroupProvider" + private static final String LDAP_USER_GROUP_PROVIDER_REGEX = + /(?s)(?:(?!).)*?\s*org\.apache\.nifi\.registry\.security\.ldap\.tenants\.LdapUserGroupProvider.*?<\/userGroupProvider>/ + /* Explanation of LDAP_USER_GROUP_PROVIDER_REGEX: + * (?s) -> single-line mode (i.e., `.` in regex matches newlines) + * -> find occurrence of `` literally (case-sensitive) + * (?: ... ) -> group but do not capture submatch + * (?! ... ) -> negative lookahead + * (?:(?!).)*? -> find everything until a new `` starts. This is for not selecting multiple userGroupProviders in one match + * -> find occurrence of `` literally (case-sensitive) + * \s* -> find any whitespace + * org\.apache\.nifi\.registry\.security\.ldap\.tenants\.LdapUserGroupProvider + * -> find occurrence of `org.apache.nifi.registry.security.ldap.tenants.LdapUserGroupProvider` literally (case-sensitive) + * .*? -> find everything as needed up until and including occurrence of '' + */ + + NiFiRegistryAuthorizersXmlEncryptor(SensitivePropertyProvider encryptionProvider, SensitivePropertyProvider decryptionProvider) { + super(encryptionProvider, decryptionProvider) + } + + /** + * Overrides the super class implementation to marking xml nodes that should be encrypted. + * This is done using logic specific to the authorizers.xml file type targeted by this subclass, + * leveraging knowledge of the XML file structure and which elements are sensitive. + * Sensitive nodes are marked by adding the encryption="none" attribute. + * When all the sensitive values are found and marked, the base class implementation + * is invoked to encrypt them. + * + * @param plainXmlContent the plaintext content of an authorizers.xml file + * @return the comment with sensitive values encrypted and marked with the cipher. + */ + @Override + String encrypt(String plainXmlContent) { + // First, mark the XML nodes to encrypt that are specific to authorizers.xml by adding an attribute encryption="none" + String markedXmlContent = markXmlNodesForEncryption(plainXmlContent, "userGroupProvider", { + it.find { + it.'class' as String == LDAP_USER_GROUP_PROVIDER_CLASS + }.property.findAll { + // Only operate on populated password properties + it.@name =~ "Password" && it.text() + } + }) + + // Now, return the results of the base implementation, which encrypts any node with an encryption="none" attribute + return super.encrypt(markedXmlContent) + } + + List serializeXmlContentAndPreserveFormat(String updatedXmlContent, String originalXmlContent) { + if (updatedXmlContent == originalXmlContent) { + // If nothing was encrypted, e.g., the sensitive properties are commented out or empty, + // then the best thing to do to preserve formatting perspective is to do nothing. + return originalXmlContent.split("\n") + } + + // Find & replace the userGroupProvider element of the updated content in the original contents + try { + def parsedXml = new XmlSlurper().parseText(updatedXmlContent) + def provider = parsedXml.userGroupProvider.find { it.'class' as String == LDAP_USER_GROUP_PROVIDER_CLASS } + if (provider) { + def serializedProvider = new XmlUtil().serialize(provider) + // Remove XML declaration from top + serializedProvider = serializedProvider.replaceFirst(XML_DECLARATION_REGEX, "") + originalXmlContent = originalXmlContent.replaceFirst(LDAP_USER_GROUP_PROVIDER_REGEX, serializedProvider) + return originalXmlContent.split("\n") + } else { + throw new SAXException("No ldap-user-group-provider element found") + } + } catch (SAXException e) { + logger.warn("No userGroupProvider with class ${LDAP_USER_GROUP_PROVIDER_CLASS} found in XML content. " + + "The file could be empty or the element may be missing or commented out") + return originalXmlContent.split("\n") + } + } + +} diff --git a/nifi-toolkit/nifi-toolkit-encrypt-config/src/main/groovy/org/apache/nifi/toolkit/encryptconfig/util/NiFiRegistryIdentityProvidersXmlEncryptor.groovy b/nifi-toolkit/nifi-toolkit-encrypt-config/src/main/groovy/org/apache/nifi/toolkit/encryptconfig/util/NiFiRegistryIdentityProvidersXmlEncryptor.groovy new file mode 100644 index 0000000000..fa6ce65c57 --- /dev/null +++ b/nifi-toolkit/nifi-toolkit-encrypt-config/src/main/groovy/org/apache/nifi/toolkit/encryptconfig/util/NiFiRegistryIdentityProvidersXmlEncryptor.groovy @@ -0,0 +1,102 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.toolkit.encryptconfig.util + +import groovy.xml.XmlUtil +import org.apache.nifi.properties.SensitivePropertyProvider +import org.slf4j.Logger +import org.slf4j.LoggerFactory +import org.xml.sax.SAXException + +class NiFiRegistryIdentityProvidersXmlEncryptor extends XmlEncryptor { + + private static final Logger logger = LoggerFactory.getLogger(NiFiRegistryIdentityProvidersXmlEncryptor.class) + + static final String LDAP_PROVIDER_CLASS = "org.apache.nifi.registry.security.ldap.LdapIdentityProvider" + private static final String LDAP_PROVIDER_REGEX = /(?s)(?:(?!).)*?\s*org\.apache\.nifi\.registry\.security\.ldap\.LdapIdentityProvider.*?<\/provider>/ + /* Explanation of LDAP_PROVIDER_REGEX: + * (?s) -> single-line mode (i.e., `.` in regex matches newlines) + * -> find occurrence of `` literally (case-sensitive) + * (?: ... ) -> group but do not capture submatch + * (?! ... ) -> negative lookahead + * (?:(?!).)*? -> find everything until a new `` starts. This is for not selecting multiple providers in one match + * -> find occurrence of `` literally (case-sensitive) + * \s* -> find any whitespace + * org\.apache\.nifi\.registry\.security\.ldap\.LdapIdentityProvider + * -> find occurrence of `org.apache.nifi.registry.security.ldap.LdapIdentityProvider` literally (case-sensitive) + * .*? -> find everything as needed up until and including occurrence of `` + */ + + NiFiRegistryIdentityProvidersXmlEncryptor(SensitivePropertyProvider encryptionProvider, SensitivePropertyProvider decryptionProvider) { + super(encryptionProvider, decryptionProvider) + } + + /** + * Overrides the super class implementation to marking xml nodes that should be encrypted. + * This is done using logic specific to the identity-providers.xml file type targeted by this + * subclass, leveraging knowledge of the XML file structure and which elements are sensitive. + * Sensitive nodes are marked by adding the encryption="none" attribute. + * When all the sensitive values are found and marked, the base class implementation + * is invoked to encrypt them. + * + * @param plainXmlContent the plaintext content of an identity-providers.xml file + * @return the comment with sensitive values encrypted and marked with the cipher. + */ + @Override + String encrypt(String plainXmlContent) { + // First, mark the XML nodes to encrypt that are specific to authorizers.xml by adding an attribute encryption="none" + String markedXmlContent = markXmlNodesForEncryption(plainXmlContent, "provider", { + it.find { + it.'class' as String == LDAP_PROVIDER_CLASS + }.property.findAll { + // Only operate on populated password properties + it.@name =~ "Password" && it.text() + } + }) + + // Now, return the results of the base implementation, which encrypts any node with an encryption="none" attribute + return super.encrypt(markedXmlContent) + } + + List serializeXmlContentAndPreserveFormat(String updatedXmlContent, String originalXmlContent) { + if (updatedXmlContent == originalXmlContent) { + // If nothing was encrypted, e.g., the sensitive properties are commented out or empty, + // then the best thing to do to preserve formatting perspective is to do nothing. + return originalXmlContent.split("\n") + } + + // Find & replace the provider element of the updated content in the original contents + try { + def parsedXml = new XmlSlurper().parseText(updatedXmlContent) + def provider = parsedXml.provider.find { it.'class' as String == LDAP_PROVIDER_CLASS } + if (provider) { + def serializedProvider = new XmlUtil().serialize(provider) + // Remove XML declaration from top + serializedProvider = serializedProvider.replaceFirst(XML_DECLARATION_REGEX, "") + originalXmlContent = originalXmlContent.replaceFirst(LDAP_PROVIDER_REGEX, serializedProvider) + return originalXmlContent.split("\n") + } else { + throw new SAXException("No ldap-provider found") + } + } catch (SAXException e) { + logger.warn("No provider with class ${LDAP_PROVIDER_CLASS} found in XML content. " + + "The file could be empty or the element may be missing or commented out") + return originalXmlContent.split("\n") + } + } + +} diff --git a/nifi-toolkit/nifi-toolkit-encrypt-config/src/main/groovy/org/apache/nifi/toolkit/encryptconfig/util/NiFiRegistryPropertiesEncryptor.groovy b/nifi-toolkit/nifi-toolkit-encrypt-config/src/main/groovy/org/apache/nifi/toolkit/encryptconfig/util/NiFiRegistryPropertiesEncryptor.groovy new file mode 100644 index 0000000000..5ea8d7bf57 --- /dev/null +++ b/nifi-toolkit/nifi-toolkit-encrypt-config/src/main/groovy/org/apache/nifi/toolkit/encryptconfig/util/NiFiRegistryPropertiesEncryptor.groovy @@ -0,0 +1,65 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.toolkit.encryptconfig.util + +import org.apache.nifi.properties.SensitivePropertyProvider +import org.slf4j.Logger +import org.slf4j.LoggerFactory + +import java.util.regex.Pattern + +class NiFiRegistryPropertiesEncryptor extends PropertiesEncryptor { + + private static final Logger logger = LoggerFactory.getLogger(NiFiRegistryPropertiesEncryptor.class) + + // TODO, if and when we add a dependency on NiFi Registry, we can import these dependencies array rather than redefining them + + // Defined in nifi-registry-properties: org.apache.nifi.registry.properties.NiFiRegistryProperties + private static final String SECURITY_KEYSTORE_PASSWD = "nifi.registry.security.keystorePasswd" + private static final String SECURITY_KEY_PASSWD = "nifi.registry.security.keyPasswd" + private static final String SECURITY_TRUSTSTORE_PASSWD = "nifi.registry.security.truststorePasswd" + + // Defined in nifi-registry-properties: org.apache.nifi.registry.properties.ProtectedNiFiRegistryProperties + private static final String ADDITIONAL_SENSITIVE_PROPERTIES_KEY = "nifi.registry.sensitive.props.additional.keys" + private static final String[] DEFAULT_SENSITIVE_PROPERTIES = [ + SECURITY_KEYSTORE_PASSWD, + SECURITY_KEY_PASSWD, + SECURITY_TRUSTSTORE_PASSWD + ] + + NiFiRegistryPropertiesEncryptor(SensitivePropertyProvider encryptionProvider, SensitivePropertyProvider decryptionProvider) { + super(encryptionProvider, decryptionProvider) + } + + @Override + Properties encrypt(Properties properties) { + Set propertiesToEncrypt = new HashSet<>() + propertiesToEncrypt.addAll(DEFAULT_SENSITIVE_PROPERTIES) + propertiesToEncrypt.addAll(getAdditionalSensitivePropertyKeys(properties)) + + return encrypt(properties, propertiesToEncrypt) + } + + private static String[] getAdditionalSensitivePropertyKeys(Properties properties) { + String rawAdditionalSensitivePropertyKeys = properties.getProperty(ADDITIONAL_SENSITIVE_PROPERTIES_KEY) + if (!rawAdditionalSensitivePropertyKeys) { + return [] + } + return rawAdditionalSensitivePropertyKeys.split(Pattern.quote(",")) + } + +} diff --git a/nifi-toolkit/nifi-toolkit-encrypt-config/src/main/groovy/org/apache/nifi/toolkit/encryptconfig/util/PropertiesEncryptor.groovy b/nifi-toolkit/nifi-toolkit-encrypt-config/src/main/groovy/org/apache/nifi/toolkit/encryptconfig/util/PropertiesEncryptor.groovy new file mode 100644 index 0000000000..189e9a551b --- /dev/null +++ b/nifi-toolkit/nifi-toolkit-encrypt-config/src/main/groovy/org/apache/nifi/toolkit/encryptconfig/util/PropertiesEncryptor.groovy @@ -0,0 +1,269 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.toolkit.encryptconfig.util + +import groovy.io.GroovyPrintWriter +import org.apache.commons.configuration2.PropertiesConfiguration +import org.apache.commons.configuration2.PropertiesConfigurationLayout +import org.apache.commons.configuration2.builder.fluent.Configurations +import org.apache.nifi.properties.SensitivePropertyProvider +import org.apache.nifi.util.StringUtils +import org.slf4j.Logger +import org.slf4j.LoggerFactory + +import java.util.regex.Pattern + +class PropertiesEncryptor { + + private static final Logger logger = LoggerFactory.getLogger(PropertiesEncryptor.class) + + private static final String SUPPORTED_PROPERTY_FILE_REGEX = /^\s*nifi\.[-.\w\s]+\s*=/ + protected static final String PROPERTY_PART_DELIMINATOR = "." + protected static final String PROTECTION_ID_PROPERTY_SUFFIX = "protected" + + protected SensitivePropertyProvider encryptionProvider + protected SensitivePropertyProvider decryptionProvider + + PropertiesEncryptor(SensitivePropertyProvider encryptionProvider, SensitivePropertyProvider decryptionProvider) { + this.encryptionProvider = encryptionProvider + this.decryptionProvider = decryptionProvider + } + + static boolean supportsFile(String filePath) { + try { + File file = new File(filePath) + if (!ToolUtilities.canRead(file)) { + return false + } + Pattern p = Pattern.compile(SUPPORTED_PROPERTY_FILE_REGEX); + return file.readLines().any { it =~ SUPPORTED_PROPERTY_FILE_REGEX } + } catch (Throwable ignored) { + return false + } + } + + static Properties loadFile(String filePath) throws IOException { + + Properties rawProperties + File inputPropertiesFile = new File(filePath) + + if (ToolUtilities.canRead(inputPropertiesFile)) { + rawProperties = new Properties() + inputPropertiesFile.withReader { reader -> + rawProperties.load(reader) + } + } else { + throw new IOException("The file at ${filePath} must exist and be readable by the user running this tool") + } + + return rawProperties + + } + + Properties decrypt(final Properties properties) { + + Set propertiesToSkip = getProtectionIdPropertyKeys(properties) + Map propertiesToDecrypt = getProtectedPropertyKeys(properties) + + if (propertiesToDecrypt.isEmpty()) { + return properties + } + + if (decryptionProvider == null) { + throw new IllegalStateException("Decryption capability not supported without provider. " + + "Usually this means a decryption password / key was not provided to the tool.") + } + + String supportedDecryptionScheme = decryptionProvider.getIdentifierKey() + if (supportedDecryptionScheme) { + propertiesToDecrypt.entrySet().each { entry -> + if (!supportedDecryptionScheme.equals(entry.getValue())) { + throw new IllegalStateException("Decryption capability not supported by this tool. " + + "This tool supports ${supportedDecryptionScheme}, but this properties file contains " + + "${entry.getKey()} protected by ${entry.getValue()}") + } + } + } + + Properties unprotectedProperties = new Properties() + + for (String propertyName : properties.stringPropertyNames()) { + String propertyValue = properties.getProperty(propertyName) + if (propertiesToSkip.contains(propertyName)) { + continue + } + if (propertiesToDecrypt.keySet().contains(propertyName)) { + String decryptedPropertyValue = decryptionProvider.unprotect(propertyValue) + unprotectedProperties.setProperty(propertyName, decryptedPropertyValue) + } else { + unprotectedProperties.setProperty(propertyName, propertyValue) + } + } + + return unprotectedProperties + } + + Properties encrypt(Properties properties) { + return encrypt(properties, properties.stringPropertyNames()) + } + + Properties encrypt(final Properties properties, final Set propertiesToEncrypt) { + + if (encryptionProvider == null) { + throw new IllegalStateException("Input properties is encrypted, but decryption capability is not enabled. " + + "Usually this means a decryption password / key was not provided to the tool.") + } + + logger.debug("Encrypting ${propertiesToEncrypt.size()} properties") + + Properties protectedProperties = new Properties(); + for (String propertyName : properties.stringPropertyNames()) { + String propertyValue = properties.getProperty(propertyName) + // empty properties are not encrypted + if (!StringUtils.isEmpty(propertyValue) && propertiesToEncrypt.contains(propertyName)) { + String encryptedPropertyValue = encryptionProvider.protect(propertyValue) + protectedProperties.setProperty(propertyName, encryptedPropertyValue) + protectedProperties.setProperty(protectionPropertyForProperty(propertyName), encryptionProvider.getIdentifierKey()) + } else { + protectedProperties.setProperty(propertyName, propertyValue) + } + } + + return protectedProperties + } + + void write(Properties updatedProperties, String outputFilePath, String inputFilePath) { + if (!outputFilePath) { + throw new IllegalArgumentException("Cannot write encrypted properties to empty file path") + } + File outputPropertiesFile = new File(outputFilePath) + + if (ToolUtilities.isSafeToWrite(outputPropertiesFile)) { + String serializedProperties = serializePropertiesAndPreserveFormatIfPossible(updatedProperties, inputFilePath) + outputPropertiesFile.text = serializedProperties + } else { + throw new IOException("The file at ${outputFilePath} must be writable by the user running this tool") + } + } + + private String serializePropertiesAndPreserveFormatIfPossible(Properties updatedProperties, String inputFilePath) { + List linesToPersist + File inputPropertiesFile = new File(inputFilePath) + if (ToolUtilities.canRead(inputPropertiesFile)) { + // Instead of just writing the Properties instance to a properties file, + // this method attempts to maintain the structure of the original file and preserves comments + linesToPersist = serializePropertiesAndPreserveFormat(updatedProperties, inputPropertiesFile) + } else { + linesToPersist = serializeProperties(updatedProperties) + } + return linesToPersist.join("\n") + } + + private List serializePropertiesAndPreserveFormat(Properties properties, File originalPropertiesFile) { + Configurations configurations = new Configurations() + try { + PropertiesConfiguration originalPropertiesConfiguration = configurations.properties(originalPropertiesFile) + def keysToAdd = properties.keySet().findAll { !originalPropertiesConfiguration.containsKey(it.toString()) } + def keysToUpdate = properties.keySet().findAll { + !keysToAdd.contains(it) && + properties.getProperty(it.toString()) != originalPropertiesConfiguration.getProperty(it.toString()) + } + def keysToRemove = originalPropertiesConfiguration.getKeys().findAll {!properties.containsKey(it) } + + keysToUpdate.forEach { + originalPropertiesConfiguration.setProperty(it.toString(), properties.getProperty(it.toString())) + } + keysToRemove.forEach { + originalPropertiesConfiguration.clearProperty(it.toString()) + } + boolean isFirst = true + keysToAdd.sort().forEach { + originalPropertiesConfiguration.setProperty(it.toString(), properties.getProperty(it.toString())) + if (isFirst) { + originalPropertiesConfiguration.getLayout().setBlancLinesBefore(it.toString(), 1) + originalPropertiesConfiguration.getLayout().setComment(it.toString(), "protection properties") + isFirst = false + } + } + + OutputStream out = new ByteArrayOutputStream() + Writer writer = new GroovyPrintWriter(out) + + PropertiesConfigurationLayout layout = originalPropertiesConfiguration.getLayout() + layout.setGlobalSeparator("=") + layout.save(originalPropertiesConfiguration, writer) + + writer.flush() + List lines = out.toString().split("\n") + + return lines + } catch(Exception e) { + throw new RuntimeException("Error serializing properties.", e) + } + } + + private List serializeProperties(final Properties properties) { + OutputStream out = new ByteArrayOutputStream() + Writer writer = new GroovyPrintWriter(out) + + properties.store(writer, null) + writer.flush() + List lines = out.toString().split("\n") + + return lines + } + + /** + * Returns a Map of the keys identifying properties that are currently protected + * and the protection identifier for each. The protection + * + * @return the Map of protected property keys and the protection identifier for each + */ + private static Map getProtectedPropertyKeys(Properties properties) { + Map protectedProperties = new HashMap<>(); + properties.stringPropertyNames().forEach({ key -> + String protectionKey = protectionPropertyForProperty(key) + String protectionIdentifier = properties.getProperty(protectionKey) + if (protectionIdentifier) { + protectedProperties.put(key, protectionIdentifier) + } + }) + return protectedProperties + } + + private static Set getProtectionIdPropertyKeys(Properties properties) { + Set protectedProperties = properties.stringPropertyNames().findAll { key -> + key.endsWith(PROPERTY_PART_DELIMINATOR + PROTECTION_ID_PROPERTY_SUFFIX) + } + return protectedProperties; + } + + private static String protectionPropertyForProperty(String propertyName) { + return propertyName + PROPERTY_PART_DELIMINATOR + PROTECTION_ID_PROPERTY_SUFFIX + } + + private static String propertyForProtectionProperty(String protectionPropertyName) { + String[] propertyNameParts = protectionPropertyName.split(Pattern.quote(PROPERTY_PART_DELIMINATOR)) + if (propertyNameParts.length >= 2 && PROTECTION_ID_PROPERTY_SUFFIX.equals(propertyNameParts[-1])) { + return propertyNameParts[(0..-2)].join(PROPERTY_PART_DELIMINATOR) + } + return null + } + + + +} diff --git a/nifi-toolkit/nifi-toolkit-encrypt-config/src/main/groovy/org/apache/nifi/toolkit/encryptconfig/util/ToolUtilities.groovy b/nifi-toolkit/nifi-toolkit-encrypt-config/src/main/groovy/org/apache/nifi/toolkit/encryptconfig/util/ToolUtilities.groovy new file mode 100644 index 0000000000..4e8c1b7a8d --- /dev/null +++ b/nifi-toolkit/nifi-toolkit-encrypt-config/src/main/groovy/org/apache/nifi/toolkit/encryptconfig/util/ToolUtilities.groovy @@ -0,0 +1,164 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.toolkit.encryptconfig.util + +import org.apache.commons.cli.CommandLine +import org.apache.commons.codec.binary.Hex +import org.apache.nifi.util.console.TextDevice +import org.apache.nifi.util.console.TextDevices +import org.bouncycastle.crypto.generators.SCrypt +import org.slf4j.Logger +import org.slf4j.LoggerFactory + +import javax.crypto.Cipher +import java.nio.charset.StandardCharsets +import java.security.KeyException + +class ToolUtilities { + + private static final Logger logger = LoggerFactory.getLogger(ToolUtilities.class) + + private static final int DEFAULT_MIN_PASSWORD_LENGTH = 12 + + // Strong parameters as of 12 Aug 2016 + private static final int SCRYPT_N = 2**16 + private static final int SCRYPT_R = 8 + private static final int SCRYPT_P = 1 + + static boolean isExactlyOneOptionSet(CommandLine commandLine, String... opt) { + Collection setOptions = opt.findAll{commandLine.hasOption(it)} + return setOptions.size() == 1 + } + + static boolean isExactlyOneTrue(Boolean... b) { + Collection trues = b.findAll{it} + return trues.size() == 1 + } + + /** + * Helper method which returns true if the provided file exists and is readable + * + * @param fileToRead the proposed file to read + * @return true if the caller should be able to successfully read from this file + */ + static boolean canRead(File fileToRead) { + fileToRead && (fileToRead.exists() && fileToRead.canRead()) + } + + /** + * Helper method which returns true if it is "safe" to write to the provided file. + * + * Conditions: + * file does not exist and the parent directory is writable + * -OR- + * file exists and is writable + * + * @param fileToWrite the proposed file to be written to + * @return true if the caller can "safely" write to this file location + */ + static boolean isSafeToWrite(File fileToWrite) { + fileToWrite && ((!fileToWrite.exists() && fileToWrite.absoluteFile.parentFile.canWrite()) || (fileToWrite.exists() && fileToWrite.canWrite())) + } + + + /** + * The method returns the provided, derived, or securely-entered key in hex format. + * + * @param device + * @param keyHex + * @param password + * @param usingPassword + * @return + */ + public static String determineKey(TextDevice device = TextDevices.defaultTextDevice(), String keyHex, String password, boolean usingPassword) { + if (usingPassword) { + if (!password) { + logger.debug("Reading password from secure console") + password = readPasswordFromConsole(device) + } + keyHex = deriveKeyFromPassword(password) + password = null + return keyHex + } else { + if (!keyHex) { + logger.debug("Reading hex key from secure console") + keyHex = readKeyFromConsole(device) + } + return keyHex + } + } + + private static String readKeyFromConsole(TextDevice textDevice) { + textDevice.printf("Enter the master key in hexadecimal format (spaces acceptable): ") + new String(textDevice.readPassword()) + } + + private static String readPasswordFromConsole(TextDevice textDevice) { + textDevice.printf("Enter the password: ") + new String(textDevice.readPassword()) + } + +// /** +// * Returns the key in uppercase hexadecimal format with delimiters (spaces, '-', etc.) removed. All non-hex chars are removed. If the result is not a valid length (32, 48, 64 chars depending on the JCE), an exception is thrown. +// * +// * @param rawKey the unprocessed key input +// * @return the formatted hex string in uppercase +// * @throws java.security.KeyException if the key is not a valid length after parsing +// */ +// public static String parseKey(String rawKey) throws KeyException { +// String hexKey = rawKey.replaceAll("[^0-9a-fA-F]", "") +// def validKeyLengths = getValidKeyLengths() +// if (!validKeyLengths.contains(hexKey.size() * 4)) { +// throw new KeyException("The key (${hexKey.size()} hex chars) must be of length ${validKeyLengths} bits (${validKeyLengths.collect { it / 4 }} hex characters)") +// } +// hexKey.toUpperCase() +// } + + /** + * Returns the list of acceptable key lengths in bits based on the current JCE policies. + * + * @return 128 , [192, 256] + */ + public static List getValidKeyLengths() { + Cipher.getMaxAllowedKeyLength("AES") > 128 ? [128, 192, 256] : [128] + } + + private static String deriveKeyFromPassword(String password, int minPasswordLength = DEFAULT_MIN_PASSWORD_LENGTH) { + password = password?.trim() + if (!password || password.length() < minPasswordLength) { + throw new KeyException("Cannot derive key from empty/short password -- password must be at least ${minPasswordLength} characters") + } + + // Generate a 128 bit salt + byte[] salt = generateScryptSalt() + int keyLengthInBytes = getValidKeyLengths().max() / 8 + byte[] derivedKeyBytes = SCrypt.generate(password.getBytes(StandardCharsets.UTF_8), salt, SCRYPT_N, SCRYPT_R, SCRYPT_P, keyLengthInBytes) + Hex.encodeHexString(derivedKeyBytes).toUpperCase() + } + + private static byte[] generateScryptSalt() { +// byte[] salt = new byte[16] +// new SecureRandom().nextBytes(salt) +// salt + /* It is not ideal to use a static salt, but the KDF operation must be deterministic + for a given password, and storing and retrieving the salt in bootstrap.conf causes + compatibility concerns + */ + "NIFI_SCRYPT_SALT".getBytes(StandardCharsets.UTF_8) + } + +} diff --git a/nifi-toolkit/nifi-toolkit-encrypt-config/src/main/groovy/org/apache/nifi/toolkit/encryptconfig/util/XmlEncryptor.groovy b/nifi-toolkit/nifi-toolkit-encrypt-config/src/main/groovy/org/apache/nifi/toolkit/encryptconfig/util/XmlEncryptor.groovy new file mode 100644 index 0000000000..83246827d8 --- /dev/null +++ b/nifi-toolkit/nifi-toolkit-encrypt-config/src/main/groovy/org/apache/nifi/toolkit/encryptconfig/util/XmlEncryptor.groovy @@ -0,0 +1,200 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.toolkit.encryptconfig.util + +import groovy.util.slurpersupport.GPathResult +import groovy.xml.XmlUtil +import org.apache.nifi.properties.SensitivePropertyProvider +import org.slf4j.Logger +import org.slf4j.LoggerFactory + +abstract class XmlEncryptor { + + protected static final String XML_DECLARATION_REGEX = /<\?xml version="1.0" encoding="UTF-8"\?>/ + protected static final ENCRYPTION_NONE = "none" + protected static final ENCRYPTION_EMPTY = "" + + private static final Logger logger = LoggerFactory.getLogger(XmlEncryptor.class) + + protected SensitivePropertyProvider decryptionProvider + protected SensitivePropertyProvider encryptionProvider + + XmlEncryptor(SensitivePropertyProvider encryptionProvider, SensitivePropertyProvider decryptionProvider) { + this.decryptionProvider = decryptionProvider + this.encryptionProvider = encryptionProvider + } + + static boolean supportsFile(String filePath) { + def doc + try { + String rawFileContents = loadXmlFile(filePath) + doc = new XmlSlurper().parseText(rawFileContents) + } catch (Throwable ignored) { + return false + } + return doc != null + } + + static String loadXmlFile(String xmlFilePath) throws IOException { + File xmlFile = new File(xmlFilePath) + if (ToolUtilities.canRead(xmlFile)) { + try { + String xmlContent = xmlFile.text + return xmlContent + } catch (RuntimeException e) { + throw new IOException("Cannot load XML from ${xmlFilePath}", e) + } + } else { + throw new IOException("File at ${xmlFilePath} must exist and be readable by user running this tool.") + } + } + + String decrypt(String encryptedXmlContent) { + try { + + def doc = new XmlSlurper().parseText(encryptedXmlContent) + GPathResult[] encryptedNodes = doc.depthFirst().findAll { GPathResult node -> + node.@encryption != ENCRYPTION_NONE && node.@encryption != ENCRYPTION_EMPTY + } + + if (encryptedNodes.size() == 0) { + return encryptedXmlContent + } + + if (decryptionProvider == null) { + throw new IllegalStateException("Input XML is encrypted, but decryption capability is not enabled. " + + "Usually this means a decryption password / key was not provided to the tool.") + } + String supportedDecryptionScheme = decryptionProvider.getIdentifierKey() + + logger.debug("Found ${encryptedNodes.size()} encrypted XML elements. Will attempt to decrypt using the provided decryption key.") + + encryptedNodes.each { node -> + logger.debug("Attempting to decrypt ${node.text()}") + if (node.@encryption != supportedDecryptionScheme) { + throw new IllegalStateException("Decryption capability not supported by this tool. " + + "This tool supports ${supportedDecryptionScheme}, but this xml file contains " + + "${node.toString()} protected by ${node.@encryption}") + } + String decryptedValue = decryptionProvider.unprotect(node.text().trim()) + node.@encryption = ENCRYPTION_NONE + node.replaceBody(decryptedValue) + } + + // Does not preserve whitespace formatting or comments + String updatedXml = XmlUtil.serialize(doc) + logger.debug("Updated XML content: ${updatedXml}") + return updatedXml + + } catch (Exception e) { + throw new RuntimeException("Cannot decrypt XML content", e) + } + } + + String encrypt(String plainXmlContent) { + try { + def doc = new XmlSlurper().parseText(plainXmlContent) + + GPathResult[] nodesToEncrypt = doc.depthFirst().findAll { GPathResult node -> + node.text() && node.@encryption == ENCRYPTION_NONE + } + + logger.debug("Encrypting ${nodesToEncrypt.size()} element(s) of XML decoument") + + if (nodesToEncrypt.size() == 0) { + return plainXmlContent + } + + nodesToEncrypt.each { node -> + String encryptedValue = this.encryptionProvider.protect(node.text().trim()) + node.@encryption = this.encryptionProvider.getIdentifierKey() + node.replaceBody(encryptedValue) + } + + // Does not preserve whitespace formatting or comments + String updatedXml = XmlUtil.serialize(doc) + logger.debug("Updated XML content: ${updatedXml}") + return updatedXml + } catch (Exception e) { + throw new RuntimeException("Cannot encrypt XML content", e) + } + } + + void writeXmlFile(String updatedXmlContent, String outputXmlPath, String inputXmlPath) throws IOException { + File outputXmlFile = new File(outputXmlPath) + if (ToolUtilities.isSafeToWrite(outputXmlFile)) { + String finalXmlContent = serializeXmlContentAndPreserveFormatIfPossible(updatedXmlContent, inputXmlPath) + outputXmlFile.text = finalXmlContent + } else { + throw new IOException("The XML file at ${outputXmlPath} must be writable by the user running this tool") + } + } + + String serializeXmlContentAndPreserveFormatIfPossible(String updatedXmlContent, String inputXmlPath) { + String finalXmlContent + File inputXmlFile = new File(inputXmlPath) + if (ToolUtilities.canRead(inputXmlFile)) { + String originalXmlContent = new File(inputXmlPath).text + // Instead of just writing the XML content to a file, this method attempts to maintain + // the structure of the original file. + finalXmlContent = serializeXmlContentAndPreserveFormat(updatedXmlContent, originalXmlContent).join("\n") + } else { + finalXmlContent = updatedXmlContent + } + return finalXmlContent + } + + /** + * Given an original XML file and updated XML content, create the lines for an updated, minimally altered, serialization. + * Concrete classes extending this class must implement this method using specific knowledge of the XML document. + * + * @param finalXmlContent the xml content to serialize + * @param inputXmlFile the original input xml file to use as a template for formatting the serialization + * @return the lines of serialized finalXmlContent that are close in raw format to originalInputXmlFile + */ + abstract List serializeXmlContentAndPreserveFormat(String updatedXmlContent, String originalXmlContent) + // TODO, replace the above abstract method with an implementation that works generically for any updated (encryption=."..") nodes + // perhaps this could be done leveraging org.apache.commons.configuration2 which is capable of preserving comments, eg: + + + static String markXmlNodesForEncryption(String plainXmlContent, String gPath, gPathCallback) { + String markedXmlContent + try { + def doc = new XmlSlurper().parseText(plainXmlContent) + // Find the provider element by class even if it has been renamed + def sensitiveProperties = gPathCallback(doc."${gPath}") + + logger.debug("Marking ${sensitiveProperties.size()} sensitive element(s) of XML to be encrypted") + + if (sensitiveProperties.size() == 0) { + logger.debug("No populated sensitive properties found in XML content") + return plainXmlContent + } + + sensitiveProperties.each { + it.@encryption = ENCRYPTION_NONE + } + + // Does not preserve whitespace formatting or comments + // TODO: Switch to XmlParser & XmlNodePrinter to maintain "empty" element structure + markedXmlContent = XmlUtil.serialize(doc) + } catch (Exception e) { + logger.debug("Encountered exception", e) + throw new RuntimeException(e) + } + } +} diff --git a/nifi-toolkit/nifi-toolkit-encrypt-config/src/main/resources/log4j.properties b/nifi-toolkit/nifi-toolkit-encrypt-config/src/main/resources/log4j.properties index fc2aaf12c4..9a13479afa 100644 --- a/nifi-toolkit/nifi-toolkit-encrypt-config/src/main/resources/log4j.properties +++ b/nifi-toolkit/nifi-toolkit-encrypt-config/src/main/resources/log4j.properties @@ -18,5 +18,6 @@ log4j.rootLogger=INFO,console log4j.appender.console=org.apache.log4j.ConsoleAppender +log4j.appender.console.Target=System.err log4j.appender.console.layout=org.apache.log4j.PatternLayout -log4j.appender.console.layout.ConversionPattern=%d{yy/MM/dd HH:mm:ss} %p %c{2}: %m%n \ No newline at end of file +log4j.appender.console.layout.ConversionPattern=%d{yyyy-MM-dd HH:mm:ss} %p %c{1}: %m%n diff --git a/nifi-toolkit/nifi-toolkit-encrypt-config/src/test/groovy/org/apache/nifi/toolkit/encryptconfig/EncryptConfigMainTest.groovy b/nifi-toolkit/nifi-toolkit-encrypt-config/src/test/groovy/org/apache/nifi/toolkit/encryptconfig/EncryptConfigMainTest.groovy new file mode 100644 index 0000000000..b2b8040821 --- /dev/null +++ b/nifi-toolkit/nifi-toolkit-encrypt-config/src/test/groovy/org/apache/nifi/toolkit/encryptconfig/EncryptConfigMainTest.groovy @@ -0,0 +1,285 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.toolkit.encryptconfig + +import org.apache.nifi.properties.AESSensitivePropertyProvider +import org.apache.nifi.properties.ConfigEncryptionTool +import org.apache.nifi.properties.NiFiPropertiesLoader +import org.apache.nifi.toolkit.encryptconfig.util.BootstrapUtil +import org.apache.nifi.util.NiFiProperties +import org.bouncycastle.jce.provider.BouncyCastleProvider +import org.junit.BeforeClass +import org.junit.Rule +import org.junit.Test +import org.junit.contrib.java.lang.system.Assertion +import org.junit.contrib.java.lang.system.ExpectedSystemExit +import org.junit.runner.RunWith +import org.junit.runners.JUnit4 +import org.slf4j.Logger +import org.slf4j.LoggerFactory + +import java.nio.file.Files +import java.security.Security + +import static org.apache.nifi.toolkit.encryptconfig.TestUtil.* + +@RunWith(JUnit4.class) +class EncryptConfigMainTest extends GroovyTestCase { + private static final Logger logger = LoggerFactory.getLogger(EncryptConfigMainTest.class) + + @Rule + public final ExpectedSystemExit exit = ExpectedSystemExit.none() + + @BeforeClass + static void setUpOnce() throws Exception { + Security.addProvider(new BouncyCastleProvider()) + + logger.metaClass.methodMissing = { String name, args -> + logger.info("[${name?.toUpperCase()}] ${(args as List).join(" ")}") + } + + setupTmpDir() + } + + @Test + void testDetermineModeFromArgsWithLegacyMode() { + // Arrange + def argsList = "-b conf/bootstrap.conf -n conf/nifi.properties".split(" ").toList() + + // Act + def toolMode = EncryptConfigMain.determineModeFromArgs(argsList) + + // Assert + toolMode != null + toolMode instanceof LegacyMode + } + + @Test + void testDetermineModeFromArgsWithNifiRegistryMode() { + // Arrange + def argsList = "--nifiRegistry".split(" ").toList() + // Act + def toolMode = EncryptConfigMain.determineModeFromArgs(argsList) + // Assert + toolMode != null + toolMode instanceof NiFiRegistryMode + !argsList.contains("--nifiRegistry") + + /* Test when --nifiRegistry is not first flag */ + + // Arrange + argsList = "-b conf/bootstrap.conf -p --nifiRegistry -r conf/nifi-registry.properties".split(" ").toList() + // Act + toolMode = EncryptConfigMain.determineModeFromArgs(argsList) + // Assert + toolMode != null + toolMode instanceof NiFiRegistryMode + !argsList.contains("--nifiRegistry") + } + + @Test + void testDetermineModeFromArgsWithNifiRegistryDecryptMode() { + // Arrange + def argsList = "--nifiRegistry --decrypt".split(" ").toList() + // Act + def toolMode = EncryptConfigMain.determineModeFromArgs(argsList) + // Assert + toolMode != null + toolMode instanceof NiFiRegistryDecryptMode + !argsList.contains("--nifiRegistry") + !argsList.contains("--decrypt") + + /* Test when --decrypt comes before --nifiRegistry */ + + // Arrange + argsList = "--b conf/bootstrap.conf --decrypt --nifiRegistry".split(" ").toList() + // Act + toolMode = EncryptConfigMain.determineModeFromArgs(argsList) + // Assert + toolMode != null + toolMode instanceof NiFiRegistryDecryptMode + !argsList.contains("--nifiRegistry") + !argsList.contains("--decrypt") + } + + @Test + void testDetermineModeFromArgsReturnsNullOnDecryptWithoutNifiRegistryPresent() { + // Arrange + def argsList = "--decrypt".split(" ").toList() + + // Act + def toolMode = EncryptConfigMain.determineModeFromArgs(argsList) + + // Assert + toolMode == null + } + + @Test + void testShouldPerformFullOperationForNiFiPropertiesAndLoginIdentityProvidersAndAuthorizers() { + // Arrange + exit.expectSystemExitWithStatus(0) + + File tmpDir = setupTmpDir() + + File emptyKeyFile = new File("src/test/resources/bootstrap_with_empty_master_key.conf") + File bootstrapFile = new File("target/tmp/tmp_bootstrap.conf") + bootstrapFile.delete() + + Files.copy(emptyKeyFile.toPath(), bootstrapFile.toPath()) + final List originalBootstrapLines = bootstrapFile.readLines() + String originalKeyLine = originalBootstrapLines.find { + it.startsWith("${BootstrapUtil.NIFI_BOOTSTRAP_KEY_PROPERTY}=") + } + logger.info("Original key line from bootstrap.conf: ${originalKeyLine}") + assert originalKeyLine == "${BootstrapUtil.NIFI_BOOTSTRAP_KEY_PROPERTY}=" + + final String EXPECTED_KEY_LINE = "${BootstrapUtil.NIFI_BOOTSTRAP_KEY_PROPERTY}=${KEY_HEX}" + + // Set up the NFP file + File inputPropertiesFile = new File("src/test/resources/nifi_with_sensitive_properties_unprotected.properties") + File outputPropertiesFile = new File("target/tmp/tmp_nifi.properties") + outputPropertiesFile.delete() + + NiFiProperties inputProperties = new NiFiPropertiesLoader().load(inputPropertiesFile) + logger.info("Loaded ${inputProperties.size()} properties from input file") + + // Set up the LIP file + File inputLIPFile = new File("src/test/resources/login-identity-providers-populated.xml") + File outputLIPFile = new File("target/tmp/tmp-lip.xml") + outputLIPFile.delete() + + String originalLipXmlContent = inputLIPFile.text + logger.info("Original LIP XML content: ${originalLipXmlContent}") + + // Set up the Authorizers file + File inputAuthorizersFile = new File("src/test/resources/authorizers-populated.xml") + File outputAuthorizersFile = new File("target/tmp/tmp-authorizers.xml") + outputAuthorizersFile.delete() + + String originalAuthorizersXmlContent = inputAuthorizersFile.text + logger.info("Original Authorizers XML content: ${originalAuthorizersXmlContent}") + + String[] args = [ + "-n", inputPropertiesFile.path, + "-l", inputLIPFile.path, + "-a", inputAuthorizersFile.path, + "-b", bootstrapFile.path, + "-o", outputPropertiesFile.path, + "-i", outputLIPFile.path, + "-u", outputAuthorizersFile.path, + "-k", KEY_HEX, + "-v"] + + AESSensitivePropertyProvider spp = new AESSensitivePropertyProvider(KEY_HEX) + + exit.checkAssertionAfterwards(new Assertion() { + void checkAssertion() { + + /*** NiFi Properties Assertions ***/ + + final List updatedPropertiesLines = outputPropertiesFile.readLines() + logger.info("Updated nifi.properties:") + logger.info("\n" * 2 + updatedPropertiesLines.join("\n")) + + // Check that the output values for sensitive properties are not the same as the original (i.e. it was encrypted) + NiFiProperties updatedProperties = new NiFiPropertiesLoader().readProtectedPropertiesFromDisk(outputPropertiesFile) + assert updatedProperties.size() >= inputProperties.size() + + // Check that the new NiFiProperties instance matches the output file (values still encrypted) + updatedProperties.getPropertyKeys().every { String key -> + assert updatedPropertiesLines.contains("${key}=${updatedProperties.getProperty(key)}".toString()) + } + + /*** Login Identity Providers Assertions ***/ + + final String updatedLipXmlContent = outputLIPFile.text + logger.info("Updated LIP XML content: ${updatedLipXmlContent}") + // Check that the output values for sensitive properties are not the same as the original (i.e. it was encrypted) + def originalLipParsedXml = new XmlSlurper().parseText(originalLipXmlContent) + def updatedLipParsedXml = new XmlSlurper().parseText(updatedLipXmlContent) + assert originalLipParsedXml != updatedLipParsedXml + assert originalLipParsedXml.'**'.findAll { it.@encryption } != updatedLipParsedXml.'**'.findAll { + it.@encryption + } + def lipEncryptedValues = updatedLipParsedXml.provider.find { + it.identifier == 'ldap-provider' + }.property.findAll { + it.@name =~ "Password" && it.@encryption =~ "aes/gcm/\\d{3}" + } + lipEncryptedValues.each { + assert spp.unprotect(it.text()) == PASSWORD + } + // Check that the comments are still there + def lipTrimmedLines = inputLIPFile.readLines().collect { it.trim() }.findAll { it } + def lipTrimmedSerializedLines = updatedLipXmlContent.split("\n").collect { it.trim() }.findAll { it } + assert lipTrimmedLines.size() == lipTrimmedSerializedLines.size() + + /*** Authorizers Assertions ***/ + + final String updatedAuthorizersXmlContent = outputAuthorizersFile.text + logger.info("Updated Authorizers XML content: ${updatedAuthorizersXmlContent}") + // Check that the output values for sensitive properties are not the same as the original (i.e. it was encrypted) + def originalAuthorizersParsedXml = new XmlSlurper().parseText(originalAuthorizersXmlContent) + def updatedAuthorizersParsedXml = new XmlSlurper().parseText(updatedAuthorizersXmlContent) + assert originalAuthorizersParsedXml != updatedAuthorizersParsedXml + assert originalAuthorizersParsedXml.'**'.findAll { it.@encryption } != updatedAuthorizersParsedXml.'**'.findAll { + it.@encryption + } + def authorizersEncryptedValues = updatedAuthorizersParsedXml.userGroupProvider.find { + it.identifier == 'ldap-user-group-provider' + }.property.findAll { + it.@name =~ "Password" && it.@encryption =~ "aes/gcm/\\d{3}" + } + authorizersEncryptedValues.each { + assert spp.unprotect(it.text()) == PASSWORD + } + // Check that the comments are still there + def authorizersTrimmedLines = inputAuthorizersFile.readLines().collect { it.trim() }.findAll { it } + def authorizersTrimmedSerializedLines = updatedAuthorizersXmlContent.split("\n").collect { it.trim() }.findAll { it } + assert authorizersTrimmedLines.size() == authorizersTrimmedSerializedLines.size() + + /*** Bootstrap assertions ***/ + + // Check that the key was persisted to the bootstrap.conf + final List updatedBootstrapLines = bootstrapFile.readLines() + String updatedKeyLine = updatedBootstrapLines.find { + it.startsWith(BootstrapUtil.NIFI_BOOTSTRAP_KEY_PROPERTY) + } + logger.info("Updated key line: ${updatedKeyLine}") + + assert updatedKeyLine == EXPECTED_KEY_LINE + assert originalBootstrapLines.size() == updatedBootstrapLines.size() + + // Clean up + outputPropertiesFile.deleteOnExit() + outputLIPFile.deleteOnExit() + outputAuthorizersFile.deleteOnExit() + bootstrapFile.deleteOnExit() + tmpDir.deleteOnExit() + } + }) + + // Act + EncryptConfigMain.main(args) + logger.info("Invoked #main with ${args.join(" ")}") + + // Assert + + // Assertions defined above + } + +} diff --git a/nifi-toolkit/nifi-toolkit-encrypt-config/src/test/groovy/org/apache/nifi/toolkit/encryptconfig/NiFiRegistryDecryptModeSpec.groovy b/nifi-toolkit/nifi-toolkit-encrypt-config/src/test/groovy/org/apache/nifi/toolkit/encryptconfig/NiFiRegistryDecryptModeSpec.groovy new file mode 100644 index 0000000000..2ff14147b3 --- /dev/null +++ b/nifi-toolkit/nifi-toolkit-encrypt-config/src/test/groovy/org/apache/nifi/toolkit/encryptconfig/NiFiRegistryDecryptModeSpec.groovy @@ -0,0 +1,117 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.toolkit.encryptconfig + +import org.bouncycastle.jce.provider.BouncyCastleProvider +import org.junit.Assume +import org.slf4j.Logger +import org.slf4j.LoggerFactory +import spock.lang.Specification + +import java.security.Security + +import static org.apache.nifi.toolkit.encryptconfig.TestUtil.* + +class NiFiRegistryDecryptModeSpec extends Specification { + private static final Logger logger = LoggerFactory.getLogger(NiFiRegistryDecryptModeSpec.class) + + ByteArrayOutputStream toolStdOutContent + PrintStream origSystemOut + + // runs before every feature method + def setup() { + origSystemOut = System.out + toolStdOutContent = new ByteArrayOutputStream(); + System.setOut(new PrintStream(toolStdOutContent)); + } + + // runs after every feature method + def cleanup() { + toolStdOutContent.flush() + System.setOut(origSystemOut); + toolStdOutContent.close() + } + + // runs before the first feature method + def setupSpec() { + Security.addProvider(new BouncyCastleProvider()) + setupTmpDir() + } + + // runs after the last feature method + def cleanupSpec() { + cleanupTmpDir() + } + + def "decrypt protected nifi-registry.properties file using -k"() { + + setup: + NiFiRegistryDecryptMode tool = new NiFiRegistryDecryptMode() + def inRegistryProperties1 = copyFileToTempFile(RESOURCE_REGISTRY_PROPERTIES_POPULATED_PROTECTED_KEY_128) + File outRegistryProperties1 = generateTmpFile() + + when: "run with args: -k -r " + tool.run("-k ${KEY_HEX_128} -r ${inRegistryProperties1}".split(" ")) + toolStdOutContent.flush() + outRegistryProperties1.text = toolStdOutContent.toString() + then: "decrypted properties file was printed to std out" + assertPropertiesFilesAreEqual(RESOURCE_REGISTRY_PROPERTIES_POPULATED_UNPROTECTED, outRegistryProperties1.getAbsolutePath(), true) + and: "input properties file is still encrypted" + assertPropertiesFilesAreEqual(RESOURCE_REGISTRY_PROPERTIES_POPULATED_PROTECTED_KEY_128, inRegistryProperties1, true) + + } + + def "decrypt protected nifi-registry.properties file using -p [256-bit]"() { + + Assume.assumeTrue("Test only runs when unlimited strength crypto is available", isUnlimitedStrengthCryptoAvailable()) + + setup: + NiFiRegistryDecryptMode tool = new NiFiRegistryDecryptMode() + def inRegistryProperties1 = copyFileToTempFile(RESOURCE_REGISTRY_PROPERTIES_POPULATED_PROTECTED_PASSWORD_256) + File outRegistryProperties1 = generateTmpFile() + + when: "run with args: -p -r " + tool.run("-p ${PASSWORD} -r ${inRegistryProperties1}".split(" ")) + toolStdOutContent.flush() + outRegistryProperties1.text = toolStdOutContent.toString() + then: "decrypted properties file was printed to std out" + assertPropertiesFilesAreEqual(RESOURCE_REGISTRY_PROPERTIES_POPULATED_UNPROTECTED, outRegistryProperties1.getAbsolutePath(), true) + and: "input properties file is still encrypted" + assertPropertiesFilesAreEqual(RESOURCE_REGISTRY_PROPERTIES_POPULATED_PROTECTED_PASSWORD_256, inRegistryProperties1, true) + + } + + def "decrypt protected nifi-registry.properties file using -b"() { + + setup: + NiFiRegistryDecryptMode tool = new NiFiRegistryDecryptMode() + def inRegistryProperties = copyFileToTempFile(RESOURCE_REGISTRY_PROPERTIES_POPULATED_PROTECTED_KEY_128) + def inBootstrap = copyFileToTempFile(RESOURCE_REGISTRY_BOOTSTRAP_KEY_128) + File outRegistryProperties = generateTmpFile() + + when: "run with args: -b -r " + tool.run("-b ${inBootstrap} -r ${inRegistryProperties}".split(" ")) + toolStdOutContent.flush() + outRegistryProperties.text = toolStdOutContent.toString() + then: "decrypted properties file was printed to std out" + assertPropertiesFilesAreEqual(RESOURCE_REGISTRY_PROPERTIES_POPULATED_UNPROTECTED, outRegistryProperties.getAbsolutePath(), true) + and: "input properties file is still encrypted" + assertPropertiesFilesAreEqual(RESOURCE_REGISTRY_PROPERTIES_POPULATED_PROTECTED_KEY_128, inRegistryProperties, true) + + } + +} diff --git a/nifi-toolkit/nifi-toolkit-encrypt-config/src/test/groovy/org/apache/nifi/toolkit/encryptconfig/NiFiRegistryModeSpec.groovy b/nifi-toolkit/nifi-toolkit-encrypt-config/src/test/groovy/org/apache/nifi/toolkit/encryptconfig/NiFiRegistryModeSpec.groovy new file mode 100644 index 0000000000..137eeea789 --- /dev/null +++ b/nifi-toolkit/nifi-toolkit-encrypt-config/src/test/groovy/org/apache/nifi/toolkit/encryptconfig/NiFiRegistryModeSpec.groovy @@ -0,0 +1,331 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.toolkit.encryptconfig + +import org.apache.nifi.toolkit.encryptconfig.util.BootstrapUtil +import org.bouncycastle.jce.provider.BouncyCastleProvider +import org.slf4j.Logger +import org.slf4j.LoggerFactory +import spock.lang.Specification + +import java.security.Security + +import static org.apache.nifi.toolkit.encryptconfig.TestUtil.* + +class NiFiRegistryModeSpec extends Specification { + private static final Logger logger = LoggerFactory.getLogger(NiFiRegistryModeSpec.class) + + // runs before every feature method + def setup() {} + + // runs after every feature method + def cleanup() {} + + // runs before the first feature method + def setupSpec() { + Security.addProvider(new BouncyCastleProvider()) + setupTmpDir() + } + + // runs after the last feature method + def cleanupSpec() { + cleanupTmpDir() + } + + def "writing key to bootstrap.conf file"() { + + setup: + NiFiRegistryMode tool = new NiFiRegistryMode() + def inBootstrapConf1 = copyFileToTempFile(RESOURCE_REGISTRY_BOOTSTRAP_DEFAULT) + def inBootstrapConf2 = copyFileToTempFile(RESOURCE_REGISTRY_BOOTSTRAP_DEFAULT) + def inBootstrapConf3 = copyFileToTempFile(RESOURCE_REGISTRY_BOOTSTRAP_DEFAULT) + def outBootstrapConf3 = generateTmpFilePath() + def inBootstrapConf4 = copyFileToTempFile(RESOURCE_REGISTRY_BOOTSTRAP_DEFAULT) + def outBootstrapConf4 = generateTmpFilePath() + def inBootstrapConf5 = copyFileToTempFile(RESOURCE_REGISTRY_BOOTSTRAP_KEY_128) + def outBootstrapConf5 = generateTmpFilePath() + + when: "run with args: -k -b " + tool.run("-k ${KEY_HEX_128} -b ${inBootstrapConf1}".split(" ")) + then: "key is written to input bootstrap.conf" + assertBootstrapFilesAreEqual(RESOURCE_REGISTRY_BOOTSTRAP_KEY_128, inBootstrapConf1, true) + + when: "run with args: -p -b " + tool.run("-p ${PASSWORD} -b ${inBootstrapConf2}".split(" ")) + then: "key derived from password is written to input bootstrap.conf" + PASSWORD_KEY_HEX == readKeyFromBootstrap(inBootstrapConf2) + + when: "run with args: -k -b -B " + tool.run("-k ${KEY_HEX_128} -b ${inBootstrapConf3} -B ${outBootstrapConf3}".split(" ")) + then: "key is written to output bootstrap.conf" + assertBootstrapFilesAreEqual(RESOURCE_REGISTRY_BOOTSTRAP_KEY_128, outBootstrapConf3, true) + and: "input bootstrap.conf is unchanged" + assertBootstrapFilesAreEqual(RESOURCE_REGISTRY_BOOTSTRAP_DEFAULT, inBootstrapConf3, true) + + when: "run with args: -p -b -B " + tool.run("-p ${PASSWORD} -b ${inBootstrapConf4} -B ${outBootstrapConf4}".split(" ")) + then: "key derived from password is written to output bootstrap.conf" + PASSWORD_KEY_HEX == readKeyFromBootstrap(outBootstrapConf4) + and: "input bootstrap.conf is unchanged" + assertBootstrapFilesAreEqual(RESOURCE_REGISTRY_BOOTSTRAP_DEFAULT, inBootstrapConf4, true) + + when: "run with args: -b -B " + tool.run("-b ${inBootstrapConf5} -B ${outBootstrapConf5}".split(" ")) + then: "key from input file is copied to output file" + KEY_HEX_128 == readKeyFromBootstrap(outBootstrapConf5) + assertBootstrapFilesAreEqual(inBootstrapConf5, outBootstrapConf5, true) + and: "input bootstrap.conf is unchanged" + assertBootstrapFilesAreEqual(RESOURCE_REGISTRY_BOOTSTRAP_KEY_128, inBootstrapConf5, true) + + } + + def "encrypt unprotected nifi-registry.properties file"() { + + setup: + NiFiRegistryMode tool = new NiFiRegistryMode() + def inBootstrapConf1 = copyFileToTempFile(RESOURCE_REGISTRY_BOOTSTRAP_DEFAULT) + def inRegistryProperties1 = copyFileToTempFile(RESOURCE_REGISTRY_PROPERTIES_POPULATED_UNPROTECTED) + def inBootstrapConf2 = copyFileToTempFile(RESOURCE_REGISTRY_BOOTSTRAP_DEFAULT) + def inRegistryProperties2 = copyFileToTempFile(RESOURCE_REGISTRY_PROPERTIES_POPULATED_UNPROTECTED) + def outRegistryProperties2 = generateTmpFilePath() + def inBootstrapConf3 = copyFileToTempFile(RESOURCE_REGISTRY_BOOTSTRAP_DEFAULT) + def inRegistryProperties3 = copyFileToTempFile(RESOURCE_REGISTRY_PROPERTIES_POPULATED_UNPROTECTED) + def inRegistryProperties4 = copyFileToTempFile(RESOURCE_REGISTRY_PROPERTIES_POPULATED_UNPROTECTED) + + when: "run with args: -k -b -r " + tool.run("-k ${KEY_HEX} -b ${inBootstrapConf1} -r ${inRegistryProperties1}".split(" ")) + then: "properties file is protected in place" + assertNiFiRegistryUnprotectedPropertiesAreProtected(inRegistryProperties1) + and: "key is written to input bootstrap.conf" + KEY_HEX == readKeyFromBootstrap(inBootstrapConf1) + + when: "run with args: -k -b -r -R " + tool.run("-k ${KEY_HEX} -b ${inBootstrapConf2} -r ${inRegistryProperties2} -R ${outRegistryProperties2}".split(" ")) + then: "output properties file is protected" + assertNiFiRegistryUnprotectedPropertiesAreProtected(outRegistryProperties2) + and: "input properties file is unchanged" + assertPropertiesFilesAreEqual(RESOURCE_REGISTRY_PROPERTIES_POPULATED_UNPROTECTED, inRegistryProperties2, true) + and: "key is written to output bootstrap.conf" + KEY_HEX == readKeyFromBootstrap(inBootstrapConf2) + + when: "run with args: -p -b -r " + tool.run("-p ${PASSWORD} -b ${inBootstrapConf3} -r ${inRegistryProperties3}".split(" ")) + then: "properties file is protected in place" + assertNiFiRegistryUnprotectedPropertiesAreProtected(inRegistryProperties3) + and: "key is written to input bootstrap.conf" + PASSWORD_KEY_HEX == readKeyFromBootstrap(inBootstrapConf3) + + when: "run with args: -b -r " + tool.run("-b ${RESOURCE_REGISTRY_BOOTSTRAP_KEY_128} -r ${inRegistryProperties4}".split(" ")) + then: "properties file is protected in place using key from bootstrap" + assertNiFiRegistryUnprotectedPropertiesAreProtected(inRegistryProperties4, PROTECTION_SCHEME_128) + + } + + def "encrypt nifi-registry.properties with no sensitive properties is a no-op"() { + + setup: + NiFiRegistryMode tool = new NiFiRegistryMode() + def inBootstrapConf1 = copyFileToTempFile(RESOURCE_REGISTRY_BOOTSTRAP_DEFAULT) + def inRegistryProperties1 = copyFileToTempFile(RESOURCE_REGISTRY_PROPERTIES_COMMENTED) + def inBootstrapConf2 = copyFileToTempFile(RESOURCE_REGISTRY_BOOTSTRAP_DEFAULT) + def inRegistryProperties2 = copyFileToTempFile(RESOURCE_REGISTRY_PROPERTIES_EMPTY) + def outRegistryProperties2 = generateTmpFilePath() + + when: "run with args: -k -b -r " + tool.run("-k ${KEY_HEX} -b ${inBootstrapConf1} -r ${inRegistryProperties1}".split(" ")) + then: "properties file is unchanged" + assertPropertiesFilesAreEqual(RESOURCE_REGISTRY_PROPERTIES_COMMENTED, inRegistryProperties1, true) + and: "key is written to input bootstrap.conf" + KEY_HEX == readKeyFromBootstrap(inBootstrapConf1) + + when: "run with args: -k -b -r -R " + tool.run("-k ${KEY_HEX} -b ${inBootstrapConf2} -r ${inRegistryProperties2} -R ${outRegistryProperties2}".split(" ")) + then: "input properties file is unchanged and output properties file matches input" + assertPropertiesFilesAreEqual(RESOURCE_REGISTRY_PROPERTIES_EMPTY, inRegistryProperties2, true) + assertPropertiesFilesAreEqual(inRegistryProperties2, outRegistryProperties2, true) + and: "key is written to output bootstrap.conf" + KEY_HEX == readKeyFromBootstrap(inBootstrapConf2) + + } + + def "encrypt unprotected authorizers.xml file"() { + + setup: + NiFiRegistryMode tool = new NiFiRegistryMode() + def inBootstrapConf1 = copyFileToTempFile(RESOURCE_REGISTRY_BOOTSTRAP_DEFAULT) + def inAuthorizers1 = copyFileToTempFile(RESOURCE_REGISTRY_AUTHORIZERS_POPULATED_UNPROTECTED) + def inBootstrapConf2 = copyFileToTempFile(RESOURCE_REGISTRY_BOOTSTRAP_DEFAULT) + def inAuthorizers2 = copyFileToTempFile(RESOURCE_REGISTRY_AUTHORIZERS_POPULATED_UNPROTECTED) + def outAuthorizers2 = generateTmpFilePath() + + when: "run with args: -k -b -a " + tool.run("-k ${KEY_HEX} -b ${inBootstrapConf1} -a ${inAuthorizers1}".split(" ")) + then: "authorizers file is protected in place" + assertRegistryAuthorizersXmlIsProtected(inAuthorizers1) + and: "key is written to input bootstrap.conf" + KEY_HEX == readKeyFromBootstrap(inBootstrapConf1) + + when: "run with args: -k -b -a -A " + tool.run("-k ${KEY_HEX} -b ${inBootstrapConf2} -a ${inAuthorizers2} -A ${outAuthorizers2}".split(" ")) + then: "authorizers file is protected in place" + assertRegistryAuthorizersXmlIsProtected(outAuthorizers2) + and: "key is written to input bootstrap.conf" + KEY_HEX == readKeyFromBootstrap(inBootstrapConf2) + + } + + def "encrypt authorizers.xml with no sensitive properties is a no-op"() { + + setup: + NiFiRegistryMode tool = new NiFiRegistryMode() + def inBootstrapConf1 = copyFileToTempFile(RESOURCE_REGISTRY_BOOTSTRAP_DEFAULT) + def inAuthorizersXml1 = copyFileToTempFile(RESOURCE_REGISTRY_AUTHORIZERS_COMMENTED) + def inBootstrapConf2 = copyFileToTempFile(RESOURCE_REGISTRY_BOOTSTRAP_DEFAULT) + def inAuthorizersXml2 = copyFileToTempFile(RESOURCE_REGISTRY_AUTHORIZERS_EMPTY) + def outAuthorizers2 = generateTmpFilePath() + + when: "run with args: -k -b -a " + tool.run("-k ${KEY_HEX} -b ${inBootstrapConf1} -a ${inAuthorizersXml1}".split(" ")) + then: "authorizers file is unchanged" + assertFilesAreEqual(RESOURCE_REGISTRY_AUTHORIZERS_COMMENTED, inAuthorizersXml1) + and: "key is written to input bootstrap.conf" + KEY_HEX == readKeyFromBootstrap(inBootstrapConf1) + + when: "run with args: -k -b -a -A " + tool.run("-k ${KEY_HEX} -b ${inBootstrapConf2} -a ${inAuthorizersXml2} -A ${outAuthorizers2}".split(" ")) + then: "input authorizers file is unchanged and output authorizers matches input" + assertFilesAreEqual(RESOURCE_REGISTRY_AUTHORIZERS_EMPTY, inAuthorizersXml2) + assertFilesAreEqual(inAuthorizersXml2, outAuthorizers2) + and: "key is written to output bootstrap.conf" + KEY_HEX == readKeyFromBootstrap(inBootstrapConf2) + + } + + def "encrypt unprotected identity-providers.xml file"() { + + setup: + NiFiRegistryMode tool = new NiFiRegistryMode() + def inBootstrapConf1 = copyFileToTempFile(RESOURCE_REGISTRY_BOOTSTRAP_DEFAULT) + def inIdentityProviders1 = copyFileToTempFile(RESOURCE_REGISTRY_IDENTITY_PROVIDERS_POPULATED_UNPROTECTED) + def inBootstrapConf2 = copyFileToTempFile(RESOURCE_REGISTRY_BOOTSTRAP_DEFAULT) + def inIdentityProviders2 = copyFileToTempFile(RESOURCE_REGISTRY_IDENTITY_PROVIDERS_POPULATED_UNPROTECTED) + def outIdentityProviders2 = generateTmpFilePath() + + when: "run with args: -k -b -i " + tool.run("-k ${KEY_HEX} -b ${inBootstrapConf1} -i ${inIdentityProviders1}".split(" ")) + then: "identity providers file is protected in place" + assertRegistryIdentityProvidersXmlIsProtected(inIdentityProviders1) + and: "key is written to input bootstrap.conf" + KEY_HEX == readKeyFromBootstrap(inBootstrapConf1) + + when: "run with args: -k -b -i -I " + tool.run("-k ${KEY_HEX} -b ${inBootstrapConf2} -i ${inIdentityProviders2} -I ${outIdentityProviders2}".split(" ")) + then: "identity providers file is protected in place" + assertRegistryIdentityProvidersXmlIsProtected(outIdentityProviders2) + and: "key is written to input bootstrap.conf" + KEY_HEX == readKeyFromBootstrap(inBootstrapConf2) + + } + + def "encrypt identity-providers.xml with no sensitive properties is a no-op"() { + + setup: + NiFiRegistryMode tool = new NiFiRegistryMode() + def inBootstrapConf1 = copyFileToTempFile(RESOURCE_REGISTRY_BOOTSTRAP_DEFAULT) + def inIdentityProviders1 = copyFileToTempFile(RESOURCE_REGISTRY_IDENTITY_PROVIDERS_COMMENTED) + def inBootstrapConf2 = copyFileToTempFile(RESOURCE_REGISTRY_BOOTSTRAP_DEFAULT) + def inIdentityProviders2 = copyFileToTempFile(RESOURCE_REGISTRY_IDENTITY_PROVIDERS_EMPTY) + def outIdentityProviders2 = generateTmpFilePath() + + when: "run with args: -k -b -i " + tool.run("-k ${KEY_HEX} -b ${inBootstrapConf1} -i ${inIdentityProviders1}".split(" ")) + then: "identity providers file is unchanged" + assertFilesAreEqual(RESOURCE_REGISTRY_IDENTITY_PROVIDERS_COMMENTED, inIdentityProviders1) + and: "key is written to input bootstrap.conf" + KEY_HEX == readKeyFromBootstrap(inBootstrapConf1) + + when: "run with args: -k -b -i -I " + tool.run("-k ${KEY_HEX} -b ${inBootstrapConf2} -i ${inIdentityProviders2} -I ${outIdentityProviders2}".split(" ")) + then: "identity providers file is unchanged and output identity providers matches input" + assertFilesAreEqual(RESOURCE_REGISTRY_IDENTITY_PROVIDERS_EMPTY, inIdentityProviders2) + assertFilesAreEqual(inIdentityProviders2, outIdentityProviders2) + and: "key is written to output bootstrap.conf" + KEY_HEX == readKeyFromBootstrap(inBootstrapConf2) + + } + + def "encrypt full configuration with properties, authorizers, and identity providers"() { + + setup: + NiFiRegistryMode tool = new NiFiRegistryMode() + def inBootstrapConf1 = copyFileToTempFile(RESOURCE_REGISTRY_BOOTSTRAP_DEFAULT) + def inRegistryProperties1 = copyFileToTempFile(RESOURCE_REGISTRY_PROPERTIES_POPULATED_UNPROTECTED) + def inAuthorizers1 = copyFileToTempFile(RESOURCE_REGISTRY_AUTHORIZERS_POPULATED_UNPROTECTED) + def inIdentityProviders1 = copyFileToTempFile(RESOURCE_REGISTRY_IDENTITY_PROVIDERS_POPULATED_UNPROTECTED) + + when: "run with args: -k -b -r -a -i " + tool.run("-k ${KEY_HEX} -b ${inBootstrapConf1} -r ${inRegistryProperties1} -a ${inAuthorizers1} -i ${inIdentityProviders1}".split(" ")) + then: "all files are protected" + assertNiFiRegistryUnprotectedPropertiesAreProtected(inRegistryProperties1) + assertRegistryAuthorizersXmlIsProtected(inAuthorizers1) + assertRegistryIdentityProvidersXmlIsProtected(inIdentityProviders1) + and: "key is written to input bootstrap.conf" + KEY_HEX == readKeyFromBootstrap(inBootstrapConf1) + + } + + //-- Helper Methods + + private static String readKeyFromBootstrap(String bootstrapPath) { + return BootstrapUtil.extractKeyFromBootstrapFile(bootstrapPath, BootstrapUtil.REGISTRY_BOOTSTRAP_KEY_PROPERTY) + } + + private static boolean assertNiFiRegistryUnprotectedPropertiesAreProtected( + String pathToProtectedProperties, + String expectedProtectionScheme = PROTECTION_SCHEME) { + return assertPropertiesAreProtected( + RESOURCE_REGISTRY_PROPERTIES_POPULATED_UNPROTECTED, + pathToProtectedProperties, + RESOURCE_REGISTRY_PROPERTIES_SENSITIVE_PROPS, + expectedProtectionScheme) + } + + static boolean assertRegistryAuthorizersXmlIsProtected( + String pathToProtectedXmlToVerify, + String expectedProtectionScheme = PROTECTION_SCHEME, + String expectedKey = KEY_HEX) { + return assertRegistryAuthorizersXmlIsProtected( + RESOURCE_REGISTRY_AUTHORIZERS_POPULATED_UNPROTECTED, + pathToProtectedXmlToVerify, + expectedProtectionScheme, + expectedKey) + } + + static boolean assertRegistryIdentityProvidersXmlIsProtected( + String pathToProtectedXmlToVerify, + String expectedProtectionScheme = PROTECTION_SCHEME, + String expectedKey = KEY_HEX) { + return assertRegistryIdentityProvidersXmlIsProtected( + RESOURCE_REGISTRY_IDENTITY_PROVIDERS_POPULATED_UNPROTECTED, + pathToProtectedXmlToVerify, + expectedProtectionScheme, + expectedKey) + } + + +} diff --git a/nifi-toolkit/nifi-toolkit-encrypt-config/src/test/groovy/org/apache/nifi/toolkit/encryptconfig/TestUtil.groovy b/nifi-toolkit/nifi-toolkit-encrypt-config/src/test/groovy/org/apache/nifi/toolkit/encryptconfig/TestUtil.groovy new file mode 100644 index 0000000000..0616a667cb --- /dev/null +++ b/nifi-toolkit/nifi-toolkit-encrypt-config/src/test/groovy/org/apache/nifi/toolkit/encryptconfig/TestUtil.groovy @@ -0,0 +1,376 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.toolkit.encryptconfig + +import groovy.util.slurpersupport.GPathResult +import org.apache.commons.lang3.SystemUtils +import org.apache.nifi.properties.AESSensitivePropertyProvider +import org.apache.nifi.toolkit.encryptconfig.util.NiFiRegistryAuthorizersXmlEncryptor +import org.apache.nifi.toolkit.encryptconfig.util.NiFiRegistryIdentityProvidersXmlEncryptor + +import javax.crypto.Cipher +import java.nio.file.Files +import java.nio.file.attribute.PosixFilePermission + +class TestUtil { + + static final String RESOURCE_REGISTRY_BOOTSTRAP_DEFAULT = absolutePathForResource('/nifi-registry/bootstrap_default.conf') + static final String RESOURCE_REGISTRY_BOOTSTRAP_NO_KEY = absolutePathForResource('/nifi-registry/bootstrap_without_master_key.conf') + static final String RESOURCE_REGISTRY_BOOTSTRAP_EMPTY_KEY = absolutePathForResource('/nifi-registry/bootstrap_with_empty_master_key.conf') + static final String RESOURCE_REGISTRY_BOOTSTRAP_KEY_128 = absolutePathForResource('/nifi-registry/bootstrap_with_master_key_128.conf') + static final String RESOURCE_REGISTRY_BOOTSTRAP_KEY_FROM_PASSWORD_128 = absolutePathForResource('/nifi-registry/bootstrap_with_master_key_from_password_128.conf') + + static final String RESOURCE_REGISTRY_PROPERTIES_COMMENTED = absolutePathForResource('/nifi-registry/nifi-registry-commented.properties') + static final String RESOURCE_REGISTRY_PROPERTIES_EMPTY = absolutePathForResource('/nifi-registry/nifi-registry-empty.properties') + static final String RESOURCE_REGISTRY_PROPERTIES_POPULATED_UNPROTECTED = absolutePathForResource('/nifi-registry/nifi-registry-populated-unprotected.properties') + static final String RESOURCE_REGISTRY_PROPERTIES_POPULATED_PROTECTED_KEY_128 = absolutePathForResource('/nifi-registry/nifi-registry-populated-protected-key-128.properties') + static final String RESOURCE_REGISTRY_PROPERTIES_POPULATED_PROTECTED_KEY_256 = absolutePathForResource('/nifi-registry/nifi-registry-populated-protected-key-256.properties') + static final String RESOURCE_REGISTRY_PROPERTIES_POPULATED_PROTECTED_PASSWORD_256 = absolutePathForResource('/nifi-registry/nifi-registry-populated-protected-password-256.properties') + + static final String RESOURCE_REGISTRY_AUTHORIZERS_COMMENTED = absolutePathForResource('/nifi-registry/authorizers-commented.xml') + static final String RESOURCE_REGISTRY_AUTHORIZERS_EMPTY = absolutePathForResource('/nifi-registry/authorizers-empty.xml') + static final String RESOURCE_REGISTRY_AUTHORIZERS_POPULATED_UNPROTECTED = absolutePathForResource('/nifi-registry/authorizers-populated-unprotected.xml') + + static final String RESOURCE_REGISTRY_IDENTITY_PROVIDERS_COMMENTED = absolutePathForResource('/nifi-registry/identity-providers-commented.xml') + static final String RESOURCE_REGISTRY_IDENTITY_PROVIDERS_EMPTY = absolutePathForResource('/nifi-registry/identity-providers-empty.xml') + static final String RESOURCE_REGISTRY_IDENTITY_PROVIDERS_POPULATED_UNPROTECTED = absolutePathForResource('/nifi-registry/identity-providers-populated-unprotected.xml') + + static final String[] RESOURCE_REGISTRY_PROPERTIES_SENSITIVE_PROPS = [ + "nifi.registry.security.keystorePasswd", + "nifi.registry.security.keyPasswd", + "nifi.registry.security.truststorePasswd", + "nifi.registry.dummy.sensitive.property.1", + "nifi.registry.dummy.sensitive.property.2" + ] + + private static final int RESOURCE_REGISTRY_IDENTITY_PROVIDERS_PASSWORD_LINE_COUNT = 3 + private static final int RESOURCE_REGISTRY_AUTHORIZERS_PASSWORD_LINE_COUNT = 3 + private final String PASSWORD_PROP_REGEX = "]* name=\".* Password\"" + + static final String KEY_HEX_128 = "0123456789ABCDEFFEDCBA9876543210" + static final String KEY_HEX_256 = KEY_HEX_128 * 2 + static final String KEY_HEX = isUnlimitedStrengthCryptoAvailable() ? KEY_HEX_256 : KEY_HEX_128 + + static final String PASSWORD = "thisIsABadPassword" + // From ToolUtilities.deriveKeyFromPassword("thisIsABadPassword") + static final String PASSWORD_KEY_HEX_256 = "2C576A9585DB862F5ECBEE5B4FFFCCA14B18D8365968D7081651006507AD2BDE" + static final String PASSWORD_KEY_HEX_128 = "2C576A9585DB862F5ECBEE5B4FFFCCA1" + static final String PASSWORD_KEY_HEX = isUnlimitedStrengthCryptoAvailable() ? PASSWORD_KEY_HEX_256 : PASSWORD_KEY_HEX_128 + + static final String PROTECTION_SCHEME_128 = "aes/gcm/128" + static final String PROTECTION_SCHEME_256 = "aes/gcm/256" + static final String PROTECTION_SCHEME = isUnlimitedStrengthCryptoAvailable() ? PROTECTION_SCHEME_256 : PROTECTION_SCHEME_128 + + private static final String DEFAULT_TMP_DIR = "target/tmp/" + + /** + * @return boolean indicating if the current Java Runtime Environment supports unlimited strength crypto functions + */ + static boolean isUnlimitedStrengthCryptoAvailable() { + Cipher.getMaxAllowedKeyLength("AES") > 128 + } + + private static absolutePathForResource(String relativeResourcePath) { + return TestUtil.class.getResource(relativeResourcePath).getPath() + } + + static File setupTmpDir(String tmpDirPath = DEFAULT_TMP_DIR) { + File tmpDir = new File(tmpDirPath) + tmpDir.mkdirs() + setFilePermissions(tmpDir, [PosixFilePermission.OWNER_READ, PosixFilePermission.OWNER_WRITE, PosixFilePermission.OWNER_EXECUTE, PosixFilePermission.GROUP_READ, PosixFilePermission.GROUP_WRITE, PosixFilePermission.GROUP_EXECUTE, PosixFilePermission.OTHERS_READ, PosixFilePermission.OTHERS_WRITE, PosixFilePermission.OTHERS_EXECUTE]) + tmpDir + } + + static void cleanupTmpDir(String tmpDirPath = DEFAULT_TMP_DIR) { + File tmpDir = new File(tmpDirPath) + tmpDir.delete() + } + + static String generateTmpFilePath() { + File tmpDir = setupTmpDir() + return "${tmpDir.getAbsolutePath()}/${UUID.randomUUID().toString()}.tmp_file" + } + + static File generateTmpFile() { + File tmpFile = new File(generateTmpFilePath()) + tmpFile + } + + static String copyFileToTempFile(String filePath) { + File tmpFile = generateTmpFile() + tmpFile.text = new File(filePath).text + return tmpFile.getAbsolutePath() + } + + /** + * OS-agnostic method for setting file permissions. On POSIX-compliant systems, accurately sets the provided permissions. On Windows, sets the corresponding permissions for the file owner only. + * + * @param file the file to modify + * @param permissions the desired permissions + */ + static void setFilePermissions(File file, List permissions = []) { + if (SystemUtils.IS_OS_WINDOWS) { + file?.setReadable(permissions.contains(PosixFilePermission.OWNER_READ)) + file?.setWritable(permissions.contains(PosixFilePermission.OWNER_WRITE)) + file?.setExecutable(permissions.contains(PosixFilePermission.OWNER_EXECUTE)) + } else { + Files.setPosixFilePermissions(file?.toPath(), permissions as Set) + } + } + + /** + * Make assertions that a properties file is protected correctly given a known starting point. + * + * @param pathToOriginalUnprotectedProperties - location of the original, plaintext properties file + * @param pathToProtectedPropertiesToVerify - location of the protected properties file + * @param sensitivePropertiesToVerify - the properties that should be considered sensitive + * @param expectedProtectionSchemeToVerify - the expected protection cipher identifier + * @return true if all assertion checks pass, otherwise assertion error is thrown + */ + static boolean assertPropertiesAreProtected( + String pathToOriginalUnprotectedProperties, + String pathToProtectedPropertiesToVerify, + String[] sensitivePropertiesToVerify, + String expectedProtectionScheme = PROTECTION_SCHEME) { + + Properties unprotectedProperties = new Properties() + unprotectedProperties.load(new FileReader(pathToOriginalUnprotectedProperties)) + + String[] populatedSensitiveProperties = sensitivePropertiesToVerify.findAll { + unprotectedProperties.getProperty(it) != null && unprotectedProperties.getProperty(it).toString().length() > 0 + } + def populatedSensitivePropertiesCount = populatedSensitiveProperties.length + + Properties protectedProperties = new Properties() + protectedProperties.load(new FileReader(pathToProtectedPropertiesToVerify)) + + // For each populated, sensitive property, one additional "*.protected" property should have been added + assert unprotectedProperties.size() + populatedSensitivePropertiesCount == protectedProperties.size() + + // For each populated, sensitive property, ensure its value differs from its original value, and + // that no two protected property values match (due to IV, which is unique per-property) + Set distinctValues = new HashSet<>() + populatedSensitiveProperties.every { key -> + def originalValue = unprotectedProperties.getProperty(key) + def protectedValue = protectedProperties.getProperty(key) + def protectionScheme = protectedProperties.getProperty("${key}.protected") + + assert null != protectedValue + assert protectedValue.length() > 0 + assert originalValue != protectedValue + assert expectedProtectionScheme == protectionScheme + + assert !distinctValues.contains(protectedValue) + distinctValues.add(protectedValue) + } + + return true + } + + /** + * Make assertions that a NiFi Registry Authorizers XML file is protected correctly given a known starting point. + * + * @param pathToOriginalUnprotectedXml - location of the original, plaintext XML file + * @param pathToProtectedXmlToVerify - location of the protected XML file + * @param expectedProtectionScheme - expected scheme/cipher used to encrypt + * @param expectedKey - key used to encrypt + * + * @return true if all assertions pass + * @throws AssertionError if any assertion fails + */ + static boolean assertRegistryAuthorizersXmlIsProtected( + String pathToOriginalUnprotectedXml, + String pathToProtectedXmlToVerify, + String expectedProtectionScheme = PROTECTION_SCHEME, + String expectedKey = KEY_HEX) { + + return assertXmlIsProtected( + pathToOriginalUnprotectedXml, + pathToProtectedXmlToVerify, + expectedProtectionScheme, + expectedKey, + { rootNode -> + try { + rootNode.userGroupProvider.find { + it.'class'.text() == NiFiRegistryAuthorizersXmlEncryptor.LDAP_USER_GROUP_PROVIDER_CLASS + }.property.findAll { + it.@name =~ "Password" + } + } catch (Exception ignored) { + null + } + + } + ) + } + + /** + * Make assertions that a NiFi Registry Identity Providers XML file is protected correctly given a known starting point. + * + * @param pathToOriginalUnprotectedXml - location of the original, plaintext XML file + * @param pathToProtectedXmlToVerify - location of the protected XML file + * @param expectedProtectionScheme - expected scheme/cipher used to encrypt + * @param expectedKey - key used to encrypt + * + * @return true if all assertions pass + * @throws AssertionError if any assertion fails + */ + static boolean assertRegistryIdentityProvidersXmlIsProtected( + String pathToOriginalUnprotectedXml, + String pathToProtectedXmlToVerify, + String expectedProtectionScheme = PROTECTION_SCHEME, + String expectedKey = KEY_HEX) { + + return assertXmlIsProtected( + pathToOriginalUnprotectedXml, + pathToProtectedXmlToVerify, + expectedProtectionScheme, + expectedKey, + { rootNode -> + try { + rootNode.provider.find { + it.'class'.text() == NiFiRegistryIdentityProvidersXmlEncryptor.LDAP_PROVIDER_CLASS + }.property.findAll { + it.@name =~ "Password" + } + } catch (Exception ignored) { + null + } + + } + ) + } + + /** + * Make assertions that an XML file is protected correctly given a known starting point. + * + * @param pathToOriginalUnprotectedXml - location of the original, plaintext XML file + * @param pathToProtectedXmlToVerify - location of the protected XML file + * @param expectedProtectionScheme - expected scheme/cipher used to encrypt + * @param expectedKey - key used to encrypt + * @param callbackToGetNodesToVerify - closure that returns GPathResult[] of all sensitive nodes that + * should be protected given a GPathResult for the root of the XML document + * + * @return true if all assertions pass + * @throws AssertionError if any assertion fails + */ + static boolean assertXmlIsProtected( + String pathToOriginalUnprotectedXml, + String pathToProtectedXmlToVerify, + String expectedProtectionScheme = PROTECTION_SCHEME, + String expectedKey = KEY_HEX, + callbackToGetNodesToVerify) { + + String originalUnprotectedXml = new File(pathToOriginalUnprotectedXml).text + String protectedXml = new File(pathToProtectedXmlToVerify).text + def originalDoc = new XmlParser().parseText(originalUnprotectedXml) + def protectedDoc = new XmlParser().parseText(protectedXml) + + def sensitiveProperties = callbackToGetNodesToVerify(originalDoc) + assert sensitiveProperties && sensitiveProperties.size > 0 // necessary as so many key assertions are based on at least one sensitive prop + def populatedSensitiveProperties = sensitiveProperties.findAll { node -> + node.text() + } + def plaintextValues = populatedSensitiveProperties.collect { + it.text() + } + + if (populatedSensitiveProperties.size() == 0) { + return assertFilesAreEqual(pathToOriginalUnprotectedXml, pathToProtectedXmlToVerify) + } + + def protectedSensitiveProperties = callbackToGetNodesToVerify(protectedDoc).findAll { node -> + node.@encryption != "none" && node.@encryption != "" } + + assert populatedSensitiveProperties.size() == protectedSensitiveProperties.size() + + AESSensitivePropertyProvider spp = new AESSensitivePropertyProvider(expectedKey) + + protectedSensitiveProperties.each { + String value = it.text() + String propertyValue = value + assert it.@encryption == expectedProtectionScheme + assert !plaintextValues.contains(propertyValue) + assert plaintextValues.contains(spp.unprotect(propertyValue)) + } + + return true + } + + /** + * Asserts the contents of files are equal, ignoring blank lines and starting / trailing whitespace + * + * @param pathToExpected - path to file with the expected content + * @param pathToActual - path to file with the actual content + * @return true if assertions pass + */ + static boolean assertFilesAreEqual(String pathToExpected, String pathToActual) { + List expectedLines = new File(pathToExpected).readLines().findAll{ + it.trim().length() > 0 + }.collect{ it.trim() } + List actualLines = new File(pathToActual).readLines().findAll{ + it.trim().length() > 0 + }.collect{ it.trim() } + + return assertLinesAreEqual(expectedLines, actualLines) + } + + /** + * Asserts the contents of a bootstrap.conf file match that of an an expected bootstrap.conf. + * + * @param pathToExpectedBootstrap + * @param pathToActualBootstrap + * @param includeComments - if false, comment lines in the bootstrap.conf files will be ignored + * @return true if assertions pass + */ + static boolean assertBootstrapFilesAreEqual(String pathToExpectedBootstrap, String pathToActualBootstrap, boolean includeComments) { + return assertConfOrPropertiesFilesAreEqual(pathToExpectedBootstrap, pathToActualBootstrap, includeComments) + } + + /** + * Asserts the contents of a properties file match that of an an expected properties file. + * + * @param pathToExpectedProperties + * @param pathToActualProperties + * @param includeComments - if false, comment lines in the properties files will be ignored + * @return true if assertions pass + */ + static boolean assertPropertiesFilesAreEqual(String pathToExpectedProperties, String pathToActualProperties, boolean includeComments) { + return assertConfOrPropertiesFilesAreEqual(pathToExpectedProperties, pathToActualProperties, includeComments) + } + + private static boolean assertConfOrPropertiesFilesAreEqual(String expected, String actual, boolean includeComments) { + List expectedLines = new File(expected).readLines().findAll{ + (it.trim().length() > 0 && (includeComments || !it.startsWith("#"))) + }.collect{ it.trim() } + List actualLines = new File(actual).readLines().findAll{ + (it.trim().length() > 0 && (includeComments || !it.startsWith("#"))) + }.collect{ it.trim() } + + return assertLinesAreEqual(expectedLines, actualLines) + } + + private static boolean assertLinesAreEqual(List expectedLines, List actualLines) { + + assert actualLines != null + assert actualLines.size() == expectedLines.size() + assert actualLines == expectedLines + + return true + } + +} diff --git a/nifi-toolkit/nifi-toolkit-encrypt-config/src/test/groovy/org/apache/nifi/toolkit/encryptconfig/util/BootstrapUtilSpec.groovy b/nifi-toolkit/nifi-toolkit-encrypt-config/src/test/groovy/org/apache/nifi/toolkit/encryptconfig/util/BootstrapUtilSpec.groovy new file mode 100644 index 0000000000..1278a58335 --- /dev/null +++ b/nifi-toolkit/nifi-toolkit-encrypt-config/src/test/groovy/org/apache/nifi/toolkit/encryptconfig/util/BootstrapUtilSpec.groovy @@ -0,0 +1,113 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.toolkit.encryptconfig.util + +import org.slf4j.Logger +import org.slf4j.LoggerFactory +import spock.lang.Specification + +import static org.apache.nifi.toolkit.encryptconfig.TestUtil.* + +class BootstrapUtilSpec extends Specification { + private static final Logger logger = LoggerFactory.getLogger(BootstrapUtilSpec.class) + + // runs before every feature method + def setup() {} + + // runs after every feature method + def cleanup() {} + + // runs before the first feature method + def setupSpec() { + setupTmpDir() + } + + // runs after the last feature method + def cleanupSpec() { + cleanupTmpDir() + } + + def "test extractKeyFromBootstrapFile with Registry bootstrap.conf"() { + + setup: + def bootstrapKeyProperty = BootstrapUtil.REGISTRY_BOOTSTRAP_KEY_PROPERTY + + + when: "bootstrap.conf has no key property" + def actualKeyHex = BootstrapUtil.extractKeyFromBootstrapFile(RESOURCE_REGISTRY_BOOTSTRAP_NO_KEY, bootstrapKeyProperty) + + then: "null is returned" + actualKeyHex == null + + + when: "bootstrap.conf has an empty key property" + actualKeyHex = BootstrapUtil.extractKeyFromBootstrapFile(RESOURCE_REGISTRY_BOOTSTRAP_EMPTY_KEY, bootstrapKeyProperty) + + then: "null is returned" + actualKeyHex == null + + + when: "bootstrap.conf has a populated key property" + actualKeyHex = BootstrapUtil.extractKeyFromBootstrapFile(RESOURCE_REGISTRY_BOOTSTRAP_KEY_128, bootstrapKeyProperty) + + then: "key is returned" + actualKeyHex == KEY_HEX_128 + + + when: "bootstrap.conf file does not exist" + BootstrapUtil.extractKeyFromBootstrapFile("__file_does_not_exist__", bootstrapKeyProperty) + + then: "expect an IOException" + thrown IOException + + } + + def "test writeKeyToBootstrapFile with Registry bootstrap.conf"() { + + setup: + def bootstrapKeyProperty = BootstrapUtil.REGISTRY_BOOTSTRAP_KEY_PROPERTY + def outFile1 = generateTmpFilePath() + def outFile2 = generateTmpFilePath() + def outFile3 = generateTmpFilePath() + def expected = RESOURCE_REGISTRY_BOOTSTRAP_KEY_128 + + + when: "input is default bootstrap.conf" + BootstrapUtil.writeKeyToBootstrapFile(KEY_HEX_128, bootstrapKeyProperty, outFile1, RESOURCE_REGISTRY_BOOTSTRAP_DEFAULT) + + then: "output file content matches populated bootstrap file" + assertBootstrapFilesAreEqual(expected, outFile1, true) + and: "key is readable from output file" + BootstrapUtil.extractKeyFromBootstrapFile(outFile1, bootstrapKeyProperty) == KEY_HEX_128 + + + when: "input bootstrap.conf has no key property" + BootstrapUtil.writeKeyToBootstrapFile(KEY_HEX_128, bootstrapKeyProperty, outFile2, RESOURCE_REGISTRY_BOOTSTRAP_NO_KEY) + + then: "output file content matches pre-populated bootstrap file" + assertBootstrapFilesAreEqual(expected, outFile2, true) + + + when: "input bootstrap.conf has existing, different master key" + BootstrapUtil.writeKeyToBootstrapFile(KEY_HEX_128, bootstrapKeyProperty, outFile3, RESOURCE_REGISTRY_BOOTSTRAP_KEY_FROM_PASSWORD_128) + + then: "output file content matches pre-populated bootstrap file" + assertBootstrapFilesAreEqual(expected, outFile3, true) + + } + +} diff --git a/nifi-toolkit/nifi-toolkit-encrypt-config/src/test/resources/nifi-registry/authorizers-commented.xml b/nifi-toolkit/nifi-toolkit-encrypt-config/src/test/resources/nifi-registry/authorizers-commented.xml new file mode 100644 index 0000000000..af0c531528 --- /dev/null +++ b/nifi-toolkit/nifi-toolkit-encrypt-config/src/test/resources/nifi-registry/authorizers-commented.xml @@ -0,0 +1,242 @@ + + + + + + + + file-user-group-provider + org.apache.nifi.registry.security.authorization.file.FileUserGroupProvider + ./conf/users.xml + + + + + + + + + + + + + + + file-access-policy-provider + org.apache.nifi.registry.security.authorization.file.FileAccessPolicyProvider + file-user-group-provider + ./conf/authorizations.xml + + + + + + + + managed-authorizer + org.apache.nifi.registry.security.authorization.StandardManagedAuthorizer + file-access-policy-provider + + + \ No newline at end of file diff --git a/nifi-toolkit/nifi-toolkit-encrypt-config/src/test/resources/nifi-registry/authorizers-empty.xml b/nifi-toolkit/nifi-toolkit-encrypt-config/src/test/resources/nifi-registry/authorizers-empty.xml new file mode 100644 index 0000000000..d70d907225 --- /dev/null +++ b/nifi-toolkit/nifi-toolkit-encrypt-config/src/test/resources/nifi-registry/authorizers-empty.xml @@ -0,0 +1,240 @@ + + + + + + + + file-user-group-provider + org.apache.nifi.registry.security.authorization.file.FileUserGroupProvider + ./conf/users.xml + + + + + + ldap-user-group-provider + org.apache.nifi.registry.security.ldap.tenants.LdapUserGroupProvider + START_TLS + + + + + + + + + + + + + + + FOLLOW + 10 secs + 10 secs + + + + 30 mins + + + person + ONE_LEVEL + + + + + + group + ONE_LEVEL + + + + + + + + + + + + + + file-access-policy-provider + org.apache.nifi.registry.security.authorization.file.FileAccessPolicyProvider + file-user-group-provider + ./conf/authorizations.xml + + + + + + + + managed-authorizer + org.apache.nifi.registry.security.authorization.StandardManagedAuthorizer + file-access-policy-provider + + + \ No newline at end of file diff --git a/nifi-toolkit/nifi-toolkit-encrypt-config/src/test/resources/nifi-registry/authorizers-populated-unprotected.xml b/nifi-toolkit/nifi-toolkit-encrypt-config/src/test/resources/nifi-registry/authorizers-populated-unprotected.xml new file mode 100644 index 0000000000..258ce285bd --- /dev/null +++ b/nifi-toolkit/nifi-toolkit-encrypt-config/src/test/resources/nifi-registry/authorizers-populated-unprotected.xml @@ -0,0 +1,246 @@ + + + + + + + + + + + ldap-user-group-provider + org.apache.nifi.registry.security.ldap.tenants.LdapUserGroupProvider + START_TLS + + someuser + thisIsABadPassword + + + thisIsABadPassword + + + thisIsABadPassword + + + + + + FOLLOW + 10 secs + 10 secs + + + + 30 mins + + + person + ONE_LEVEL + + + + + + + group + ONE_LEVEL + + + + + + + + + + + + + + + file-access-policy-provider + org.apache.nifi.registry.security.authorization.file.FileAccessPolicyProvider + ldap-user-group-provider + ./conf/authorizations.xml + nobel + + + + + + + managed-authorizer + org.apache.nifi.registry.security.authorization.StandardManagedAuthorizer + file-access-policy-provider + + + \ No newline at end of file diff --git a/nifi-toolkit/nifi-toolkit-encrypt-config/src/test/resources/nifi-registry/bootstrap_default.conf b/nifi-toolkit/nifi-toolkit-encrypt-config/src/test/resources/nifi-registry/bootstrap_default.conf new file mode 100644 index 0000000000..637eb64667 --- /dev/null +++ b/nifi-toolkit/nifi-toolkit-encrypt-config/src/test/resources/nifi-registry/bootstrap_default.conf @@ -0,0 +1,48 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# Java command to use when running nifi-registry +java=java + +# Username to use when running nifi-registry. This value will be ignored on Windows. +run.as= + +# Configure where nifi-registry's lib and conf directories live +lib.dir=./lib +conf.dir=./conf + +# How long to wait after telling nifi-registry to shutdown before explicitly killing the Process +graceful.shutdown.seconds=20 + +# Disable JSR 199 so that we can use JSP's without running a JDK +java.arg.1=-Dorg.apache.jasper.compiler.disablejsr199=true + +# JVM memory settings +java.arg.2=-Xms512m +java.arg.3=-Xmx512m + +# Enable Remote Debugging +#java.arg.debug=-agentlib:jdwp=transport=dt_socket,server=y,suspend=n,address=8000 + +java.arg.4=-Djava.net.preferIPv4Stack=true + +# allowRestrictedHeaders is required for Cluster/Node communications to work properly +java.arg.5=-Dsun.net.http.allowRestrictedHeaders=true +java.arg.6=-Djava.protocol.handler.pkgs=sun.net.www.protocol + +# Master key in hexadecimal format for encrypted sensitive configuration values +nifi.registry.bootstrap.sensitive.key= \ No newline at end of file diff --git a/nifi-toolkit/nifi-toolkit-encrypt-config/src/test/resources/nifi-registry/bootstrap_with_empty_master_key.conf b/nifi-toolkit/nifi-toolkit-encrypt-config/src/test/resources/nifi-registry/bootstrap_with_empty_master_key.conf new file mode 100644 index 0000000000..637eb64667 --- /dev/null +++ b/nifi-toolkit/nifi-toolkit-encrypt-config/src/test/resources/nifi-registry/bootstrap_with_empty_master_key.conf @@ -0,0 +1,48 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# Java command to use when running nifi-registry +java=java + +# Username to use when running nifi-registry. This value will be ignored on Windows. +run.as= + +# Configure where nifi-registry's lib and conf directories live +lib.dir=./lib +conf.dir=./conf + +# How long to wait after telling nifi-registry to shutdown before explicitly killing the Process +graceful.shutdown.seconds=20 + +# Disable JSR 199 so that we can use JSP's without running a JDK +java.arg.1=-Dorg.apache.jasper.compiler.disablejsr199=true + +# JVM memory settings +java.arg.2=-Xms512m +java.arg.3=-Xmx512m + +# Enable Remote Debugging +#java.arg.debug=-agentlib:jdwp=transport=dt_socket,server=y,suspend=n,address=8000 + +java.arg.4=-Djava.net.preferIPv4Stack=true + +# allowRestrictedHeaders is required for Cluster/Node communications to work properly +java.arg.5=-Dsun.net.http.allowRestrictedHeaders=true +java.arg.6=-Djava.protocol.handler.pkgs=sun.net.www.protocol + +# Master key in hexadecimal format for encrypted sensitive configuration values +nifi.registry.bootstrap.sensitive.key= \ No newline at end of file diff --git a/nifi-toolkit/nifi-toolkit-encrypt-config/src/test/resources/nifi-registry/bootstrap_with_master_key_128.conf b/nifi-toolkit/nifi-toolkit-encrypt-config/src/test/resources/nifi-registry/bootstrap_with_master_key_128.conf new file mode 100644 index 0000000000..6e128bdfa5 --- /dev/null +++ b/nifi-toolkit/nifi-toolkit-encrypt-config/src/test/resources/nifi-registry/bootstrap_with_master_key_128.conf @@ -0,0 +1,48 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# Java command to use when running nifi-registry +java=java + +# Username to use when running nifi-registry. This value will be ignored on Windows. +run.as= + +# Configure where nifi-registry's lib and conf directories live +lib.dir=./lib +conf.dir=./conf + +# How long to wait after telling nifi-registry to shutdown before explicitly killing the Process +graceful.shutdown.seconds=20 + +# Disable JSR 199 so that we can use JSP's without running a JDK +java.arg.1=-Dorg.apache.jasper.compiler.disablejsr199=true + +# JVM memory settings +java.arg.2=-Xms512m +java.arg.3=-Xmx512m + +# Enable Remote Debugging +#java.arg.debug=-agentlib:jdwp=transport=dt_socket,server=y,suspend=n,address=8000 + +java.arg.4=-Djava.net.preferIPv4Stack=true + +# allowRestrictedHeaders is required for Cluster/Node communications to work properly +java.arg.5=-Dsun.net.http.allowRestrictedHeaders=true +java.arg.6=-Djava.protocol.handler.pkgs=sun.net.www.protocol + +# Master key in hexadecimal format for encrypted sensitive configuration values +nifi.registry.bootstrap.sensitive.key=0123456789ABCDEFFEDCBA9876543210 \ No newline at end of file diff --git a/nifi-toolkit/nifi-toolkit-encrypt-config/src/test/resources/nifi-registry/bootstrap_with_master_key_from_password_128.conf b/nifi-toolkit/nifi-toolkit-encrypt-config/src/test/resources/nifi-registry/bootstrap_with_master_key_from_password_128.conf new file mode 100644 index 0000000000..16788a0536 --- /dev/null +++ b/nifi-toolkit/nifi-toolkit-encrypt-config/src/test/resources/nifi-registry/bootstrap_with_master_key_from_password_128.conf @@ -0,0 +1,48 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# Java command to use when running nifi-registry +java=java + +# Username to use when running nifi-registry. This value will be ignored on Windows. +run.as= + +# Configure where nifi-registry's lib and conf directories live +lib.dir=./lib +conf.dir=./conf + +# How long to wait after telling nifi-registry to shutdown before explicitly killing the Process +graceful.shutdown.seconds=20 + +# Disable JSR 199 so that we can use JSP's without running a JDK +java.arg.1=-Dorg.apache.jasper.compiler.disablejsr199=true + +# JVM memory settings +java.arg.2=-Xms512m +java.arg.3=-Xmx512m + +# Enable Remote Debugging +#java.arg.debug=-agentlib:jdwp=transport=dt_socket,server=y,suspend=n,address=8000 + +java.arg.4=-Djava.net.preferIPv4Stack=true + +# allowRestrictedHeaders is required for Cluster/Node communications to work properly +java.arg.5=-Dsun.net.http.allowRestrictedHeaders=true +java.arg.6=-Djava.protocol.handler.pkgs=sun.net.www.protocol + +# Master key in hexadecimal format for encrypted sensitive configuration values +nifi.registry.bootstrap.sensitive.key=2C576A9585DB862F5ECBEE5B4FFFCCA1 \ No newline at end of file diff --git a/nifi-toolkit/nifi-toolkit-encrypt-config/src/test/resources/nifi-registry/bootstrap_without_master_key.conf b/nifi-toolkit/nifi-toolkit-encrypt-config/src/test/resources/nifi-registry/bootstrap_without_master_key.conf new file mode 100644 index 0000000000..7abf30cfb4 --- /dev/null +++ b/nifi-toolkit/nifi-toolkit-encrypt-config/src/test/resources/nifi-registry/bootstrap_without_master_key.conf @@ -0,0 +1,45 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# Java command to use when running nifi-registry +java=java + +# Username to use when running nifi-registry. This value will be ignored on Windows. +run.as= + +# Configure where nifi-registry's lib and conf directories live +lib.dir=./lib +conf.dir=./conf + +# How long to wait after telling nifi-registry to shutdown before explicitly killing the Process +graceful.shutdown.seconds=20 + +# Disable JSR 199 so that we can use JSP's without running a JDK +java.arg.1=-Dorg.apache.jasper.compiler.disablejsr199=true + +# JVM memory settings +java.arg.2=-Xms512m +java.arg.3=-Xmx512m + +# Enable Remote Debugging +#java.arg.debug=-agentlib:jdwp=transport=dt_socket,server=y,suspend=n,address=8000 + +java.arg.4=-Djava.net.preferIPv4Stack=true + +# allowRestrictedHeaders is required for Cluster/Node communications to work properly +java.arg.5=-Dsun.net.http.allowRestrictedHeaders=true +java.arg.6=-Djava.protocol.handler.pkgs=sun.net.www.protocol diff --git a/nifi-toolkit/nifi-toolkit-encrypt-config/src/test/resources/nifi-registry/identity-providers-commented.xml b/nifi-toolkit/nifi-toolkit-encrypt-config/src/test/resources/nifi-registry/identity-providers-commented.xml new file mode 100644 index 0000000000..1e8cf6467e --- /dev/null +++ b/nifi-toolkit/nifi-toolkit-encrypt-config/src/test/resources/nifi-registry/identity-providers-commented.xml @@ -0,0 +1,106 @@ + + + + + + + + + + + \ No newline at end of file diff --git a/nifi-toolkit/nifi-toolkit-encrypt-config/src/test/resources/nifi-registry/identity-providers-empty.xml b/nifi-toolkit/nifi-toolkit-encrypt-config/src/test/resources/nifi-registry/identity-providers-empty.xml new file mode 100644 index 0000000000..1d075ace55 --- /dev/null +++ b/nifi-toolkit/nifi-toolkit-encrypt-config/src/test/resources/nifi-registry/identity-providers-empty.xml @@ -0,0 +1,104 @@ + + + + + + + ldap-identity-provider + org.apache.nifi.registry.security.ldap.LdapIdentityProvider + SIMPLE + + + + + FOLLOW + 10 secs + 10 secs + + + + + + USE_USERNAME + 12 hours + + + + + + \ No newline at end of file diff --git a/nifi-toolkit/nifi-toolkit-encrypt-config/src/test/resources/nifi-registry/identity-providers-populated-unprotected.xml b/nifi-toolkit/nifi-toolkit-encrypt-config/src/test/resources/nifi-registry/identity-providers-populated-unprotected.xml new file mode 100644 index 0000000000..67b7a18b67 --- /dev/null +++ b/nifi-toolkit/nifi-toolkit-encrypt-config/src/test/resources/nifi-registry/identity-providers-populated-unprotected.xml @@ -0,0 +1,97 @@ + + + + + + + ldap-identity-provider + org.apache.nifi.registry.security.ldap.LdapIdentityProvider + START_TLS + + someuser + thisIsABadPassword + + + thisIsABadPassword + + + thisIsABadPassword + + + + + + FOLLOW + 10 secs + 10 secs + + + + + + 12 hours + + + \ No newline at end of file diff --git a/nifi-toolkit/nifi-toolkit-encrypt-config/src/test/resources/nifi-registry/nifi-registry-commented.properties b/nifi-toolkit/nifi-toolkit-encrypt-config/src/test/resources/nifi-registry/nifi-registry-commented.properties new file mode 100644 index 0000000000..05d233b900 --- /dev/null +++ b/nifi-toolkit/nifi-toolkit-encrypt-config/src/test/resources/nifi-registry/nifi-registry-commented.properties @@ -0,0 +1,31 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# web properties # +nifi.registry.web.war.directory=./lib +nifi.registry.web.http.host=localhost +nifi.registry.web.http.port=18080 +#nifi.registry.web.https.host=localhost +#nifi.registry.web.https.port=18443 +nifi.registry.web.jetty.working.directory=./work/jetty +nifi.registry.web.jetty.threads=10 + +# security properties # +#nifi.registry.security.keystorePasswd= +#nifi.registry.security.keyPasswd= +#nifi.registry.security.truststorePasswd= + +# sensitive property protection properties # +#nifi.registry.sensitive.props.additional.keys= \ No newline at end of file diff --git a/nifi-toolkit/nifi-toolkit-encrypt-config/src/test/resources/nifi-registry/nifi-registry-empty.properties b/nifi-toolkit/nifi-toolkit-encrypt-config/src/test/resources/nifi-registry/nifi-registry-empty.properties new file mode 100644 index 0000000000..8f7907ffe0 --- /dev/null +++ b/nifi-toolkit/nifi-toolkit-encrypt-config/src/test/resources/nifi-registry/nifi-registry-empty.properties @@ -0,0 +1,31 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# web properties # +nifi.registry.web.war.directory=./lib +nifi.registry.web.http.host=localhost +nifi.registry.web.http.port=18080 +#nifi.registry.web.https.host=localhost +#nifi.registry.web.https.port=18443 +nifi.registry.web.jetty.working.directory=./work/jetty +nifi.registry.web.jetty.threads=10 + +# security properties # +nifi.registry.security.keystorePasswd= +nifi.registry.security.keyPasswd= +nifi.registry.security.truststorePasswd= + +# sensitive property protection properties # +nifi.registry.sensitive.props.additional.keys= \ No newline at end of file diff --git a/nifi-toolkit/nifi-toolkit-encrypt-config/src/test/resources/nifi-registry/nifi-registry-populated-protected-key-128.properties b/nifi-toolkit/nifi-toolkit-encrypt-config/src/test/resources/nifi-registry/nifi-registry-populated-protected-key-128.properties new file mode 100644 index 0000000000..546443676a --- /dev/null +++ b/nifi-toolkit/nifi-toolkit-encrypt-config/src/test/resources/nifi-registry/nifi-registry-populated-protected-key-128.properties @@ -0,0 +1,50 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# web properties # +nifi.registry.web.war.directory=./lib +#nifi.registry.web.http.host=localhost +#nifi.registry.web.http.port=8080 +nifi.registry.web.https.host=localhost +nifi.registry.web.https.port=8443 +nifi.registry.web.jetty.working.directory=./work/jetty +nifi.registry.web.jetty.threads=10 + +# security properties # +nifi.registry.security.keystore=/path/to/keystore.jks +nifi.registry.security.keystoreType=JKS +nifi.registry.security.keystorePasswd=IpbMQE9HsW5ZcwbK||/5jdbYqORj8aZfw5UPx2PSVUamunQt6uxmaYcaDbeCxAaPKhAUCN3bnX +nifi.registry.security.keystorePasswd.protected=aes/gcm/128 +nifi.registry.security.keyPasswd=MuTrxqJQBrIJE5eq||VDbTIPVA9zijI6eZ1Z6VMU7xhpBOxPjJLQ48h5GhIX7BY/bQzA +nifi.registry.security.keyPasswd.protected=aes/gcm/128 +nifi.registry.security.truststore=/path/to/truststore.jks +nifi.registry.security.truststoreType=JKS +nifi.registry.security.truststorePasswd=Pso0P5eiT+HF0sfy||cev+b7rbqR+s94t9uWkhZly6AT00AV5bsS8D+ok/oTx81FV3IMkZzIzlsEI +nifi.registry.security.truststorePasswd.protected=aes/gcm/128 +nifi.registry.security.needClientAuth=false +nifi.registry.security.authorizers.configuration.file=./conf/authorizers.xml +nifi.registry.security.authorizer=managed-authorizer +nifi.registry.security.identity.providers.configuration.file=./conf/identity-providers.xml +nifi.registry.security.identity.provider=ldap-identity-provider + +# sensitive property protection properties # +nifi.registry.sensitive.props.additional.keys=nifi.registry.dummy.sensitive.property.1,nifi.registry.dummy.sensitive.property.2 +nifi.registry.dummy.sensitive.property.1=XDXDfZ2e2dqZF4HM||kfeu78d1HxCNZ5Ljq/RXrAQd3PEXPA +nifi.registry.dummy.sensitive.property.1.protected=aes/gcm/128 +nifi.registry.dummy.sensitive.property.2=GElHCO9gRNkV8EPh||jfdlaaU82FJZ9SSHWsyEtgYqBIancA +nifi.registry.dummy.sensitive.property.2.protected=aes/gcm/128 + +# providers properties # +nifi.registry.providers.configuration.file=./conf/providers.xml \ No newline at end of file diff --git a/nifi-toolkit/nifi-toolkit-encrypt-config/src/test/resources/nifi-registry/nifi-registry-populated-protected-key-256.properties b/nifi-toolkit/nifi-toolkit-encrypt-config/src/test/resources/nifi-registry/nifi-registry-populated-protected-key-256.properties new file mode 100644 index 0000000000..37eb356cda --- /dev/null +++ b/nifi-toolkit/nifi-toolkit-encrypt-config/src/test/resources/nifi-registry/nifi-registry-populated-protected-key-256.properties @@ -0,0 +1,50 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# web properties # +nifi.registry.web.war.directory=./lib +#nifi.registry.web.http.host=localhost +#nifi.registry.web.http.port=8080 +nifi.registry.web.https.host=localhost +nifi.registry.web.https.port=8443 +nifi.registry.web.jetty.working.directory=./work/jetty +nifi.registry.web.jetty.threads=10 + +# security properties # +nifi.registry.security.keystore=/path/to/keystore.jks +nifi.registry.security.keystoreType=JKS +nifi.registry.security.keystorePasswd=UxL26wbxvVVOwxDb||mWErKKQ4WpGbLWWrvPNfxVswVwrO68GMLa3RbRyxLyogvGg9Zj79INuu +nifi.registry.security.keystorePasswd.protected=aes/gcm/256 +nifi.registry.security.keyPasswd=n5np+8Fc7QlSGicG||1eOSgx39YwGJAwqbVW/t1Lwjoz7aYUzySCUoWttR+HRct9nQLg +nifi.registry.security.keyPasswd.protected=aes/gcm/256 +nifi.registry.security.truststore=/path/to/truststore.jks +nifi.registry.security.truststoreType=JKS +nifi.registry.security.truststorePasswd=zpPQ4kXKwWFsAE0R||e+Ht1rplq7S1Nn5UMt8lmTK4FhCqScuXf2ERFhpeo8QF/Pd017F7NB/sIbE +nifi.registry.security.truststorePasswd.protected=aes/gcm/256 +nifi.registry.security.needClientAuth=false +nifi.registry.security.authorizers.configuration.file=./conf/authorizers.xml +nifi.registry.security.authorizer=managed-authorizer +nifi.registry.security.identity.providers.configuration.file=./conf/identity-providers.xml +nifi.registry.security.identity.provider=ldap-identity-provider + +# sensitive property protection properties # +nifi.registry.sensitive.props.additional.keys=nifi.registry.dummy.sensitive.property.1,nifi.registry.dummy.sensitive.property.2 +nifi.registry.dummy.sensitive.property.1=vwSuUUXRZVI4Jau7||pA5Y1TiUt7jlPMgLz6fyozGk3Kywog +nifi.registry.dummy.sensitive.property.1.protected=aes/gcm/256 +nifi.registry.dummy.sensitive.property.2=bFdNvsYJwGOd36IX||qhmzpXmDIOBbjiUs2QDo5uwtlRZRuQ +nifi.registry.dummy.sensitive.property.2.protected=aes/gcm/256 + +# providers properties # +nifi.registry.providers.configuration.file=./conf/providers.xml \ No newline at end of file diff --git a/nifi-toolkit/nifi-toolkit-encrypt-config/src/test/resources/nifi-registry/nifi-registry-populated-protected-password-256.properties b/nifi-toolkit/nifi-toolkit-encrypt-config/src/test/resources/nifi-registry/nifi-registry-populated-protected-password-256.properties new file mode 100644 index 0000000000..e59c885ca0 --- /dev/null +++ b/nifi-toolkit/nifi-toolkit-encrypt-config/src/test/resources/nifi-registry/nifi-registry-populated-protected-password-256.properties @@ -0,0 +1,52 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# web properties # +nifi.registry.web.war.directory=./lib +#nifi.registry.web.http.host=localhost +#nifi.registry.web.http.port=8080 +nifi.registry.web.https.host=localhost +nifi.registry.web.https.port=8443 +nifi.registry.web.jetty.working.directory=./work/jetty +nifi.registry.web.jetty.threads=10 + +# security properties # +nifi.registry.security.keystore=/path/to/keystore.jks +nifi.registry.security.keystoreType=JKS +nifi.registry.security.keystorePasswd=hYUecef4Tl7j82Ml||1VvxnSFxzrrMU6gcHgt/1M69uuk4OsbsRuN4x9FUY3p7frQ3m15SjyV1 +nifi.registry.security.keyPasswd=Ex+EVvW31ZFwTxoe||obIlGXnqhfAHngV6tie577PPKOuU1+B7osTL3wJ6t4z74C5PKw +nifi.registry.security.truststore=/path/to/truststore.jks +nifi.registry.security.truststoreType=JKS +nifi.registry.security.truststorePasswd=qVLfrvg+UgwD1J5p||b6FHicQmve1toY56MLrwJDturN3GrYptSMJ+DP6FaixiTrtyo8L+cwHZkiA +nifi.registry.security.needClientAuth=false +nifi.registry.security.authorizers.configuration.file=./conf/authorizers.xml +nifi.registry.security.authorizer=managed-authorizer +nifi.registry.security.identity.providers.configuration.file=./conf/identity-providers.xml +nifi.registry.security.identity.provider=ldap-identity-provider + +# sensitive property protection properties # +nifi.registry.sensitive.props.additional.keys=nifi.registry.dummy.sensitive.property.1,nifi.registry.dummy.sensitive.property.2 +nifi.registry.dummy.sensitive.property.1=oGB/wu12Cb0xAqsl||tEuOJNxIAJQdNGh1bRXLWeskI7MUTg +nifi.registry.dummy.sensitive.property.2=RnYVB0CZC2CerkYY||bhGllLX3oIwSxJy9HqBX/DV8gKwSKA + +# providers properties # +nifi.registry.providers.configuration.file=./conf/providers.xml + +# protection properties +nifi.registry.dummy.sensitive.property.1.protected=aes/gcm/256 +nifi.registry.dummy.sensitive.property.2.protected=aes/gcm/256 +nifi.registry.security.keyPasswd.protected=aes/gcm/256 +nifi.registry.security.keystorePasswd.protected=aes/gcm/256 +nifi.registry.security.truststorePasswd.protected=aes/gcm/256 \ No newline at end of file diff --git a/nifi-toolkit/nifi-toolkit-encrypt-config/src/test/resources/nifi-registry/nifi-registry-populated-unprotected.properties b/nifi-toolkit/nifi-toolkit-encrypt-config/src/test/resources/nifi-registry/nifi-registry-populated-unprotected.properties new file mode 100644 index 0000000000..1a46c2e5ae --- /dev/null +++ b/nifi-toolkit/nifi-toolkit-encrypt-config/src/test/resources/nifi-registry/nifi-registry-populated-unprotected.properties @@ -0,0 +1,45 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# web properties # +nifi.registry.web.war.directory=./lib +#nifi.registry.web.http.host=localhost +#nifi.registry.web.http.port=8080 +nifi.registry.web.https.host=localhost +nifi.registry.web.https.port=8443 +nifi.registry.web.jetty.working.directory=./work/jetty +nifi.registry.web.jetty.threads=10 + +# security properties # +nifi.registry.security.keystore=/path/to/keystore.jks +nifi.registry.security.keystoreType=JKS +nifi.registry.security.keystorePasswd=thisIsABadKeystorePassword +nifi.registry.security.keyPasswd=thisIsABadKeyPassword +nifi.registry.security.truststore=/path/to/truststore.jks +nifi.registry.security.truststoreType=JKS +nifi.registry.security.truststorePasswd=thisIsABadTruststorePassword +nifi.registry.security.needClientAuth=false +nifi.registry.security.authorizers.configuration.file=./conf/authorizers.xml +nifi.registry.security.authorizer=managed-authorizer +nifi.registry.security.identity.providers.configuration.file=./conf/identity-providers.xml +nifi.registry.security.identity.provider=ldap-identity-provider + +# sensitive property protection properties # +nifi.registry.sensitive.props.additional.keys=nifi.registry.dummy.sensitive.property.1,nifi.registry.dummy.sensitive.property.2 +nifi.registry.dummy.sensitive.property.1=secret +nifi.registry.dummy.sensitive.property.2=secret + +# providers properties # +nifi.registry.providers.configuration.file=./conf/providers.xml