diff --git a/nifi-bootstrap/pom.xml b/nifi-bootstrap/pom.xml index c6e8428f30..68663855bd 100644 --- a/nifi-bootstrap/pom.xml +++ b/nifi-bootstrap/pom.xml @@ -40,6 +40,12 @@ language governing permissions and limitations under the License. --> nifi-security-utils 1.14.0-SNAPSHOT + + org.apache.nifi + nifi-flow-encryptor + 1.14.0-SNAPSHOT + runtime + javax.mail mail diff --git a/nifi-commons/nifi-flow-encryptor/pom.xml b/nifi-commons/nifi-flow-encryptor/pom.xml new file mode 100644 index 0000000000..d4ed9ee2b8 --- /dev/null +++ b/nifi-commons/nifi-flow-encryptor/pom.xml @@ -0,0 +1,38 @@ + + + + 4.0.0 + + org.apache.nifi + nifi-commons + 1.14.0-SNAPSHOT + + nifi-flow-encryptor + + + org.apache.nifi + nifi-property-encryptor + 1.14.0-SNAPSHOT + + + + org.apache.nifi + nifi-properties + + + + + diff --git a/nifi-commons/nifi-flow-encryptor/src/main/java/org/apache/nifi/flow/encryptor/FlowEncryptor.java b/nifi-commons/nifi-flow-encryptor/src/main/java/org/apache/nifi/flow/encryptor/FlowEncryptor.java new file mode 100644 index 0000000000..5ff68bdea2 --- /dev/null +++ b/nifi-commons/nifi-flow-encryptor/src/main/java/org/apache/nifi/flow/encryptor/FlowEncryptor.java @@ -0,0 +1,37 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.flow.encryptor; + +import org.apache.nifi.encrypt.PropertyEncryptor; + +import java.io.InputStream; +import java.io.OutputStream; + +/** + * Flow Encryptor for reading a Flow Configuration and writing a new Flow Configuration using a new password + */ +public interface FlowEncryptor { + /** + * Process Flow Configuration Stream + * + * @param inputStream Flow Configuration Input Stream + * @param outputStream Flow Configuration Output Stream encrypted using new password + * @param inputEncryptor Property Encryptor for Input Configuration + * @param outputEncryptor Property Encryptor for Output Configuration + */ + void processFlow(InputStream inputStream, OutputStream outputStream, PropertyEncryptor inputEncryptor, PropertyEncryptor outputEncryptor); +} diff --git a/nifi-commons/nifi-flow-encryptor/src/main/java/org/apache/nifi/flow/encryptor/StandardFlowEncryptor.java b/nifi-commons/nifi-flow-encryptor/src/main/java/org/apache/nifi/flow/encryptor/StandardFlowEncryptor.java new file mode 100644 index 0000000000..b29613b6af --- /dev/null +++ b/nifi-commons/nifi-flow-encryptor/src/main/java/org/apache/nifi/flow/encryptor/StandardFlowEncryptor.java @@ -0,0 +1,75 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.flow.encryptor; + +import org.apache.nifi.encrypt.PropertyEncryptor; + +import java.io.BufferedReader; +import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.io.OutputStream; +import java.io.OutputStreamWriter; +import java.io.PrintWriter; +import java.io.UncheckedIOException; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +/** + * Standard Flow Encryptor handles reading Input Steam and writing Output Stream + */ +public class StandardFlowEncryptor implements FlowEncryptor { + private static final Pattern ENCRYPTED_PATTERN = Pattern.compile("enc\\{([^\\}]+?)\\}"); + + private static final int FIRST_GROUP = 1; + + private static final String ENCRYPTED_FORMAT = "enc{%s}"; + + /** + * Process Flow Configuration Stream replacing existing encrypted properties with new encrypted properties + * + * @param inputStream Flow Configuration Input Stream + * @param outputStream Flow Configuration Output Stream encrypted using new password + * @param inputEncryptor Property Encryptor for Input Configuration + * @param outputEncryptor Property Encryptor for Output Configuration + */ + @Override + public void processFlow(final InputStream inputStream, final OutputStream outputStream, final PropertyEncryptor inputEncryptor, final PropertyEncryptor outputEncryptor) { + try (final PrintWriter writer = new PrintWriter(new OutputStreamWriter(outputStream))) { + try (final BufferedReader reader = new BufferedReader(new InputStreamReader(inputStream))) { + reader.lines().forEach(line -> { + final Matcher matcher = ENCRYPTED_PATTERN.matcher(line); + if (matcher.find()) { + final String outputEncrypted = getOutputEncrypted(matcher.group(FIRST_GROUP), inputEncryptor, outputEncryptor); + final String outputLine = matcher.replaceFirst(outputEncrypted); + writer.println(outputLine); + } else { + writer.println(line); + } + }); + } + } catch (final IOException e) { + throw new UncheckedIOException("Failed Processing Flow Configuration", e); + } + } + + private String getOutputEncrypted(final String inputEncrypted, final PropertyEncryptor inputEncryptor, final PropertyEncryptor outputEncryptor) { + final String inputDecrypted = inputEncryptor.decrypt(inputEncrypted); + final String outputEncrypted = outputEncryptor.encrypt(inputDecrypted); + return String.format(ENCRYPTED_FORMAT, outputEncrypted); + } +} diff --git a/nifi-commons/nifi-flow-encryptor/src/main/java/org/apache/nifi/flow/encryptor/command/SetSensitivePropertiesKey.java b/nifi-commons/nifi-flow-encryptor/src/main/java/org/apache/nifi/flow/encryptor/command/SetSensitivePropertiesKey.java new file mode 100644 index 0000000000..7f61aae45d --- /dev/null +++ b/nifi-commons/nifi-flow-encryptor/src/main/java/org/apache/nifi/flow/encryptor/command/SetSensitivePropertiesKey.java @@ -0,0 +1,177 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.flow.encryptor.command; + +import org.apache.nifi.encrypt.PropertyEncryptor; +import org.apache.nifi.encrypt.PropertyEncryptorBuilder; +import org.apache.nifi.flow.encryptor.FlowEncryptor; +import org.apache.nifi.flow.encryptor.StandardFlowEncryptor; +import org.apache.nifi.security.util.EncryptionMethod; + +import java.io.File; +import java.io.FileInputStream; +import java.io.FileOutputStream; +import java.io.FileReader; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.io.UncheckedIOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.StandardCopyOption; +import java.util.List; +import java.util.Properties; +import java.util.stream.Collectors; +import java.util.zip.GZIPInputStream; +import java.util.zip.GZIPOutputStream; + +/** + * Set Sensitive Properties Key for NiFi Properties and update encrypted Flow Configuration + */ +public class SetSensitivePropertiesKey { + protected static final String PROPERTIES_FILE_PATH = "nifi.properties.file.path"; + + protected static final String PROPS_KEY = "nifi.sensitive.props.key"; + + protected static final String PROPS_ALGORITHM = "nifi.sensitive.props.algorithm"; + + protected static final String CONFIGURATION_FILE = "nifi.flow.configuration.file"; + + private static final int MINIMUM_REQUIRED_LENGTH = 12; + + private static final String FLOW_XML_PREFIX = "flow.xml."; + + private static final String GZ_EXTENSION = ".gz"; + + private static final String DEFAULT_PROPERTIES_ALGORITHM = EncryptionMethod.MD5_256AES.getAlgorithm(); + + private static final String DEFAULT_PROPERTIES_KEY = "nififtw!"; + + private static final String SENSITIVE_PROPERTIES_KEY = String.format("%s=", PROPS_KEY); + + public static void main(final String[] arguments) { + if (arguments.length == 1) { + final String outputPropertiesKey = arguments[0]; + if (outputPropertiesKey.length() < MINIMUM_REQUIRED_LENGTH) { + System.err.printf("Sensitive Properties Key length less than required [%d]%n", MINIMUM_REQUIRED_LENGTH); + } else { + run(outputPropertiesKey); + } + } else { + System.err.printf("Unexpected number of arguments [%d]%n", arguments.length); + System.err.printf("Usage: %s %n", SetSensitivePropertiesKey.class.getSimpleName()); + } + } + + private static void run(final String outputPropertiesKey) { + final String propertiesFilePath = System.getProperty(PROPERTIES_FILE_PATH); + final File propertiesFile = new File(propertiesFilePath); + final Properties properties = loadProperties(propertiesFile); + + final File flowConfigurationFile = getFlowConfigurationFile(properties); + try { + storeProperties(propertiesFile, outputPropertiesKey); + System.out.printf("NiFi Properties Processed [%s]%n", propertiesFilePath); + } catch (final IOException e) { + final String message = String.format("Failed to Process NiFi Properties [%s]", propertiesFilePath); + throw new UncheckedIOException(message, e); + } + + if (flowConfigurationFile.exists()) { + final String algorithm = getAlgorithm(properties); + final PropertyEncryptor outputEncryptor = getPropertyEncryptor(outputPropertiesKey, algorithm); + processFlowConfiguration(properties, outputEncryptor); + } + } + + private static void processFlowConfiguration(final Properties properties, final PropertyEncryptor outputEncryptor) { + final File flowConfigurationFile = getFlowConfigurationFile(properties); + try (final InputStream flowInputStream = new GZIPInputStream(new FileInputStream(flowConfigurationFile))) { + final File flowOutputFile = getFlowOutputFile(); + final Path flowOutputPath = flowOutputFile.toPath(); + try (final OutputStream flowOutputStream = new GZIPOutputStream(new FileOutputStream(flowOutputFile))) { + final String inputAlgorithm = getAlgorithm(properties); + final String inputPropertiesKey = getKey(properties); + final PropertyEncryptor inputEncryptor = getPropertyEncryptor(inputPropertiesKey, inputAlgorithm); + + final FlowEncryptor flowEncryptor = new StandardFlowEncryptor(); + flowEncryptor.processFlow(flowInputStream, flowOutputStream, inputEncryptor, outputEncryptor); + } + + final Path flowConfigurationPath = flowConfigurationFile.toPath(); + Files.move(flowOutputPath, flowConfigurationPath, StandardCopyOption.REPLACE_EXISTING); + System.out.printf("Flow Configuration Processed [%s]%n", flowConfigurationPath); + } catch (final IOException|RuntimeException e) { + System.err.printf("Failed to process Flow Configuration [%s]%n", flowConfigurationFile); + e.printStackTrace(); + } + } + + private static String getAlgorithm(final Properties properties) { + String algorithm = properties.getProperty(PROPS_ALGORITHM, DEFAULT_PROPERTIES_ALGORITHM); + if (algorithm.length() == 0) { + algorithm = DEFAULT_PROPERTIES_ALGORITHM; + } + return algorithm; + } + + private static String getKey(final Properties properties) { + String key = properties.getProperty(PROPS_KEY, DEFAULT_PROPERTIES_KEY); + if (key.length() == 0) { + key = DEFAULT_PROPERTIES_KEY; + } + return key; + } + + private static File getFlowOutputFile() throws IOException { + final File flowOutputFile = File.createTempFile(FLOW_XML_PREFIX, GZ_EXTENSION); + flowOutputFile.deleteOnExit(); + return flowOutputFile; + } + + private static Properties loadProperties(final File propertiesFile) { + final Properties properties = new Properties(); + try (final FileReader reader = new FileReader(propertiesFile)) { + properties.load(reader); + } catch (final IOException e) { + final String message = String.format("Failed to read NiFi Properties [%s]", propertiesFile); + throw new UncheckedIOException(message, e); + } + return properties; + } + + private static void storeProperties(final File propertiesFile, final String propertiesKey) throws IOException { + final Path propertiesFilePath = propertiesFile.toPath(); + final List lines = Files.readAllLines(propertiesFilePath); + final List updatedLines = lines.stream().map(line -> { + if (line.startsWith(SENSITIVE_PROPERTIES_KEY)) { + return SENSITIVE_PROPERTIES_KEY + propertiesKey; + } else { + return line; + } + }).collect(Collectors.toList()); + Files.write(propertiesFilePath, updatedLines); + } + + private static PropertyEncryptor getPropertyEncryptor(final String propertiesKey, final String propertiesAlgorithm) { + return new PropertyEncryptorBuilder(propertiesKey).setAlgorithm(propertiesAlgorithm).build(); + } + + private static File getFlowConfigurationFile(final Properties properties) { + return new File(properties.getProperty(CONFIGURATION_FILE)); + } +} diff --git a/nifi-commons/nifi-flow-encryptor/src/test/java/org/apache/nifi/flow/encryptor/StandardFlowEncryptorTest.java b/nifi-commons/nifi-flow-encryptor/src/test/java/org/apache/nifi/flow/encryptor/StandardFlowEncryptorTest.java new file mode 100644 index 0000000000..d9cfd6a411 --- /dev/null +++ b/nifi-commons/nifi-flow-encryptor/src/test/java/org/apache/nifi/flow/encryptor/StandardFlowEncryptorTest.java @@ -0,0 +1,95 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.flow.encryptor; + +import org.apache.nifi.encrypt.PropertyEncryptor; +import org.apache.nifi.encrypt.PropertyEncryptorBuilder; +import org.apache.nifi.security.util.EncryptionMethod; +import org.junit.Before; +import org.junit.Test; + +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.InputStream; +import java.nio.charset.StandardCharsets; +import java.util.UUID; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +public class StandardFlowEncryptorTest { + + private static final String INPUT_KEY = UUID.randomUUID().toString(); + + private static final String OUTPUT_KEY = UUID.randomUUID().toString(); + + private static final String ENCRYPTED_FORMAT = "enc{%s}"; + + private static final Pattern OUTPUT_PATTERN = Pattern.compile("^enc\\{([^}]+?)}$"); + + private PropertyEncryptor inputEncryptor; + + private PropertyEncryptor outputEncryptor; + + private StandardFlowEncryptor flowEncryptor; + + @Before + public void setEncryptors() { + inputEncryptor = getPropertyEncryptor(INPUT_KEY, EncryptionMethod.MD5_256AES.getAlgorithm()); + outputEncryptor = getPropertyEncryptor(OUTPUT_KEY, EncryptionMethod.SHA256_256AES.getAlgorithm()); + flowEncryptor = new StandardFlowEncryptor(); + } + + @Test + public void testProcessEncrypted() { + final String property = StandardFlowEncryptorTest.class.getSimpleName(); + final String encryptedProperty = String.format(ENCRYPTED_FORMAT, inputEncryptor.encrypt(property)); + final String encryptedRow = String.format("%s%n", encryptedProperty); + + final InputStream inputStream = new ByteArrayInputStream(encryptedRow.getBytes(StandardCharsets.UTF_8)); + final ByteArrayOutputStream outputStream = new ByteArrayOutputStream(); + + flowEncryptor.processFlow(inputStream, outputStream, inputEncryptor, outputEncryptor); + + final String outputEncrypted = new String(outputStream.toByteArray()); + final Matcher matcher = OUTPUT_PATTERN.matcher(outputEncrypted); + assertTrue(String.format("Encrypted Pattern not found [%s]", outputEncrypted), matcher.find()); + + final String outputEncryptedProperty = matcher.group(1); + final String outputDecrypted = outputEncryptor.decrypt(outputEncryptedProperty); + assertEquals(property, outputDecrypted); + } + + @Test + public void testProcessNoEncrypted() { + final String property = String.format("%s%n", StandardFlowEncryptorTest.class.getSimpleName()); + + final InputStream inputStream = new ByteArrayInputStream(property.getBytes(StandardCharsets.UTF_8)); + final ByteArrayOutputStream outputStream = new ByteArrayOutputStream(); + + flowEncryptor.processFlow(inputStream, outputStream, inputEncryptor, outputEncryptor); + + final String outputProperty = new String(outputStream.toByteArray()); + assertEquals(property, outputProperty); + } + + private PropertyEncryptor getPropertyEncryptor(final String propertiesKey, final String propertiesAlgorithm) { + return new PropertyEncryptorBuilder(propertiesKey).setAlgorithm(propertiesAlgorithm).build(); + } +} diff --git a/nifi-commons/nifi-flow-encryptor/src/test/java/org/apache/nifi/flow/encryptor/command/SetSensitivePropertiesKeyTest.java b/nifi-commons/nifi-flow-encryptor/src/test/java/org/apache/nifi/flow/encryptor/command/SetSensitivePropertiesKeyTest.java new file mode 100644 index 0000000000..13f2837dd2 --- /dev/null +++ b/nifi-commons/nifi-flow-encryptor/src/test/java/org/apache/nifi/flow/encryptor/command/SetSensitivePropertiesKeyTest.java @@ -0,0 +1,118 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.flow.encryptor.command; + +import org.apache.nifi.stream.io.GZIPOutputStream; +import org.junit.After; +import org.junit.Test; + +import java.io.File; +import java.io.FileOutputStream; +import java.io.IOException; +import java.net.URISyntaxException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.List; +import java.util.Optional; +import java.util.UUID; +import java.util.stream.Collectors; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +public class SetSensitivePropertiesKeyTest { + private static final String FLOW_CONTENTS = "PROPERTY"; + + @After + public void clearProperties() { + System.clearProperty(SetSensitivePropertiesKey.PROPERTIES_FILE_PATH); + } + + @Test + public void testMainNoArguments() { + SetSensitivePropertiesKey.main(new String[]{}); + } + + @Test + public void testMainBlankKeyAndAlgorithm() throws IOException, URISyntaxException { + final Path flowConfiguration = getFlowConfiguration(); + final Path propertiesPath = getNiFiProperties(flowConfiguration, "/blank.nifi.properties"); + + System.setProperty(SetSensitivePropertiesKey.PROPERTIES_FILE_PATH, propertiesPath.toString()); + + final String sensitivePropertiesKey = UUID.randomUUID().toString(); + SetSensitivePropertiesKey.main(new String[]{sensitivePropertiesKey}); + + assertPropertiesKeyUpdated(propertiesPath, sensitivePropertiesKey); + assertTrue("Flow Configuration not found", flowConfiguration.toFile().exists()); + } + + @Test + public void testMainPopulatedKeyAndAlgorithm() throws IOException, URISyntaxException { + final Path flowConfiguration = getFlowConfiguration(); + final Path propertiesPath = getNiFiProperties(flowConfiguration, "/populated.nifi.properties"); + + System.setProperty(SetSensitivePropertiesKey.PROPERTIES_FILE_PATH, propertiesPath.toString()); + + final String sensitivePropertiesKey = UUID.randomUUID().toString(); + SetSensitivePropertiesKey.main(new String[]{sensitivePropertiesKey}); + + assertPropertiesKeyUpdated(propertiesPath, sensitivePropertiesKey); + assertTrue("Flow Configuration not found", flowConfiguration.toFile().exists()); + } + + private void assertPropertiesKeyUpdated(final Path propertiesPath, final String sensitivePropertiesKey) throws IOException { + final Optional keyProperty = Files.readAllLines(propertiesPath) + .stream() + .filter(line -> line.startsWith(SetSensitivePropertiesKey.PROPS_KEY)) + .findFirst(); + assertTrue("Sensitive Key Property not found", keyProperty.isPresent()); + + final String expectedProperty = String.format("%s=%s", SetSensitivePropertiesKey.PROPS_KEY, sensitivePropertiesKey); + assertEquals("Sensitive Key Property not updated", expectedProperty, keyProperty.get()); + } + + private Path getNiFiProperties(final Path flowConfigurationPath, String propertiesResource) throws IOException, URISyntaxException { + final Path sourcePropertiesPath = Paths.get(SetSensitivePropertiesKey.class.getResource(propertiesResource).toURI()); + final List sourceProperties = Files.readAllLines(sourcePropertiesPath); + final List flowProperties = sourceProperties.stream().map(line -> { + if (line.startsWith(SetSensitivePropertiesKey.CONFIGURATION_FILE)) { + return line + flowConfigurationPath.toString(); + } else { + return line; + } + }).collect(Collectors.toList()); + + final Path propertiesPath = Files.createTempFile(SetSensitivePropertiesKey.class.getSimpleName(), ".properties"); + propertiesPath.toFile().deleteOnExit(); + Files.write(propertiesPath, flowProperties); + return propertiesPath; + } + + private Path getFlowConfiguration() throws IOException { + final Path flowConfigurationPath = Files.createTempFile(SetSensitivePropertiesKey.class.getSimpleName(), ".xml.gz"); + final File flowConfigurationFile = flowConfigurationPath.toFile(); + flowConfigurationFile.deleteOnExit(); + + try (final GZIPOutputStream outputStream = new GZIPOutputStream(new FileOutputStream(flowConfigurationFile))) { + outputStream.write(FLOW_CONTENTS.getBytes(StandardCharsets.UTF_8)); + } + return flowConfigurationPath; + } +} diff --git a/nifi-commons/nifi-flow-encryptor/src/test/resources/blank.nifi.properties b/nifi-commons/nifi-flow-encryptor/src/test/resources/blank.nifi.properties new file mode 100644 index 0000000000..8c16c51f95 --- /dev/null +++ b/nifi-commons/nifi-flow-encryptor/src/test/resources/blank.nifi.properties @@ -0,0 +1,17 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +nifi.sensitive.props.key= +nifi.sensitive.props.algorithm= +nifi.flow.configuration.file= diff --git a/nifi-commons/nifi-flow-encryptor/src/test/resources/populated.nifi.properties b/nifi-commons/nifi-flow-encryptor/src/test/resources/populated.nifi.properties new file mode 100644 index 0000000000..36e2707c71 --- /dev/null +++ b/nifi-commons/nifi-flow-encryptor/src/test/resources/populated.nifi.properties @@ -0,0 +1,17 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +nifi.sensitive.props.key=D5E41AC1-EEF8-4A54-930D-593F749AE95C +nifi.sensitive.props.algorithm=NIFI_ARGON2_AES_GCM_256 +nifi.flow.configuration.file= diff --git a/nifi-commons/nifi-property-encryptor/pom.xml b/nifi-commons/nifi-property-encryptor/pom.xml new file mode 100644 index 0000000000..92cea865fb --- /dev/null +++ b/nifi-commons/nifi-property-encryptor/pom.xml @@ -0,0 +1,45 @@ + + + + 4.0.0 + + org.apache.nifi + nifi-commons + 1.14.0-SNAPSHOT + + nifi-property-encryptor + + + org.apache.nifi + nifi-security-utils + 1.14.0-SNAPSHOT + + + org.bouncycastle + bcprov-jdk15on + + + org.apache.commons + commons-lang3 + 3.11 + + + commons-codec + commons-codec + 1.14 + + + diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/main/java/org/apache/nifi/encrypt/CipherPropertyEncryptor.java b/nifi-commons/nifi-property-encryptor/src/main/java/org/apache/nifi/encrypt/CipherPropertyEncryptor.java similarity index 100% rename from nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/main/java/org/apache/nifi/encrypt/CipherPropertyEncryptor.java rename to nifi-commons/nifi-property-encryptor/src/main/java/org/apache/nifi/encrypt/CipherPropertyEncryptor.java diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/main/java/org/apache/nifi/encrypt/EncryptionException.java b/nifi-commons/nifi-property-encryptor/src/main/java/org/apache/nifi/encrypt/EncryptionException.java similarity index 100% rename from nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/main/java/org/apache/nifi/encrypt/EncryptionException.java rename to nifi-commons/nifi-property-encryptor/src/main/java/org/apache/nifi/encrypt/EncryptionException.java diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/main/java/org/apache/nifi/encrypt/KeyedCipherPropertyEncryptor.java b/nifi-commons/nifi-property-encryptor/src/main/java/org/apache/nifi/encrypt/KeyedCipherPropertyEncryptor.java similarity index 100% rename from nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/main/java/org/apache/nifi/encrypt/KeyedCipherPropertyEncryptor.java rename to nifi-commons/nifi-property-encryptor/src/main/java/org/apache/nifi/encrypt/KeyedCipherPropertyEncryptor.java diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/main/java/org/apache/nifi/encrypt/PasswordBasedCipherPropertyEncryptor.java b/nifi-commons/nifi-property-encryptor/src/main/java/org/apache/nifi/encrypt/PasswordBasedCipherPropertyEncryptor.java similarity index 100% rename from nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/main/java/org/apache/nifi/encrypt/PasswordBasedCipherPropertyEncryptor.java rename to nifi-commons/nifi-property-encryptor/src/main/java/org/apache/nifi/encrypt/PasswordBasedCipherPropertyEncryptor.java diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/main/java/org/apache/nifi/encrypt/PropertyEncryptionMethod.java b/nifi-commons/nifi-property-encryptor/src/main/java/org/apache/nifi/encrypt/PropertyEncryptionMethod.java similarity index 100% rename from nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/main/java/org/apache/nifi/encrypt/PropertyEncryptionMethod.java rename to nifi-commons/nifi-property-encryptor/src/main/java/org/apache/nifi/encrypt/PropertyEncryptionMethod.java diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/main/java/org/apache/nifi/encrypt/PropertyEncryptor.java b/nifi-commons/nifi-property-encryptor/src/main/java/org/apache/nifi/encrypt/PropertyEncryptor.java similarity index 100% rename from nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/main/java/org/apache/nifi/encrypt/PropertyEncryptor.java rename to nifi-commons/nifi-property-encryptor/src/main/java/org/apache/nifi/encrypt/PropertyEncryptor.java diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/main/java/org/apache/nifi/encrypt/PropertyEncryptorFactory.java b/nifi-commons/nifi-property-encryptor/src/main/java/org/apache/nifi/encrypt/PropertyEncryptorBuilder.java similarity index 50% rename from nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/main/java/org/apache/nifi/encrypt/PropertyEncryptorFactory.java rename to nifi-commons/nifi-property-encryptor/src/main/java/org/apache/nifi/encrypt/PropertyEncryptorBuilder.java index a8a222a063..fcb4a89888 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/main/java/org/apache/nifi/encrypt/PropertyEncryptorFactory.java +++ b/nifi-commons/nifi-property-encryptor/src/main/java/org/apache/nifi/encrypt/PropertyEncryptorBuilder.java @@ -16,70 +16,55 @@ */ package org.apache.nifi.encrypt; -import org.apache.commons.lang3.StringUtils; import org.apache.nifi.security.util.EncryptionMethod; import org.apache.nifi.security.util.crypto.AESKeyedCipherProvider; import org.apache.nifi.security.util.crypto.KeyedCipherProvider; import org.apache.nifi.security.util.crypto.PBECipherProvider; -import org.apache.nifi.util.NiFiProperties; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import javax.crypto.SecretKey; import java.util.Objects; /** - * Property Encryptor Factory for encapsulating instantiation of Property Encryptors based on various parameters + * Property Encryptor Builder */ -public class PropertyEncryptorFactory { - private static final Logger LOGGER = LoggerFactory.getLogger(PropertyEncryptorFactory.class); - +public class PropertyEncryptorBuilder { private static final PropertySecretKeyProvider SECRET_KEY_PROVIDER = new StandardPropertySecretKeyProvider(); - private static final String DEFAULT_PASSWORD = "nififtw!"; + private final String password; - private static final String NOTIFICATION_BORDER = "*"; - - private static final int NOTIFICATION_WIDTH = 80; - - private static final String NOTIFICATION_DELIMITER = StringUtils.repeat(NOTIFICATION_BORDER, NOTIFICATION_WIDTH); - - private static final String NOTIFICATION = StringUtils.joinWith(System.lineSeparator(), - System.lineSeparator(), - NOTIFICATION_DELIMITER, - StringUtils.center(String.format("FOUND BLANK SENSITIVE PROPERTIES KEY [%s]", NiFiProperties.SENSITIVE_PROPS_KEY), NOTIFICATION_WIDTH), - StringUtils.center("USING DEFAULT KEY FOR ENCRYPTION", NOTIFICATION_WIDTH), - StringUtils.center(String.format("SET [%s] TO SECURE SENSITIVE PROPERTIES", NiFiProperties.SENSITIVE_PROPS_KEY), NOTIFICATION_WIDTH), - NOTIFICATION_DELIMITER - ); + private String algorithm = PropertyEncryptionMethod.NIFI_ARGON2_AES_GCM_256.toString(); /** - * Get Property Encryptor using NiFi Properties + * Property Encryptor Builder with required password + * + * @param password Password required + */ + public PropertyEncryptorBuilder(final String password) { + Objects.requireNonNull(password, "Password required"); + this.password = password; + } + + /** + * Set Algorithm as either Property Encryption Method or Encryption Method + * + * @param algorithm Algorithm + * @return Property Encryptor Builder + */ + public PropertyEncryptorBuilder setAlgorithm(final String algorithm) { + Objects.requireNonNull(algorithm, "Algorithm required"); + this.algorithm = algorithm; + return this; + } + + /** + * Build Property Encryptor using current configuration * - * @param properties NiFi Properties * @return Property Encryptor */ - @SuppressWarnings("deprecation") - public static PropertyEncryptor getPropertyEncryptor(final NiFiProperties properties) { - Objects.requireNonNull(properties, "NiFi Properties is required"); - final String algorithm = properties.getProperty(NiFiProperties.SENSITIVE_PROPS_ALGORITHM); - String password = properties.getProperty(NiFiProperties.SENSITIVE_PROPS_KEY); - - if (StringUtils.isBlank(password)) { - LOGGER.error(NOTIFICATION); - password = DEFAULT_PASSWORD; - } - + public PropertyEncryptor build() { final PropertyEncryptionMethod propertyEncryptionMethod = findPropertyEncryptionAlgorithm(algorithm); if (propertyEncryptionMethod == null) { - final EncryptionMethod encryptionMethod = findEncryptionMethod(algorithm); - if (encryptionMethod.isPBECipher()) { - final PBECipherProvider cipherProvider = new org.apache.nifi.security.util.crypto.NiFiLegacyCipherProvider(); - return new PasswordBasedCipherPropertyEncryptor(cipherProvider, encryptionMethod, password); - } else { - final String message = String.format("Algorithm [%s] not supported for Sensitive Properties", encryptionMethod.getAlgorithm()); - throw new UnsupportedOperationException(message); - } + return getPasswordBasedCipherPropertyEncryptor(); } else { final KeyedCipherProvider keyedCipherProvider = new AESKeyedCipherProvider(); final SecretKey secretKey = SECRET_KEY_PROVIDER.getSecretKey(propertyEncryptionMethod, password); @@ -88,7 +73,19 @@ public class PropertyEncryptorFactory { } } - private static PropertyEncryptionMethod findPropertyEncryptionAlgorithm(final String algorithm) { + @SuppressWarnings("deprecation") + private PasswordBasedCipherPropertyEncryptor getPasswordBasedCipherPropertyEncryptor() { + final EncryptionMethod encryptionMethod = findEncryptionMethod(algorithm); + if (encryptionMethod.isPBECipher()) { + final PBECipherProvider cipherProvider = new org.apache.nifi.security.util.crypto.NiFiLegacyCipherProvider(); + return new PasswordBasedCipherPropertyEncryptor(cipherProvider, encryptionMethod, password); + } else { + final String message = String.format("Algorithm [%s] not supported for Sensitive Properties", encryptionMethod.getAlgorithm()); + throw new UnsupportedOperationException(message); + } + } + + private PropertyEncryptionMethod findPropertyEncryptionAlgorithm(final String algorithm) { PropertyEncryptionMethod foundPropertyEncryptionMethod = null; for (final PropertyEncryptionMethod propertyEncryptionMethod : PropertyEncryptionMethod.values()) { @@ -101,7 +98,7 @@ public class PropertyEncryptorFactory { return foundPropertyEncryptionMethod; } - private static EncryptionMethod findEncryptionMethod(final String algorithm) { + private EncryptionMethod findEncryptionMethod(final String algorithm) { final EncryptionMethod encryptionMethod = EncryptionMethod.forAlgorithm(algorithm); if (encryptionMethod == null) { final String message = String.format("Encryption Method not found for Algorithm [%s]", algorithm); diff --git a/nifi-commons/nifi-property-encryptor/src/main/java/org/apache/nifi/encrypt/PropertyEncryptorFactory.java b/nifi-commons/nifi-property-encryptor/src/main/java/org/apache/nifi/encrypt/PropertyEncryptorFactory.java new file mode 100644 index 0000000000..ac95ad3558 --- /dev/null +++ b/nifi-commons/nifi-property-encryptor/src/main/java/org/apache/nifi/encrypt/PropertyEncryptorFactory.java @@ -0,0 +1,47 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.encrypt; + +import org.apache.commons.lang3.StringUtils; +import org.apache.nifi.util.NiFiProperties; + +import java.util.Objects; + +/** + * Property Encryptor Factory for encapsulating instantiation of Property Encryptors based on various parameters + */ +public class PropertyEncryptorFactory { + private static final String KEY_REQUIRED = String.format("NiFi Sensitive Properties Key [%s] is required", NiFiProperties.SENSITIVE_PROPS_KEY); + + /** + * Get Property Encryptor using NiFi Properties + * + * @param properties NiFi Properties + * @return Property Encryptor + */ + public static PropertyEncryptor getPropertyEncryptor(final NiFiProperties properties) { + Objects.requireNonNull(properties, "NiFi Properties is required"); + final String algorithm = properties.getProperty(NiFiProperties.SENSITIVE_PROPS_ALGORITHM); + String password = properties.getProperty(NiFiProperties.SENSITIVE_PROPS_KEY); + + if (StringUtils.isBlank(password)) { + throw new IllegalArgumentException(KEY_REQUIRED); + } + + return new PropertyEncryptorBuilder(password).setAlgorithm(algorithm).build(); + } +} diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/main/java/org/apache/nifi/encrypt/PropertySecretKeyProvider.java b/nifi-commons/nifi-property-encryptor/src/main/java/org/apache/nifi/encrypt/PropertySecretKeyProvider.java similarity index 100% rename from nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/main/java/org/apache/nifi/encrypt/PropertySecretKeyProvider.java rename to nifi-commons/nifi-property-encryptor/src/main/java/org/apache/nifi/encrypt/PropertySecretKeyProvider.java diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/main/java/org/apache/nifi/encrypt/StandardPropertySecretKeyProvider.java b/nifi-commons/nifi-property-encryptor/src/main/java/org/apache/nifi/encrypt/StandardPropertySecretKeyProvider.java similarity index 100% rename from nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/main/java/org/apache/nifi/encrypt/StandardPropertySecretKeyProvider.java rename to nifi-commons/nifi-property-encryptor/src/main/java/org/apache/nifi/encrypt/StandardPropertySecretKeyProvider.java diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/test/java/org/apache/nifi/encrypt/KeyedCipherPropertyEncryptorTest.java b/nifi-commons/nifi-property-encryptor/src/test/java/org/apache/nifi/encrypt/KeyedCipherPropertyEncryptorTest.java similarity index 100% rename from nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/test/java/org/apache/nifi/encrypt/KeyedCipherPropertyEncryptorTest.java rename to nifi-commons/nifi-property-encryptor/src/test/java/org/apache/nifi/encrypt/KeyedCipherPropertyEncryptorTest.java diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/test/java/org/apache/nifi/encrypt/PasswordBasedCipherPropertyEncryptorTest.java b/nifi-commons/nifi-property-encryptor/src/test/java/org/apache/nifi/encrypt/PasswordBasedCipherPropertyEncryptorTest.java similarity index 100% rename from nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/test/java/org/apache/nifi/encrypt/PasswordBasedCipherPropertyEncryptorTest.java rename to nifi-commons/nifi-property-encryptor/src/test/java/org/apache/nifi/encrypt/PasswordBasedCipherPropertyEncryptorTest.java diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/test/java/org/apache/nifi/encrypt/PropertyEncryptorFactoryTest.java b/nifi-commons/nifi-property-encryptor/src/test/java/org/apache/nifi/encrypt/PropertyEncryptorFactoryTest.java similarity index 95% rename from nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/test/java/org/apache/nifi/encrypt/PropertyEncryptorFactoryTest.java rename to nifi-commons/nifi-property-encryptor/src/test/java/org/apache/nifi/encrypt/PropertyEncryptorFactoryTest.java index 55fe2e5027..c20835d9c1 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/test/java/org/apache/nifi/encrypt/PropertyEncryptorFactoryTest.java +++ b/nifi-commons/nifi-property-encryptor/src/test/java/org/apache/nifi/encrypt/PropertyEncryptorFactoryTest.java @@ -49,8 +49,7 @@ public class PropertyEncryptorFactoryTest { properties.setProperty(NiFiProperties.SENSITIVE_PROPS_KEY, StringUtils.EMPTY); final NiFiProperties niFiProperties = NiFiProperties.createBasicNiFiProperties(null, properties); - final PropertyEncryptor encryptor = PropertyEncryptorFactory.getPropertyEncryptor(niFiProperties); - assertNotNull(encryptor); + assertThrows(IllegalArgumentException.class, () -> PropertyEncryptorFactory.getPropertyEncryptor(niFiProperties)); } @Test diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/test/java/org/apache/nifi/encrypt/StandardPropertySecretKeyProviderTest.java b/nifi-commons/nifi-property-encryptor/src/test/java/org/apache/nifi/encrypt/StandardPropertySecretKeyProviderTest.java similarity index 100% rename from nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/test/java/org/apache/nifi/encrypt/StandardPropertySecretKeyProviderTest.java rename to nifi-commons/nifi-property-encryptor/src/test/java/org/apache/nifi/encrypt/StandardPropertySecretKeyProviderTest.java diff --git a/nifi-commons/pom.xml b/nifi-commons/pom.xml index a1fc3d27fa..c66973e854 100644 --- a/nifi-commons/pom.xml +++ b/nifi-commons/pom.xml @@ -27,11 +27,13 @@ nifi-data-provenance-utils nifi-expression-language nifi-flowfile-packager + nifi-flow-encryptor nifi-hl7-query-language nifi-json-utils nifi-logging-utils nifi-metrics nifi-parameter + nifi-property-encryptor nifi-properties nifi-record nifi-record-path diff --git a/nifi-docs/src/main/asciidoc/administration-guide.adoc b/nifi-docs/src/main/asciidoc/administration-guide.adoc index 2c306bf698..b9b4499dd2 100644 --- a/nifi-docs/src/main/asciidoc/administration-guide.adoc +++ b/nifi-docs/src/main/asciidoc/administration-guide.adoc @@ -3926,6 +3926,18 @@ where: For more information see the <> section in the NiFi Toolkit Guide. +==== Updating the Sensitive Properties Key + +Starting with version 1.14.0, NiFi requires a value for 'nifi.sensitive.props.key' in _nifi.properties_. + +The following command can be used to read an existing _flow.xml.gz_ configuration and set a new sensitive properties key in _nifi.properties_: + +``` +$ ./bin/nifi.sh set-sensitive-properties-key +``` + +The minimum required length for a new sensitive properties key is 12 characters. + === Start New NiFi In your new NiFi installation: diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-components/pom.xml b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-components/pom.xml index db97374f4e..e3aee63fe4 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-components/pom.xml +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-components/pom.xml @@ -66,6 +66,11 @@ nifi-nar-utils 1.14.0-SNAPSHOT + + org.apache.nifi + nifi-property-encryptor + 1.14.0-SNAPSHOT + org.apache.nifi.registry diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/groovy/org/apache/nifi/controller/serialization/FlowFromDOMFactoryTest.groovy b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/groovy/org/apache/nifi/controller/serialization/FlowFromDOMFactoryTest.groovy index 74ae05275f..200f368f1e 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/groovy/org/apache/nifi/controller/serialization/FlowFromDOMFactoryTest.groovy +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/groovy/org/apache/nifi/controller/serialization/FlowFromDOMFactoryTest.groovy @@ -16,13 +16,8 @@ */ package org.apache.nifi.controller.serialization -import org.apache.commons.codec.binary.Hex import org.apache.nifi.encrypt.EncryptionException import org.apache.nifi.encrypt.PropertyEncryptor -import org.apache.nifi.encrypt.PropertyEncryptorFactory -import org.apache.nifi.security.kms.CryptoUtils -import org.apache.nifi.security.util.EncryptionMethod -import org.bouncycastle.jce.provider.BouncyCastleProvider import org.junit.BeforeClass import org.junit.Test import org.junit.runner.RunWith @@ -30,27 +25,14 @@ import org.junit.runners.JUnit4 import org.slf4j.Logger import org.slf4j.LoggerFactory -import javax.crypto.Cipher -import javax.crypto.SecretKey -import javax.crypto.SecretKeyFactory -import javax.crypto.spec.PBEKeySpec -import javax.crypto.spec.PBEParameterSpec -import java.security.Security - import static groovy.test.GroovyAssert.shouldFail @RunWith(JUnit4.class) class FlowFromDOMFactoryTest { private static final Logger logger = LoggerFactory.getLogger(FlowFromDOMFactoryTest.class) - private static final String DEFAULT_PASSWORD = "nififtw!" - private static final byte[] DEFAULT_SALT = new byte[8] - private static final int DEFAULT_ITERATION_COUNT = 0 - private static final EncryptionMethod DEFAULT_ENCRYPTION_METHOD = EncryptionMethod.MD5_128AES - @BeforeClass static void setUpOnce() throws Exception { - Security.addProvider(new BouncyCastleProvider()) logger.metaClass.methodMissing = { String name, args -> logger.info("[${name?.toUpperCase()}] ${(args as List).join(" ")}") } diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-properties-loader/src/main/java/org/apache/nifi/properties/NiFiPropertiesLoader.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-properties-loader/src/main/java/org/apache/nifi/properties/NiFiPropertiesLoader.java index 7b01ec3a55..250227f9d0 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-properties-loader/src/main/java/org/apache/nifi/properties/NiFiPropertiesLoader.java +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-properties-loader/src/main/java/org/apache/nifi/properties/NiFiPropertiesLoader.java @@ -21,12 +21,19 @@ import java.io.File; import java.io.FileInputStream; import java.io.IOException; import java.io.InputStream; +import java.io.UncheckedIOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; import java.security.NoSuchAlgorithmException; +import java.security.SecureRandom; import java.security.Security; +import java.util.Base64; +import java.util.List; import java.util.Properties; +import java.util.stream.Collectors; import java.util.Set; import javax.crypto.Cipher; -import org.apache.commons.lang3.StringUtils; import org.apache.nifi.security.kms.CryptoUtils; import org.apache.nifi.util.NiFiProperties; import org.bouncycastle.jce.provider.BouncyCastleProvider; @@ -36,7 +43,13 @@ import org.slf4j.LoggerFactory; public class NiFiPropertiesLoader { private static final Logger logger = LoggerFactory.getLogger(NiFiPropertiesLoader.class); + private static final Base64.Encoder KEY_ENCODER = Base64.getEncoder().withoutPadding(); + private static final int SENSITIVE_PROPERTIES_KEY_LENGTH = 24; + private static final String EMPTY_SENSITIVE_PROPERTIES_KEY = String.format("%s=", NiFiProperties.SENSITIVE_PROPS_KEY); + private static final String MIGRATION_INSTRUCTIONS = "See Admin Guide section [Updating the Sensitive Properties Key]"; + private static final String PROPERTIES_KEY_MESSAGE = String.format("Sensitive Properties Key [%s] not found: %s", NiFiProperties.SENSITIVE_PROPS_KEY, MIGRATION_INSTRUCTIONS); + private final String defaultPropertiesFilePath = CryptoUtils.getDefaultFilePath(); private NiFiProperties instance; private String keyHex; @@ -132,7 +145,7 @@ public class NiFiPropertiesLoader { } private NiFiProperties loadDefault() { - return load(CryptoUtils.getDefaultFilePath()); + return load(defaultPropertiesFilePath); } static String getDefaultProviderKey() { @@ -168,36 +181,22 @@ public class NiFiPropertiesLoader { throw new IllegalArgumentException("NiFi properties file missing or unreadable"); } - Properties rawProperties = new Properties(); - - InputStream inStream = null; - try { - inStream = new BufferedInputStream(new FileInputStream(file)); - rawProperties.load(inStream); + final Properties rawProperties = new Properties(); + try (final InputStream inputStream = new BufferedInputStream(new FileInputStream(file))) { + rawProperties.load(inputStream); logger.info("Loaded {} properties from {}", rawProperties.size(), file.getAbsolutePath()); - Set keys = rawProperties.stringPropertyNames(); + final Set keys = rawProperties.stringPropertyNames(); for (final String key : keys) { - String prop = rawProperties.getProperty(key); - rawProperties.setProperty(key, StringUtils.stripEnd(prop, null)); + final String property = rawProperties.getProperty(key); + rawProperties.setProperty(key, property.trim()); } - ProtectedNiFiProperties protectedNiFiProperties = new ProtectedNiFiProperties(rawProperties); - return protectedNiFiProperties; + return new ProtectedNiFiProperties(rawProperties); } catch (final Exception ex) { - logger.error("Cannot load properties file due to " + ex.getLocalizedMessage()); + logger.error("Cannot load properties file due to {}", ex.getLocalizedMessage()); throw new RuntimeException("Cannot load properties file due to " + ex.getLocalizedMessage(), ex); - } finally { - if (null != inStream) { - try { - inStream.close(); - } catch (final Exception ex) { - /** - * do nothing * - */ - } - } } } @@ -249,9 +248,58 @@ public class NiFiPropertiesLoader { */ public NiFiProperties get() { if (instance == null) { - instance = loadDefault(); + instance = getDefaultProperties(); } return instance; } + + private NiFiProperties getDefaultProperties() { + NiFiProperties defaultProperties = loadDefault(); + if (isKeyGenerationRequired(defaultProperties)) { + if (defaultProperties.isClustered()) { + logger.error("Clustered Configuration Found: Shared Sensitive Properties Key [{}] required for cluster nodes", NiFiProperties.SENSITIVE_PROPS_KEY); + throw new SensitivePropertyProtectionException(PROPERTIES_KEY_MESSAGE); + } + + final File flowConfiguration = defaultProperties.getFlowConfigurationFile(); + if (flowConfiguration.exists()) { + logger.error("Flow Configuration [{}] Found: Migration Required for blank Sensitive Properties Key [{}]", flowConfiguration, NiFiProperties.SENSITIVE_PROPS_KEY); + throw new SensitivePropertyProtectionException(PROPERTIES_KEY_MESSAGE); + } + setSensitivePropertiesKey(); + defaultProperties = loadDefault(); + } + return defaultProperties; + } + + private void setSensitivePropertiesKey() { + logger.warn("Generating Random Sensitive Properties Key [{}]", NiFiProperties.SENSITIVE_PROPS_KEY); + final SecureRandom secureRandom = new SecureRandom(); + final byte[] sensitivePropertiesKeyBinary = new byte[SENSITIVE_PROPERTIES_KEY_LENGTH]; + secureRandom.nextBytes(sensitivePropertiesKeyBinary); + final String sensitivePropertiesKey = KEY_ENCODER.encodeToString(sensitivePropertiesKeyBinary); + try { + final File niFiPropertiesFile = new File(defaultPropertiesFilePath); + final Path niFiPropertiesPath = Paths.get(niFiPropertiesFile.toURI()); + final List lines = Files.readAllLines(niFiPropertiesPath); + final List updatedLines = lines.stream().map(line -> { + if (line.equals(EMPTY_SENSITIVE_PROPERTIES_KEY)) { + return line + sensitivePropertiesKey; + } else { + return line; + } + }).collect(Collectors.toList()); + Files.write(niFiPropertiesPath, updatedLines); + + logger.info("NiFi Properties [{}] updated with Sensitive Properties Key", niFiPropertiesPath); + } catch (final IOException e) { + throw new UncheckedIOException("Failed to set Sensitive Properties Key", e); + } + } + + private static boolean isKeyGenerationRequired(final NiFiProperties properties) { + final String configuredSensitivePropertiesKey = properties.getProperty(NiFiProperties.SENSITIVE_PROPS_KEY); + return (configuredSensitivePropertiesKey == null || configuredSensitivePropertiesKey.length() == 0); + } } diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-properties-loader/src/test/groovy/org/apache/nifi/properties/AESSensitivePropertyProviderTest.groovy b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-properties-loader/src/test/groovy/org/apache/nifi/properties/AESSensitivePropertyProviderTest.groovy index 05b4365b12..7bc22b6de3 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-properties-loader/src/test/groovy/org/apache/nifi/properties/AESSensitivePropertyProviderTest.groovy +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-properties-loader/src/test/groovy/org/apache/nifi/properties/AESSensitivePropertyProviderTest.groovy @@ -447,7 +447,7 @@ class AESSensitivePropertyProviderTest extends GroovyTestCase { @Test void testShouldEncryptArbitraryValues() { // Arrange - def values = ["thisIsABadPassword", "thisIsABadSensitiveKeyPassword", "thisIsABadKeystorePassword", "thisIsABadKeyPassword", "thisIsABadTruststorePassword", "This is an encrypted banner message", "nififtw!"] + def values = ["thisIsABadPassword", "thisIsABadSensitiveKeyPassword", "thisIsABadKeystorePassword", "thisIsABadKeyPassword", "thisIsABadTruststorePassword", "This is an encrypted banner message"] String key = "2C576A9585DB862F5ECBEE5B4FFFCCA1" //getKeyOfSize(128) // key = "0" * 64 diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-properties-loader/src/test/groovy/org/apache/nifi/properties/NiFiPropertiesLoaderGroovyTest.groovy b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-properties-loader/src/test/groovy/org/apache/nifi/properties/NiFiPropertiesLoaderGroovyTest.groovy index 3d15686f49..ef9ff4b07b 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-properties-loader/src/test/groovy/org/apache/nifi/properties/NiFiPropertiesLoaderGroovyTest.groovy +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-properties-loader/src/test/groovy/org/apache/nifi/properties/NiFiPropertiesLoaderGroovyTest.groovy @@ -18,6 +18,7 @@ package org.apache.nifi.properties import org.apache.commons.lang3.SystemUtils import org.apache.nifi.util.NiFiProperties +import org.apache.nifi.util.file.FileUtils import org.bouncycastle.jce.provider.BouncyCastleProvider import org.junit.After import org.junit.AfterClass @@ -201,6 +202,44 @@ class NiFiPropertiesLoaderGroovyTest extends GroovyTestCase { assert msg =~ "Cannot read from bootstrap.conf" } + @Test + void testShouldLoadUnprotectedPropertiesFromPathWithGeneratedSensitivePropertiesKey() throws Exception { + // Arrange + final File propertiesFile = File.createTempFile("nifi.without.key", ".properties") + propertiesFile.deleteOnExit() + final OutputStream outputStream = new FileOutputStream(propertiesFile) + final InputStream inputStream = getClass().getResourceAsStream("/conf/nifi.without.key.properties") + FileUtils.copy(inputStream, outputStream) + + System.setProperty(NiFiProperties.PROPERTIES_FILE_PATH, propertiesFile.absolutePath); + NiFiPropertiesLoader niFiPropertiesLoader = new NiFiPropertiesLoader() + + // Act + NiFiProperties niFiProperties = niFiPropertiesLoader.get() + + // Assert + final String sensitivePropertiesKey = niFiProperties.getProperty(NiFiProperties.SENSITIVE_PROPS_KEY) + assert sensitivePropertiesKey.length() == 32 + } + + @Test + void testShouldNotLoadUnprotectedPropertiesFromPathWithBlankKeyForClusterNode() throws Exception { + // Arrange + final File propertiesFile = File.createTempFile("nifi.without.key", ".properties") + propertiesFile.deleteOnExit() + final OutputStream outputStream = new FileOutputStream(propertiesFile) + final InputStream inputStream = getClass().getResourceAsStream("/conf/nifi.cluster.without.key.properties") + FileUtils.copy(inputStream, outputStream) + + System.setProperty(NiFiProperties.PROPERTIES_FILE_PATH, propertiesFile.absolutePath); + NiFiPropertiesLoader niFiPropertiesLoader = new NiFiPropertiesLoader() + + // Act + shouldFail(SensitivePropertyProtectionException) { + niFiPropertiesLoader.get() + } + } + @Test void testShouldNotLoadUnprotectedPropertiesFromNullFile() throws Exception { // Arrange diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-properties-loader/src/test/resources/conf/nifi.cluster.without.key.properties b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-properties-loader/src/test/resources/conf/nifi.cluster.without.key.properties new file mode 100644 index 0000000000..f9c2f7f295 --- /dev/null +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-properties-loader/src/test/resources/conf/nifi.cluster.without.key.properties @@ -0,0 +1,95 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Core Properties # +nifi.flow.configuration.file=./target/flow.xml.gz +nifi.flow.configuration.archive.dir=./target/archive/ +nifi.flowcontroller.autoResumeState=true +nifi.flowcontroller.graceful.shutdown.period=10 sec +nifi.flowservice.writedelay.interval=2 sec +nifi.administrative.yield.duration=30 sec + +nifi.reporting.task.configuration.file=./target/reporting-tasks.xml +nifi.controller.service.configuration.file=./target/controller-services.xml +nifi.templates.directory=./target/templates +nifi.ui.banner.text=UI Banner Text +nifi.ui.autorefresh.interval=30 sec +nifi.nar.library.directory=./target/resources/NiFiProperties/lib/ +nifi.nar.library.directory.alt=./target/resources/NiFiProperties/lib2/ +nifi.nar.working.directory=./target/work/nar/ + +# H2 Settings +nifi.database.directory=./target/database_repository +nifi.h2.url.append=;LOCK_TIMEOUT=25000;WRITE_DELAY=0;AUTO_SERVER=FALSE + +# FlowFile Repository +nifi.flowfile.repository.directory=./target/test-repo +nifi.flowfile.repository.partitions=1 +nifi.flowfile.repository.checkpoint.interval=2 mins +nifi.queue.swap.threshold=20000 +nifi.swap.storage.directory=./target/test-repo/swap +nifi.swap.in.period=5 sec +nifi.swap.in.threads=1 +nifi.swap.out.period=5 sec +nifi.swap.out.threads=4 + +# Content Repository +nifi.content.claim.max.appendable.size=10 MB +nifi.content.claim.max.flow.files=100 +nifi.content.repository.directory.default=./target/content_repository + +# Provenance Repository Properties +nifi.provenance.repository.storage.directory=./target/provenance_repository +nifi.provenance.repository.max.storage.time=24 hours +nifi.provenance.repository.max.storage.size=1 GB +nifi.provenance.repository.rollover.time=30 secs +nifi.provenance.repository.rollover.size=100 MB + +# Site to Site properties +nifi.remote.input.socket.port=9990 +nifi.remote.input.secure=true + +# web properties # +nifi.web.war.directory=./target/lib +nifi.web.http.host= +nifi.web.http.port=8080 +nifi.web.https.host= +nifi.web.https.port= +nifi.web.jetty.working.directory=./target/work/jetty + +# security properties # +nifi.sensitive.props.key= +nifi.sensitive.props.algorithm=PBEWITHMD5AND256BITAES-CBC-OPENSSL +nifi.sensitive.props.provider=BC + +nifi.security.keystore= +nifi.security.keystoreType= +nifi.security.keystorePasswd= +nifi.security.keyPasswd= +nifi.security.truststore= +nifi.security.truststoreType= +nifi.security.truststorePasswd= +nifi.security.user.authorizer= + +# cluster common properties (cluster manager and nodes must have same values) # +nifi.cluster.protocol.heartbeat.interval=5 sec +nifi.cluster.protocol.is.secure=false +nifi.cluster.protocol.socket.timeout=30 sec + +# cluster node properties (only configure for cluster nodes) # +nifi.cluster.is.node=true +nifi.cluster.node.address= +nifi.cluster.node.protocol.port= +nifi.cluster.node.protocol.threads=2 diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-properties-loader/src/test/resources/conf/nifi.without.key.properties b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-properties-loader/src/test/resources/conf/nifi.without.key.properties new file mode 100644 index 0000000000..dc0229832e --- /dev/null +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-properties-loader/src/test/resources/conf/nifi.without.key.properties @@ -0,0 +1,95 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Core Properties # +nifi.flow.configuration.file=./target/flow.xml.gz +nifi.flow.configuration.archive.dir=./target/archive/ +nifi.flowcontroller.autoResumeState=true +nifi.flowcontroller.graceful.shutdown.period=10 sec +nifi.flowservice.writedelay.interval=2 sec +nifi.administrative.yield.duration=30 sec + +nifi.reporting.task.configuration.file=./target/reporting-tasks.xml +nifi.controller.service.configuration.file=./target/controller-services.xml +nifi.templates.directory=./target/templates +nifi.ui.banner.text=UI Banner Text +nifi.ui.autorefresh.interval=30 sec +nifi.nar.library.directory=./target/resources/NiFiProperties/lib/ +nifi.nar.library.directory.alt=./target/resources/NiFiProperties/lib2/ +nifi.nar.working.directory=./target/work/nar/ + +# H2 Settings +nifi.database.directory=./target/database_repository +nifi.h2.url.append=;LOCK_TIMEOUT=25000;WRITE_DELAY=0;AUTO_SERVER=FALSE + +# FlowFile Repository +nifi.flowfile.repository.directory=./target/test-repo +nifi.flowfile.repository.partitions=1 +nifi.flowfile.repository.checkpoint.interval=2 mins +nifi.queue.swap.threshold=20000 +nifi.swap.storage.directory=./target/test-repo/swap +nifi.swap.in.period=5 sec +nifi.swap.in.threads=1 +nifi.swap.out.period=5 sec +nifi.swap.out.threads=4 + +# Content Repository +nifi.content.claim.max.appendable.size=10 MB +nifi.content.claim.max.flow.files=100 +nifi.content.repository.directory.default=./target/content_repository + +# Provenance Repository Properties +nifi.provenance.repository.storage.directory=./target/provenance_repository +nifi.provenance.repository.max.storage.time=24 hours +nifi.provenance.repository.max.storage.size=1 GB +nifi.provenance.repository.rollover.time=30 secs +nifi.provenance.repository.rollover.size=100 MB + +# Site to Site properties +nifi.remote.input.socket.port=9990 +nifi.remote.input.secure=true + +# web properties # +nifi.web.war.directory=./target/lib +nifi.web.http.host= +nifi.web.http.port=8080 +nifi.web.https.host= +nifi.web.https.port= +nifi.web.jetty.working.directory=./target/work/jetty + +# security properties # +nifi.sensitive.props.key= +nifi.sensitive.props.algorithm=PBEWITHMD5AND256BITAES-CBC-OPENSSL +nifi.sensitive.props.provider=BC + +nifi.security.keystore= +nifi.security.keystoreType= +nifi.security.keystorePasswd= +nifi.security.keyPasswd= +nifi.security.truststore= +nifi.security.truststoreType= +nifi.security.truststorePasswd= +nifi.security.user.authorizer= + +# cluster common properties (cluster manager and nodes must have same values) # +nifi.cluster.protocol.heartbeat.interval=5 sec +nifi.cluster.protocol.is.secure=false +nifi.cluster.protocol.socket.timeout=30 sec + +# cluster node properties (only configure for cluster nodes) # +nifi.cluster.is.node=false +nifi.cluster.node.address= +nifi.cluster.node.protocol.port= +nifi.cluster.node.protocol.threads=2 diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-resources/src/main/resources/bin/nifi.sh b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-resources/src/main/resources/bin/nifi.sh index 63fbfecc5d..da8080606e 100755 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-resources/src/main/resources/bin/nifi.sh +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-resources/src/main/resources/bin/nifi.sh @@ -334,6 +334,16 @@ run() { run_nifi_cmd="exec ${run_nifi_cmd}" fi + if [ "$1" = "set-sensitive-properties-key" ]; then + run_command="'${JAVA}' -cp '${BOOTSTRAP_CLASSPATH}' '-Dnifi.properties.file.path=${NIFI_HOME}/conf/nifi.properties' 'org.apache.nifi.flow.encryptor.command.SetSensitivePropertiesKey'" + eval "cd ${NIFI_HOME}" + shift + eval "${run_command}" '"$@"' + EXIT_STATUS=$? + echo + return; + fi + if [ "$1" = "stateless" ]; then STATELESS_JAVA_OPTS="${STATELESS_JAVA_OPTS:=-Xms1024m -Xmx1024m}" @@ -430,7 +440,7 @@ case "$1" in install "$@" ;; - start|stop|decommission|run|status|is_loaded|dump|diagnostics|env|stateless) + start|stop|decommission|run|status|is_loaded|dump|diagnostics|env|stateless|set-sensitive-properties-key) main "$@" ;; @@ -440,6 +450,6 @@ case "$1" in run "start" ;; *) - echo "Usage nifi {start|stop|decommission|run|restart|status|dump|diagnostics|install|stateless}" + echo "Usage nifi {start|stop|decommission|run|restart|status|dump|diagnostics|install|stateless|set-sensitive-properties-key}" ;; esac diff --git a/nifi-toolkit/nifi-toolkit-encrypt-config/pom.xml b/nifi-toolkit/nifi-toolkit-encrypt-config/pom.xml index 6358806929..319e2d95bf 100644 --- a/nifi-toolkit/nifi-toolkit-encrypt-config/pom.xml +++ b/nifi-toolkit/nifi-toolkit-encrypt-config/pom.xml @@ -24,6 +24,11 @@ nifi-toolkit-encrypt-config Tool to encrypt sensitive configuration values + + org.apache.nifi + nifi-flow-encryptor + 1.14.0-SNAPSHOT + org.apache.nifi nifi-properties diff --git a/nifi-toolkit/nifi-toolkit-encrypt-config/src/main/groovy/org/apache/nifi/properties/ConfigEncryptionTool.groovy b/nifi-toolkit/nifi-toolkit-encrypt-config/src/main/groovy/org/apache/nifi/properties/ConfigEncryptionTool.groovy index 539456dba1..f6f4453ed2 100644 --- a/nifi-toolkit/nifi-toolkit-encrypt-config/src/main/groovy/org/apache/nifi/properties/ConfigEncryptionTool.groovy +++ b/nifi-toolkit/nifi-toolkit-encrypt-config/src/main/groovy/org/apache/nifi/properties/ConfigEncryptionTool.groovy @@ -27,6 +27,10 @@ import org.apache.commons.cli.Option import org.apache.commons.cli.Options import org.apache.commons.cli.ParseException import org.apache.commons.codec.binary.Hex +import org.apache.nifi.encrypt.PropertyEncryptor +import org.apache.nifi.encrypt.PropertyEncryptorFactory +import org.apache.nifi.flow.encryptor.FlowEncryptor +import org.apache.nifi.flow.encryptor.StandardFlowEncryptor import org.apache.nifi.security.kms.CryptoUtils import org.apache.nifi.toolkit.tls.commandLine.CommandLineParseException import org.apache.nifi.toolkit.tls.commandLine.ExitCode @@ -41,22 +45,17 @@ import org.xml.sax.SAXException import javax.crypto.BadPaddingException import javax.crypto.Cipher -import javax.crypto.SecretKey -import javax.crypto.SecretKeyFactory -import javax.crypto.spec.PBEKeySpec -import javax.crypto.spec.PBEParameterSpec import java.nio.charset.StandardCharsets import java.nio.file.Path import java.nio.file.Paths -import java.nio.file.StandardCopyOption; +import java.nio.file.StandardCopyOption import java.security.KeyException -import java.security.SecureRandom import java.security.Security import java.util.regex.Matcher import java.util.zip.GZIPInputStream import java.util.zip.GZIPOutputStream import java.util.zip.ZipException -import java.nio.file.Files; +import java.nio.file.Files class ConfigEncryptionTool { private static final Logger logger = LoggerFactory.getLogger(ConfigEncryptionTool.class) @@ -69,7 +68,7 @@ class ConfigEncryptionTool { public String authorizersPath public String outputAuthorizersPath public static flowXmlPath - public outputFlowXmlPath + public String outputFlowXmlPath private String keyHex private String migrationKeyHex @@ -124,7 +123,7 @@ class ConfigEncryptionTool { // Static holder to avoid re-generating the options object multiple times in an invocation private static Options staticOptions - // Hard-coded fallback value from {@link org.apache.nifi.encrypt.PropertyEncryptorFactory} + // Hard-coded fallback value from historical defaults private static final String DEFAULT_NIFI_SENSITIVE_PROPS_KEY = "nififtw!" private static final int MIN_PASSWORD_LENGTH = 12 @@ -133,11 +132,6 @@ class ConfigEncryptionTool { private static final int SCRYPT_N = 2**16 private static final int SCRYPT_R = 8 private static final int SCRYPT_P = 1 - static final String CURRENT_SCRYPT_VERSION = "s0" - - // Hard-coded values from StandardPBEByteEncryptor which will be removed during refactor of all flow encryption code in NIFI-1465 - private static final int DEFAULT_KDF_ITERATIONS = 1000 - private static final int DEFAULT_SALT_SIZE_BYTES = 16 private static final String BOOTSTRAP_KEY_COMMENT = "# Root key in hexadecimal format for encrypted sensitive configuration values" @@ -191,7 +185,6 @@ class ConfigEncryptionTool { private static final String XML_DECLARATION_REGEX = /<\?xml version="1.0" encoding="UTF-8"\?>/ private static final String WRAPPED_FLOW_XML_CIPHER_TEXT_REGEX = /enc\{[a-fA-F0-9]+?\}/ - private static final String DEFAULT_PROVIDER = BouncyCastleProvider.PROVIDER_NAME private static final String DEFAULT_FLOW_ALGORITHM = "PBEWITHMD5AND256BITAES-CBC-OPENSSL" private static final Map PROPERTY_KEY_MAP = [ @@ -682,113 +675,6 @@ class ConfigEncryptionTool { } } - /** - * Decrypts a single element encrypted in the flow.xml.gz style (hex-encoded and wrapped with "enc{" and "}"). - * - * Example: - * {@code enc{0123456789ABCDEF} } -> "some text" - * - * @param wrappedCipherText the wrapped and hex-encoded cipher text - * @param password the password used to encrypt the content (UTF-8 encoded) - * @param algorithm the encryption and KDF algorithm (defaults to PBEWITHMD5AND256BITAES-CBC-OPENSSL) - * @param provider the security provider (defaults to BC) - * @return the plaintext in UTF-8 encoding - */ - private - static String decryptFlowElement(String wrappedCipherText, String password, String algorithm = DEFAULT_FLOW_ALGORITHM, String provider = DEFAULT_PROVIDER) { - // Drop the "enc{" and closing "}" - if (!(wrappedCipherText =~ WRAPPED_FLOW_XML_CIPHER_TEXT_REGEX)) { - throw new SensitivePropertyProtectionException("The provided cipher text does not match the expected format 'enc{0123456789ABCDEF...}'") - } - String unwrappedCipherText = wrappedCipherText.replaceAll(/enc\{/, "")[0..<-1] - if (unwrappedCipherText.length() % 2 == 1 || unwrappedCipherText.length() == 0) { - throw new SensitivePropertyProtectionException("The provided cipher text must have an even number of hex characters") - } - - // Decode the hex - byte[] cipherBytes = Hex.decodeHex(unwrappedCipherText.chars) - - /* The structure of each cipher text is 16 bytes of salt || actual cipher text, - * so extract the salt (32 bytes encoded as hex, 16 bytes raw) and combine that - * with the default (and unchanged) iteration count that is hardcoded in - * {@link StandardPBEByteEncryptor}. I am extracting - * these values to magic numbers here so when the refactoring is performed, - * stronger decisions can be implemented here - */ - byte[] saltBytes = cipherBytes[0.. {@code enc{0123456789ABCDEF} } - * - * @param plaintext the plaintext in UTF-8 encoding - * @param saltBytes the salt to embed in the cipher text to allow key derivation and decryption later in raw format - * @param encryptCipher the configured Cipher instance - * @return the wrapped and hex-encoded cipher text - */ - private static String encryptFlowElement(String plaintext, byte[] saltBytes, Cipher encryptCipher) { - byte[] plainBytes = plaintext?.getBytes(StandardCharsets.UTF_8) ?: new byte[0] - - /* The structure of each cipher text is 16 bytes of salt || actual cipher text, - * so extract the salt (32 bytes encoded as hex, 16 bytes raw) and combine that - * with the default (and unchanged) iteration count that is hardcoded in - * {@link StandardPBEByteEncryptor}. I am extracting - * these values to magic numbers here so when the refactoring is performed, - * stronger decisions can be implemented here - */ - if (saltBytes.length != DEFAULT_SALT_SIZE_BYTES) { - throw new SensitivePropertyProtectionException("The salt must be ${DEFAULT_SALT_SIZE_BYTES} bytes") - } - - byte[] cipherBytes = encryptCipher.doFinal(plainBytes) - byte[] saltAndCipherBytes = concatByteArrays(saltBytes, cipherBytes) - - // Encode the hex - String hexEncodedCipherText = Hex.encodeHexString(saltAndCipherBytes) - "enc{${hexEncodedCipherText}}" - } - - /** - * Utility method to quickly concatenate an arbitrary number of byte[]. - * - * @param arrays the byte[] arrays - * @returna single byte[] containing the values concatenated - */ - private static byte[] concatByteArrays(byte[] ... arrays) { - ByteArrayOutputStream outputStream = new ByteArrayOutputStream() - arrays.each { byte[] it -> outputStream.write(it) } - outputStream.toByteArray() - } - /** * Scans XML content and decrypts each encrypted element, then re-encrypts it with the new key, and returns the final XML content. * @@ -799,57 +685,37 @@ class ConfigEncryptionTool { * @param existingProvider the {@link java.security.Provider} to use (defaults to BC) * @return the encrypted XML content as an InputStream */ - private InputStream migrateFlowXmlContent(InputStream flowXmlContent, String existingFlowPassword, String newFlowPassword, String existingAlgorithm = DEFAULT_FLOW_ALGORITHM, String existingProvider = DEFAULT_PROVIDER, String newAlgorithm = DEFAULT_FLOW_ALGORITHM, String newProvider = DEFAULT_PROVIDER) { - /* For re-encryption, for performance reasons, we will use a fixed salt for all of - * the operations. These values are stored in the same file and the default key is in the - * source code (see NIFI-1465 and NIFI-1277), so the security trade-off is minimal - * but the performance hit is substantial. We can't make this decision for - * decryption because the FlowSerializer still uses PropertyEncryptor which does not - * follow this pattern - */ - byte[] encryptionSalt = new byte[DEFAULT_SALT_SIZE_BYTES] - new SecureRandom().nextBytes(encryptionSalt) - Cipher encryptCipher = generateFlowEncryptionCipher(newFlowPassword, encryptionSalt, newAlgorithm, newProvider) - - int elementCount = 0 + private InputStream migrateFlowXmlContent(InputStream flowXmlContent, String existingFlowPassword, String newFlowPassword, String existingAlgorithm = DEFAULT_FLOW_ALGORITHM, String newAlgorithm = DEFAULT_FLOW_ALGORITHM) { File tempFlowXmlFile = new File(getTemporaryFlowXmlFile(outputFlowXmlPath).toString()) - BufferedWriter tempFlowXmlWriter = getFlowOutputStream(tempFlowXmlFile, flowXmlContent instanceof GZIPInputStream) + final OutputStream flowOutputStream = getFlowOutputStream(tempFlowXmlFile, flowXmlContent instanceof GZIPInputStream) - // Scan through XML content as a stream, decrypt and re-encrypt fields with a new flow password - final BufferedReader reader = new BufferedReader(new InputStreamReader(flowXmlContent)) - String line; + NiFiProperties inputProperties = NiFiProperties.createBasicNiFiProperties("", [ + (NiFiProperties.SENSITIVE_PROPS_KEY): existingFlowPassword, + (NiFiProperties.SENSITIVE_PROPS_ALGORITHM): existingAlgorithm + ]) - while((line = reader.readLine()) != null) { - def matcher = line =~ WRAPPED_FLOW_XML_CIPHER_TEXT_REGEX - if(matcher.find()) { - String plaintext = decryptFlowElement(matcher.getAt(0), existingFlowPassword, existingAlgorithm, existingProvider) - byte[] cipherBytes = encryptCipher.doFinal(plaintext.bytes) - byte[] saltAndCipherBytes = concatByteArrays(encryptionSalt, cipherBytes) - elementCount++ - tempFlowXmlWriter.writeLine(line.replaceFirst(WRAPPED_FLOW_XML_CIPHER_TEXT_REGEX, "enc{${Hex.encodeHex(saltAndCipherBytes)}}")) - } else { - tempFlowXmlWriter.writeLine(line) - } - } - tempFlowXmlWriter.flush() - tempFlowXmlWriter.close() + NiFiProperties outputProperties = NiFiProperties.createBasicNiFiProperties("", [ + (NiFiProperties.SENSITIVE_PROPS_KEY): newFlowPassword, + (NiFiProperties.SENSITIVE_PROPS_ALGORITHM): newAlgorithm + ]) + + final PropertyEncryptor inputEncryptor = PropertyEncryptorFactory.getPropertyEncryptor(inputProperties) + final PropertyEncryptor outputEncryptor = PropertyEncryptorFactory.getPropertyEncryptor(outputProperties) + + final FlowEncryptor flowEncryptor = new StandardFlowEncryptor() + flowEncryptor.processFlow(flowXmlContent, flowOutputStream, inputEncryptor, outputEncryptor) // Overwrite the original flow file with the migrated flow file Files.move(tempFlowXmlFile.toPath(), Paths.get(outputFlowXmlPath), StandardCopyOption.ATOMIC_MOVE) - - if (isVerbose) { - logger.info("Decrypted and re-encrypted ${elementCount} elements for flow.xml.gz") - } - loadFlowXml(outputFlowXmlPath) } - private BufferedWriter getFlowOutputStream(File outputFlowXmlPath, boolean isFileGZipped) { + private OutputStream getFlowOutputStream(File outputFlowXmlPath, boolean isFileGZipped) { OutputStream flowOutputStream = new FileOutputStream(outputFlowXmlPath) if(isFileGZipped) { flowOutputStream = new GZIPOutputStream(flowOutputStream) } - new BufferedWriter(new OutputStreamWriter(flowOutputStream)) + return flowOutputStream } // Create a temporary output file we can write the stream to @@ -859,35 +725,6 @@ class ConfigEncryptionTool { Paths.get(originalOutputFlowXmlPath).resolveSibling(migratedFileName) } - /** - * Returns an initialized encryption cipher for the flow.xml.gz content. - * - * @param newFlowPassword the new encryption password - * @param saltBytes the salt [16 bytes in raw format] - * @param algorithm the KDF/encryption algorithm - * @param provider the security provider - * @return the initialized cipher instance - */ - private - static Cipher generateFlowEncryptionCipher(String newFlowPassword, byte[] saltBytes, String algorithm = DEFAULT_FLOW_ALGORITHM, String provider = DEFAULT_PROVIDER) { - // Use the standard Cipher with the password and algorithm provided - Cipher encryptCipher = Cipher.getInstance(algorithm, provider) - - /* For re-encryption, for performance reasons, we will use a fixed salt for all of - * the operations. These values are stored in the same file and the default key is in the - * source code (see NIFI-1465 and NIFI-1277), so the security trade-off is minimal - * but the performance hit is substantial. We can't make this decision for - * decryption because the FlowSerializer still uses PropertyEncryptor which does not - * follow this pattern - */ - PBEKeySpec keySpec = new PBEKeySpec(newFlowPassword.chars) - SecretKeyFactory keyFactory = SecretKeyFactory.getInstance(algorithm, provider) - SecretKey pbeKey = keyFactory.generateSecret(keySpec) - PBEParameterSpec parameterSpec = new PBEParameterSpec(saltBytes, DEFAULT_KDF_ITERATIONS) - encryptCipher.init(Cipher.ENCRYPT_MODE, pbeKey, parameterSpec) - encryptCipher - } - String decryptLoginIdentityProviders(String encryptedXml, String existingKeyHex = keyHex) { AESSensitivePropertyProvider sensitivePropertyProvider = new AESSensitivePropertyProvider(existingKeyHex) @@ -1619,14 +1456,12 @@ class ConfigEncryptionTool { // Get the algorithms and providers NiFiProperties nfp = niFiProperties String existingAlgorithm = nfp?.getProperty(NiFiProperties.SENSITIVE_PROPS_ALGORITHM) ?: DEFAULT_FLOW_ALGORITHM - String existingProvider = nfp?.getProperty(NiFiProperties.SENSITIVE_PROPS_PROVIDER) ?: DEFAULT_PROVIDER String newAlgorithm = newFlowAlgorithm ?: existingAlgorithm - String newProvider = newFlowProvider ?: existingProvider try { logger.info("Migrating flow.xml file at ${flowXmlPath}. This could take a while if the flow XML is very large.") - migrateFlowXmlContent(flowXmlInputStream, existingFlowPassword, newFlowPassword, existingAlgorithm, existingProvider, newAlgorithm, newProvider) + migrateFlowXmlContent(flowXmlInputStream, existingFlowPassword, newFlowPassword, existingAlgorithm, newAlgorithm) } catch (Exception e) { logger.error("Encountered an error: ${e.getLocalizedMessage()}") if (e instanceof BadPaddingException) { diff --git a/nifi-toolkit/nifi-toolkit-encrypt-config/src/test/groovy/org/apache/nifi/properties/ConfigEncryptionToolTest.groovy b/nifi-toolkit/nifi-toolkit-encrypt-config/src/test/groovy/org/apache/nifi/properties/ConfigEncryptionToolTest.groovy index de2d37941c..acb118b482 100644 --- a/nifi-toolkit/nifi-toolkit-encrypt-config/src/test/groovy/org/apache/nifi/properties/ConfigEncryptionToolTest.groovy +++ b/nifi-toolkit/nifi-toolkit-encrypt-config/src/test/groovy/org/apache/nifi/properties/ConfigEncryptionToolTest.groovy @@ -83,16 +83,6 @@ class ConfigEncryptionToolTest extends GroovyTestCase { private static final String PASSWORD = "thisIsABadPassword" private static final String ANOTHER_PASSWORD = "thisIsAnotherBadPassword" - private static final String STATIC_SALT = "\$s0\$40801\$ABCDEFGHIJKLMNOPQRSTUV" - private static final String SCRYPT_SALT_PATTERN = /\$\w{2}\$\w{5,}\$[\w\/\=\+]+/ - - // Hash of "password" with 00 * 16 salt - private static - final String HASHED_PASSWORD = "\$s0\$40801\$AAAAAAAAAAAAAAAAAAAAAA\$gLSh7ChbHdOIMvZ74XGjV6qF65d9qvQ8n75FeGnM8YM" - // Hash of [key derived from "password"] with 00 * 16 salt - private static - final String HASHED_KEY_HEX = "\$s0\$40801\$AAAAAAAAAAAAAAAAAAAAAA\$pJOGA9sPL+pRzynnwt6G2FfVTyLQdbKSbk6W8IKId8E" - // From ConfigEncryptionTool.deriveKeyFromPassword("thisIsABadPassword") private static final String PASSWORD_KEY_HEX_256 = "2C576A9585DB862F5ECBEE5B4FFFCCA14B18D8365968D7081651006507AD2BDE" @@ -137,6 +127,7 @@ class ConfigEncryptionToolTest extends GroovyTestCase { @After void tearDown() throws Exception { + System.clearProperty(NiFiProperties.PROPERTIES_FILE_PATH) TestAppender.reset() } @@ -842,37 +833,6 @@ class ConfigEncryptionToolTest extends GroovyTestCase { workingFile.deleteOnExit() } - @Ignore("Setting the Windows file permissions fails in the test harness, so the test does not throw the expected exception") - @Test - void testLoadNiFiPropertiesShouldHandleReadFailureOnWindows() { - // Arrange - Assume.assumeTrue("Test only runs on Windows", SystemUtils.IS_OS_WINDOWS) - - File inputPropertiesFile = new File("src/test/resources/nifi_with_sensitive_properties_unprotected.properties") - File workingFile = new File("target/tmp_nifi.properties") - workingFile.delete() - - Files.copy(inputPropertiesFile.toPath(), workingFile.toPath()) - // Empty set of permissions - workingFile.setReadable(false) - - ConfigEncryptionTool tool = new ConfigEncryptionTool() - String[] args = ["-n", workingFile.path, "-k", KEY_HEX] - tool.parse(args) - - // Act - def msg = shouldFail(IOException) { - tool.loadNiFiProperties() - logger.info("Read nifi.properties") - } - logger.expected(msg) - - // Assert - assert msg == "Cannot load NiFiProperties from [${workingFile.path}]".toString() - - workingFile.deleteOnExit() - } - @Test void testShouldEncryptSensitiveProperties() { // Arrange @@ -1028,66 +988,6 @@ class ConfigEncryptionToolTest extends GroovyTestCase { workingFile.deleteOnExit() } - @Ignore("this test needs to be updated to ensure any created files are done under target") - @Test - void testWriteKeyToBootstrapConfShouldHandleReadFailure() { - // Arrange - File emptyKeyFile = new File("src/test/resources/bootstrap_with_empty_root_key.conf") - File workingFile = new File("target/tmp_bootstrap.conf") - workingFile.delete() - - Files.copy(emptyKeyFile.toPath(), workingFile.toPath()) - // Empty set of permissions - setFilePermissions(workingFile, []) - logger.info("Set POSIX permissions to ${getFilePermissions(workingFile)}") - - ConfigEncryptionTool tool = new ConfigEncryptionTool() - String[] args = ["-b", workingFile.path, "-k", KEY_HEX, "-n", "nifi.properties"] - tool.parse(args) - - // Act - def msg = shouldFail(IOException) { - tool.writeKeyToBootstrapConf() - logger.info("Updated bootstrap.conf") - } - logger.expected(msg) - - // Assert - assert msg == "The bootstrap.conf file at tmp_bootstrap.conf must exist and be readable and writable by the user running this tool" - - workingFile.deleteOnExit() - } - - @Ignore("this test needs to be updated to ensure any created files are done under target") - @Test - void testWriteKeyToBootstrapConfShouldHandleWriteFailure() { - // Arrange - File emptyKeyFile = new File("src/test/resources/bootstrap_with_empty_root_key.conf") - File workingFile = new File("target/tmp_bootstrap.conf") - workingFile.delete() - - Files.copy(emptyKeyFile.toPath(), workingFile.toPath()) - // Read-only set of permissions - setFilePermissions(workingFile, [PosixFilePermission.OWNER_READ, PosixFilePermission.GROUP_READ, PosixFilePermission.OTHERS_READ]) - logger.info("Set POSIX permissions to ${getFilePermissions(workingFile)}") - - ConfigEncryptionTool tool = new ConfigEncryptionTool() - String[] args = ["-b", workingFile.path, "-k", KEY_HEX, "-n", "nifi.properties"] - tool.parse(args) - - // Act - def msg = shouldFail(IOException) { - tool.writeKeyToBootstrapConf() - logger.info("Updated bootstrap.conf") - } - logger.expected(msg) - - // Assert - assert msg == "The bootstrap.conf file at tmp_bootstrap.conf must exist and be readable and writable by the user running this tool" - - workingFile.deleteOnExit() - } - @Test void testShouldEncryptNiFiPropertiesWithEmptyProtectionScheme() { // Arrange @@ -1474,43 +1374,6 @@ class ConfigEncryptionToolTest extends GroovyTestCase { setupTmpDir() } - @Ignore("Setting the Windows file permissions fails in the test harness, so the test does not throw the expected exception") - @Test - void testWriteNiFiPropertiesShouldHandleWriteFailureWhenFileDoesNotExistOnWindows() { - // Arrange - Assume.assumeTrue("Test only runs on Windows", SystemUtils.IS_OS_WINDOWS) - - File inputPropertiesFile = new File("src/test/resources/nifi_with_sensitive_properties_unprotected.properties") - File tmpDir = new File("target/tmp/") - tmpDir.mkdirs() - File workingFile = new File("target/tmp/tmp_nifi.properties") - workingFile.delete() - - // Read-only set of permissions - tmpDir.setWritable(false) - - ConfigEncryptionTool tool = new ConfigEncryptionTool() - String[] args = ["-n", inputPropertiesFile.path, "-o", workingFile.path, "-k", KEY_HEX] - tool.parse(args) - NiFiProperties niFiProperties = tool.loadNiFiProperties() - tool.@niFiProperties = niFiProperties - logger.info("Loaded ${niFiProperties.size()} properties from ${inputPropertiesFile.path}") - - // Act - def msg = shouldFail(IOException) { - tool.writeNiFiProperties() - logger.info("Wrote to ${workingFile.path}") - } - logger.expected(msg) - - // Assert - assert msg == "The nifi.properties file at ${workingFile.path} must be writable by the user running this tool".toString() - - workingFile.deleteOnExit() - setFilePermissions(tmpDir, [PosixFilePermission.OWNER_READ, PosixFilePermission.OWNER_WRITE]) - tmpDir.deleteOnExit() - } - @Test void testShouldPerformFullOperation() { // Arrange @@ -3821,11 +3684,6 @@ class ConfigEncryptionToolTest extends GroovyTestCase { def updatedFlowCipherTexts = findFieldsInStream(updatedFlowXmlContent, WFXCTR) logger.info("Updated flow.xml.gz cipher texts: ${updatedFlowCipherTexts}") assert updatedFlowCipherTexts.size() == CIPHER_TEXT_COUNT - updatedFlowCipherTexts.each { - String decryptedValue = ConfigEncryptionTool.decryptFlowElement(it, newFlowPassword) - logger.info("Decrypted value of migrated ${workingFlowXmlFile.path} was: ${decryptedValue}") - assert decryptedValue == PASSWORD || decryptedValue == ANOTHER_PASSWORD - } } }) @@ -3927,11 +3785,6 @@ class ConfigEncryptionToolTest extends GroovyTestCase { def migratedFlowCipherTexts = findFieldsInStream(migratedFlowXmlContent, WFXCTR) logger.info("Updated flow.xml.gz cipher texts: ${migratedFlowCipherTexts}") assert migratedFlowCipherTexts.size() == CIPHER_TEXT_COUNT - migratedFlowCipherTexts.each { - String decryptedValue = ConfigEncryptionTool.decryptFlowElement(it, newFlowPassword) - logger.info("Decrypted value of migrated ${workingFlowXmlFile.path} was: ${decryptedValue}") - assert decryptedValue == PASSWORD || decryptedValue == ANOTHER_PASSWORD - } } }) @@ -4068,11 +3921,6 @@ class ConfigEncryptionToolTest extends GroovyTestCase { logger.info("Original flow.xml.gz cipher texts: ${originalFlowCipherTexts}") logger.info("Updated flow.xml.gz cipher texts: ${migratedFlowCipherTexts}") assert migratedFlowCipherTexts.size() == CIPHER_TEXT_COUNT - migratedFlowCipherTexts.each { - String decryptedValue = ConfigEncryptionTool.decryptFlowElement(it, newFlowPassword) - logger.info("Decrypted value of migrated ${workingFlowXmlFile.path} was: ${decryptedValue}") - assert decryptedValue == PASSWORD || decryptedValue == ANOTHER_PASSWORD - } } }) @@ -4204,11 +4052,6 @@ class ConfigEncryptionToolTest extends GroovyTestCase { logger.info("Original " + workingFlowXmlFile.path + " unique cipher texts: ${originalFlowCipherTexts}") logger.info("Migrated " + workingFlowXmlFile.path + " unique cipher texts: ${migratedFlowCipherTexts}") assert migratedFlowCipherTexts.size() == CIPHER_TEXT_COUNT - migratedFlowCipherTexts.each { - String decryptedValue = ConfigEncryptionTool.decryptFlowElement(it, newFlowPassword) - logger.info("Decrypted value of migrated ${workingFlowXmlFile.path} was: ${decryptedValue}") - assert decryptedValue == PASSWORD || decryptedValue == ANOTHER_PASSWORD - } } }) @@ -4362,11 +4205,6 @@ class ConfigEncryptionToolTest extends GroovyTestCase { def flowCipherTexts = findFieldsInStream(updatedFlowXmlContent, WFXCTR) logger.info("Updated flow.xml.gz cipher texts: ${flowCipherTexts}") assert flowCipherTexts.size() == CIPHER_TEXT_COUNT - flowCipherTexts.each { - String decryptedValue = ConfigEncryptionTool.decryptFlowElement(it, newFlowPassword) - logger.info("Decrypted value of migrated ${workingFlowXmlFile.path} was: ${decryptedValue}") - assert decryptedValue == PASSWORD || decryptedValue == ANOTHER_PASSWORD - } // Update the "original" flow cipher texts for the next run to the current values originalFlowCipherTexts = flowCipherTexts @@ -4374,178 +4212,6 @@ class ConfigEncryptionToolTest extends GroovyTestCase { } } - @Test - void testDecryptFlowXmlContentShouldVerifyPattern() { - // Arrange - String existingFlowPassword = "flowPassword" - String sensitivePropertyValue = "thisIsABadProcessorPassword" - - final Map properties = new HashMap<>() - properties.put(NiFiProperties.SENSITIVE_PROPS_ALGORITHM, DEFAULT_ENCRYPTION_METHOD.algorithm) - properties.put(NiFiProperties.SENSITIVE_PROPS_PROVIDER, DEFAULT_ENCRYPTION_METHOD.provider) - properties.put(NiFiProperties.SENSITIVE_PROPS_KEY, existingFlowPassword) - final NiFiProperties niFiProperties = NiFiProperties.createBasicNiFiProperties(null, properties) - - PropertyEncryptor sanityEncryptor = PropertyEncryptorFactory.getPropertyEncryptor(niFiProperties) - String sanityCipherText = "enc{${sanityEncryptor.encrypt(sensitivePropertyValue)}}" - logger.info("Sanity check value: \t${sensitivePropertyValue} -> ${sanityCipherText}") - - def validCipherTexts = (0..4).collect { - "enc{${sanityEncryptor.encrypt(sensitivePropertyValue)}}" - } - logger.info("Generated valid cipher texts: \n${validCipherTexts.join("\n")}") - - def invalidCipherTexts = ["enc{}", - "enc{x}", - "encx", - "enc{012}", - "enc{01", - "enc{aBc19+===}", - "enc{aB=c19+}", - "enc{aB@}", - "", - "}", - "\"", - ">", - null] - - // Act - def successfulResults = validCipherTexts.collect { String cipherText -> - ConfigEncryptionTool.decryptFlowElement(cipherText, existingFlowPassword) - } - - def failedResults = invalidCipherTexts.collect { String cipherText -> - def msg = shouldFail(SensitivePropertyProtectionException) { - ConfigEncryptionTool.decryptFlowElement(cipherText, existingFlowPassword) - } - logger.expected(msg) - msg - } - - // Assert - assert successfulResults.every { it == sensitivePropertyValue } - assert failedResults.every { - it =~ /The provided cipher text does not match the expected format 'enc\{0123456789ABCDEF\.\.\.\}'/ || - it == "The provided cipher text must have an even number of hex characters" - } - } - - /** - * This test verifies that the crypto logic in the tool is compatible with the default encryptor implementation. - */ - @Test - void testShouldDecryptFlowXmlContent() { - // Arrange - String existingFlowPassword = "nififtw!" - String sensitivePropertyValue = "thisIsABadProcessorPassword" - - final Map properties = new HashMap<>() - properties.put(NiFiProperties.SENSITIVE_PROPS_ALGORITHM, DEFAULT_ENCRYPTION_METHOD.algorithm) - properties.put(NiFiProperties.SENSITIVE_PROPS_PROVIDER, DEFAULT_ENCRYPTION_METHOD.provider) - properties.put(NiFiProperties.SENSITIVE_PROPS_KEY, existingFlowPassword) - final NiFiProperties niFiProperties = NiFiProperties.createBasicNiFiProperties(null, properties) - - PropertyEncryptor sanityEncryptor = PropertyEncryptorFactory.getPropertyEncryptor(niFiProperties) - String sanityCipherText = "enc{${sanityEncryptor.encrypt(sensitivePropertyValue)}}" - logger.info("Sanity check value: \t${sensitivePropertyValue} -> ${sanityCipherText}") - - // Act - String decryptedElement = ConfigEncryptionTool.decryptFlowElement(sanityCipherText, existingFlowPassword, DEFAULT_ALGORITHM, DEFAULT_PROVIDER) - logger.info("Decrypted flow element: ${decryptedElement}") - String decryptedElementWithDefaultParameters = ConfigEncryptionTool.decryptFlowElement(sanityCipherText, existingFlowPassword) - logger.info("Decrypted flow element: ${decryptedElementWithDefaultParameters}") - - // Assert - assert decryptedElement == sensitivePropertyValue - assert decryptedElementWithDefaultParameters == sensitivePropertyValue - } - - /** - * This test verifies that the crypto logic in the tool is compatible with an encrypted value taken from a production flow.xml.gz. - */ - @Test - void testShouldDecryptFlowXmlContentFromLegacyFlow() { - // Arrange - - // DEFAULT_SENSITIVE_PROPS_KEY = "nififtw!" at the time this test - // was written and for the encrypted value, but it could change, so don't - // reference transitively here - String existingFlowPassword = DEFAULT_LEGACY_SENSITIVE_PROPS_KEY - - final String EXPECTED_PLAINTEXT = "thisIsABadPassword" - - final String ENCRYPTED_VALUE_FROM_FLOW = "enc{5d8c45f04790e73cba72e5e3fbee1145f2e18256c3b33c283e17f5281611cb5e5f9e6cc988c5be0e8cca7b5dc8fa7cf7}" - - // Act - String decryptedElement = ConfigEncryptionTool.decryptFlowElement(ENCRYPTED_VALUE_FROM_FLOW, existingFlowPassword, DEFAULT_ALGORITHM, DEFAULT_PROVIDER) - logger.info("Decrypted flow element: ${decryptedElement}") - - // Assert - assert decryptedElement == EXPECTED_PLAINTEXT - } - - @Test - void testShouldEncryptFlowXmlContent() { - // Arrange - String flowPassword = "nififtw!" - String sensitivePropertyValue = "thisIsAnotherBadPassword" - byte[] saltBytes = "thisIsABadSalt..".bytes - - final Map properties = new HashMap<>() - properties.put(NiFiProperties.SENSITIVE_PROPS_ALGORITHM, DEFAULT_ENCRYPTION_METHOD.algorithm) - properties.put(NiFiProperties.SENSITIVE_PROPS_PROVIDER, DEFAULT_ENCRYPTION_METHOD.provider) - properties.put(NiFiProperties.SENSITIVE_PROPS_KEY, flowPassword) - final NiFiProperties niFiProperties = NiFiProperties.createBasicNiFiProperties(null, properties) - - PropertyEncryptor sanityEncryptor = PropertyEncryptorFactory.getPropertyEncryptor(niFiProperties) - - Cipher encryptionCipher = generateEncryptionCipher(flowPassword) - - // Act - String encryptedElement = ConfigEncryptionTool.encryptFlowElement(sensitivePropertyValue, saltBytes, encryptionCipher) - logger.info("Encrypted flow element: ${encryptedElement}") - - // Assert - assert encryptedElement =~ WFXCTR - String sanityPlaintext = sanityEncryptor.decrypt(encryptedElement[4..<-1]) - logger.info("Sanity check value: \t${encryptedElement} -> ${sanityPlaintext}") - - assert sanityPlaintext == sensitivePropertyValue - } - - @Test - void testShouldEncryptAndDecryptFlowXmlContent() { - // Arrange - String flowPassword = "flowPassword" - String sensitivePropertyValue = "thisIsABadProcessorPassword" - byte[] saltBytes = "thisIsABadSalt..".bytes - - Cipher encryptionCipher = generateEncryptionCipher(flowPassword) - - // Act - String encryptedElement = ConfigEncryptionTool.encryptFlowElement(sensitivePropertyValue, saltBytes, encryptionCipher) - logger.info("Encrypted flow element: ${encryptedElement}") - - String decryptedElement = ConfigEncryptionTool.decryptFlowElement(encryptedElement, flowPassword) - logger.info("Decrypted flow element: ${decryptedElement}") - - // Assert - assert encryptedElement =~ WFXCTR - assert decryptedElement == sensitivePropertyValue - } - - private - static Cipher generateEncryptionCipher(String password, String algorithm = DEFAULT_ALGORITHM, String provider = DEFAULT_PROVIDER) { - Cipher cipher = Cipher.getInstance(algorithm, provider) - PBEKeySpec keySpec = new PBEKeySpec(password.chars) - SecretKeyFactory keyFactory = SecretKeyFactory.getInstance(algorithm, provider) - SecretKey pbeKey = keyFactory.generateSecret(keySpec) - byte[] saltBytes = "thisIsABadSalt..".bytes - PBEParameterSpec parameterSpec = new PBEParameterSpec(saltBytes, 1000) - cipher.init(Cipher.ENCRYPT_MODE, pbeKey, parameterSpec) - cipher - } - @Test void testShouldMigrateFlowXmlContent() { // Arrange @@ -4584,11 +4250,6 @@ class ConfigEncryptionToolTest extends GroovyTestCase { def migratedCipherTexts = findFieldsInStream(migratedFlowXmlFile, WFXCTR) assert migratedCipherTexts.size() == cipherTextCount - migratedCipherTexts.each { - String decryptedValue = ConfigEncryptionTool.decryptFlowElement(it, newFlowPassword) - logger.info("Decrypted value of migrated " + workingFile.path + " was: " + decryptedValue) - assert decryptedValue == PASSWORD || decryptedValue == ANOTHER_PASSWORD - } // Ensure that everything else is identical assert flowXmlFile.text.replaceAll(WFXCTR, "") == @@ -4642,11 +4303,6 @@ class ConfigEncryptionToolTest extends GroovyTestCase { assert newCipherTexts.size() == ORIGINAL_CIPHER_TEXT_COUNT - newCipherTexts.each { - String decryptedValue = ConfigEncryptionTool.decryptFlowElement(it, newFlowPassword) - assert decryptedValue == PASSWORD || decryptedValue == ANOTHER_PASSWORD - } - // Ensure that everything else is identical assert new File(workingFile.path).text.replaceAll(WFXCTR, "") == flowXmlFile.text.replaceAll(WFXCTR, "") @@ -4699,42 +4355,6 @@ class ConfigEncryptionToolTest extends GroovyTestCase { } } - /** - * This test is tightly scoped to the migration of the flow XML content to ensure the expected exception type is thrown. - */ - @Test - void testMigrateFlowXmlContentWithIncorrectExistingPasswordShouldFailWithBadPaddingException() { - // Arrange - String flowXmlPath = "src/test/resources/flow.xml" - File flowXmlFile = new File(flowXmlPath) - - File tmpDir = setupTmpDir() - - File workingFile = new File("target/tmp/tmp-flow.xml") - workingFile.delete() - Files.copy(flowXmlFile.toPath(), workingFile.toPath()) - ConfigEncryptionTool tool = new ConfigEncryptionTool() - tool.isVerbose = true - tool.flowXmlPath = workingFile.path - tool.outputFlowXmlPath = workingFile.path - - // Use the wrong existing password - String wrongExistingFlowPassword = DEFAULT_LEGACY_SENSITIVE_PROPS_KEY.reverse() - String newFlowPassword = FLOW_PASSWORD - - InputStream xmlContent = new ByteArrayInputStream(workingFile.bytes) - - // Act - def message = shouldFail(BadPaddingException) { - InputStream migratedXmlContent = tool.migrateFlowXmlContent(xmlContent, wrongExistingFlowPassword, newFlowPassword) - logger.info("Migrated flow.xml.") - } - logger.expected(message) - - // Assert - assert message =~ "pad block corrupted" - } - /** * This test is scoped to the higher-level method to ensure that if a bad padding exception is thrown, the right errors are displayed. */