Apache
/
nifi
mirror of https://github.com/apache/nifi.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

NIFI-8230 Removed default Sensitive Properties Key and added random generation 

- Retained legacy default Sensitive Properties Key in ConfigEncryptionTool to support migration
- Streamlined default file path and moved key generation conditional
- Refactored with getDefaultProperties()
- Cleared System Property in ConfigEncryptionToolTest
- Added checking and error handling for clustered status
- Added set-sensitive-properties-key command
- Refactored PropertyEncryptor classes to nifi-property-encryptor
- Added nifi-flow-encryptor
- Refactored ConfigEncryptionTool to use FlowEncryptor for supporting AEAD algorithms
- Added Admin Guide section Updating the Sensitive Properties Key

This closes #4857.

Signed-off-by: Mark Payne <markap14@hotmail.com>
This commit is contained in:
exceptionfactory 2021-02-27 09:17:56 -06:00 committed by Mark Payne
parent a74c17616d
commit 13d5be622b
37 changed files with 1083 additions and 667 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
nifi-bootstrap/pom.xml
View File

@ -40,6 +40,12 @@ language governing permissions and limitations under the License. -->
<artifactId>nifi-security-utils</artifactId>
<version>1.14.0-SNAPSHOT</version>
</dependency>
<dependency>
<groupId>org.apache.nifi</groupId>
<artifactId>nifi-flow-encryptor</artifactId>
<version>1.14.0-SNAPSHOT</version>
<scope>runtime</scope>
</dependency>
<dependency>
<groupId>javax.mail</groupId>
<artifactId>mail</artifactId>

38
nifi-commons/nifi-flow-encryptor/pom.xml Normal file
View File

@ -0,0 +1,38 @@
<?xml version="1.0"?>
<!--
Licensed to the Apache Software Foundation (ASF) under one or more
contributor license agreements. See the NOTICE file distributed with
this work for additional information regarding copyright ownership.
The ASF licenses this file to You under the Apache License, Version 2.0
(the "License"); you may not use this file except in compliance with
the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
-->
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>org.apache.nifi</groupId>
<artifactId>nifi-commons</artifactId>
<version>1.14.0-SNAPSHOT</version>
</parent>
<artifactId>nifi-flow-encryptor</artifactId>
<dependencies>
<dependency>
<groupId>org.apache.nifi</groupId>
<artifactId>nifi-property-encryptor</artifactId>
<version>1.14.0-SNAPSHOT</version>
<exclusions>
<!-- Excluded to avoid unnecessary runtime dependencies -->
<exclusion>
<groupId>org.apache.nifi</groupId>
<artifactId>nifi-properties</artifactId>
</exclusion>
</exclusions>
</dependency>
</dependencies>
</project>

37
nifi-commons/nifi-flow-encryptor/src/main/java/org/apache/nifi/flow/encryptor/FlowEncryptor.java Normal file
View File

@ -0,0 +1,37 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.nifi.flow.encryptor;
import org.apache.nifi.encrypt.PropertyEncryptor;
import java.io.InputStream;
import java.io.OutputStream;
/**
* Flow Encryptor for reading a Flow Configuration and writing a new Flow Configuration using a new password
*/
public interface FlowEncryptor {
/**
* Process Flow Configuration Stream
*
* @param inputStream Flow Configuration Input Stream
* @param outputStream Flow Configuration Output Stream encrypted using new password
* @param inputEncryptor Property Encryptor for Input Configuration
* @param outputEncryptor Property Encryptor for Output Configuration
*/
void processFlow(InputStream inputStream, OutputStream outputStream, PropertyEncryptor inputEncryptor, PropertyEncryptor outputEncryptor);
}

75
nifi-commons/nifi-flow-encryptor/src/main/java/org/apache/nifi/flow/encryptor/StandardFlowEncryptor.java Normal file
View File

@ -0,0 +1,75 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.nifi.flow.encryptor;
import org.apache.nifi.encrypt.PropertyEncryptor;
import java.io.BufferedReader;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.io.OutputStream;
import java.io.OutputStreamWriter;
import java.io.PrintWriter;
import java.io.UncheckedIOException;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
/**
* Standard Flow Encryptor handles reading Input Steam and writing Output Stream
*/
public class StandardFlowEncryptor implements FlowEncryptor {
private static final Pattern ENCRYPTED_PATTERN = Pattern.compile("enc\\{([^\\}]+?)\\}");
private static final int FIRST_GROUP = 1;
private static final String ENCRYPTED_FORMAT = "enc{%s}";
/**
* Process Flow Configuration Stream replacing existing encrypted properties with new encrypted properties
*
* @param inputStream Flow Configuration Input Stream
* @param outputStream Flow Configuration Output Stream encrypted using new password
* @param inputEncryptor Property Encryptor for Input Configuration
* @param outputEncryptor Property Encryptor for Output Configuration
*/
@Override
public void processFlow(final InputStream inputStream, final OutputStream outputStream, final PropertyEncryptor inputEncryptor, final PropertyEncryptor outputEncryptor) {
try (final PrintWriter writer = new PrintWriter(new OutputStreamWriter(outputStream))) {
try (final BufferedReader reader = new BufferedReader(new InputStreamReader(inputStream))) {
reader.lines().forEach(line -> {
final Matcher matcher = ENCRYPTED_PATTERN.matcher(line);
if (matcher.find()) {
final String outputEncrypted = getOutputEncrypted(matcher.group(FIRST_GROUP), inputEncryptor, outputEncryptor);
final String outputLine = matcher.replaceFirst(outputEncrypted);
writer.println(outputLine);
} else {
writer.println(line);
}
});
}
} catch (final IOException e) {
throw new UncheckedIOException("Failed Processing Flow Configuration", e);
}
}
private String getOutputEncrypted(final String inputEncrypted, final PropertyEncryptor inputEncryptor, final PropertyEncryptor outputEncryptor) {
final String inputDecrypted = inputEncryptor.decrypt(inputEncrypted);
final String outputEncrypted = outputEncryptor.encrypt(inputDecrypted);
return String.format(ENCRYPTED_FORMAT, outputEncrypted);
}
}

177
nifi-commons/nifi-flow-encryptor/src/main/java/org/apache/nifi/flow/encryptor/command/SetSensitivePropertiesKey.java Normal file
View File

@ -0,0 +1,177 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.nifi.flow.encryptor.command;
import org.apache.nifi.encrypt.PropertyEncryptor;
import org.apache.nifi.encrypt.PropertyEncryptorBuilder;
import org.apache.nifi.flow.encryptor.FlowEncryptor;
import org.apache.nifi.flow.encryptor.StandardFlowEncryptor;
import org.apache.nifi.security.util.EncryptionMethod;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.FileReader;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.io.UncheckedIOException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.StandardCopyOption;
import java.util.List;
import java.util.Properties;
import java.util.stream.Collectors;
import java.util.zip.GZIPInputStream;
import java.util.zip.GZIPOutputStream;
/**
* Set Sensitive Properties Key for NiFi Properties and update encrypted Flow Configuration
*/
public class SetSensitivePropertiesKey {
protected static final String PROPERTIES_FILE_PATH = "nifi.properties.file.path";
protected static final String PROPS_KEY = "nifi.sensitive.props.key";
protected static final String PROPS_ALGORITHM = "nifi.sensitive.props.algorithm";
protected static final String CONFIGURATION_FILE = "nifi.flow.configuration.file";
private static final int MINIMUM_REQUIRED_LENGTH = 12;
private static final String FLOW_XML_PREFIX = "flow.xml.";
private static final String GZ_EXTENSION = ".gz";
private static final String DEFAULT_PROPERTIES_ALGORITHM = EncryptionMethod.MD5_256AES.getAlgorithm();
private static final String DEFAULT_PROPERTIES_KEY = "nififtw!";
private static final String SENSITIVE_PROPERTIES_KEY = String.format("%s=", PROPS_KEY);
public static void main(final String[] arguments) {
if (arguments.length == 1) {
final String outputPropertiesKey = arguments[0];
if (outputPropertiesKey.length() < MINIMUM_REQUIRED_LENGTH) {
System.err.printf("Sensitive Properties Key length less than required [%d]%n", MINIMUM_REQUIRED_LENGTH);
} else {
run(outputPropertiesKey);
}
} else {
System.err.printf("Unexpected number of arguments [%d]%n", arguments.length);
System.err.printf("Usage: %s <sensitivePropertiesKey>%n", SetSensitivePropertiesKey.class.getSimpleName());
}
}
private static void run(final String outputPropertiesKey) {
final String propertiesFilePath = System.getProperty(PROPERTIES_FILE_PATH);
final File propertiesFile = new File(propertiesFilePath);
final Properties properties = loadProperties(propertiesFile);
final File flowConfigurationFile = getFlowConfigurationFile(properties);
try {
storeProperties(propertiesFile, outputPropertiesKey);
System.out.printf("NiFi Properties Processed [%s]%n", propertiesFilePath);
} catch (final IOException e) {
final String message = String.format("Failed to Process NiFi Properties [%s]", propertiesFilePath);
throw new UncheckedIOException(message, e);
}
if (flowConfigurationFile.exists()) {
final String algorithm = getAlgorithm(properties);
final PropertyEncryptor outputEncryptor = getPropertyEncryptor(outputPropertiesKey, algorithm);
processFlowConfiguration(properties, outputEncryptor);
}
}
private static void processFlowConfiguration(final Properties properties, final PropertyEncryptor outputEncryptor) {
final File flowConfigurationFile = getFlowConfigurationFile(properties);
try (final InputStream flowInputStream = new GZIPInputStream(new FileInputStream(flowConfigurationFile))) {
final File flowOutputFile = getFlowOutputFile();
final Path flowOutputPath = flowOutputFile.toPath();
try (final OutputStream flowOutputStream = new GZIPOutputStream(new FileOutputStream(flowOutputFile))) {
final String inputAlgorithm = getAlgorithm(properties);
final String inputPropertiesKey = getKey(properties);
final PropertyEncryptor inputEncryptor = getPropertyEncryptor(inputPropertiesKey, inputAlgorithm);
final FlowEncryptor flowEncryptor = new StandardFlowEncryptor();
flowEncryptor.processFlow(flowInputStream, flowOutputStream, inputEncryptor, outputEncryptor);
}
final Path flowConfigurationPath = flowConfigurationFile.toPath();
Files.move(flowOutputPath, flowConfigurationPath, StandardCopyOption.REPLACE_EXISTING);
System.out.printf("Flow Configuration Processed [%s]%n", flowConfigurationPath);
} catch (final IOException|RuntimeException e) {
System.err.printf("Failed to process Flow Configuration [%s]%n", flowConfigurationFile);
e.printStackTrace();
}
}
private static String getAlgorithm(final Properties properties) {
String algorithm = properties.getProperty(PROPS_ALGORITHM, DEFAULT_PROPERTIES_ALGORITHM);
if (algorithm.length() == 0) {
algorithm = DEFAULT_PROPERTIES_ALGORITHM;
}
return algorithm;
}
private static String getKey(final Properties properties) {
String key = properties.getProperty(PROPS_KEY, DEFAULT_PROPERTIES_KEY);
if (key.length() == 0) {
key = DEFAULT_PROPERTIES_KEY;
}
return key;
}
private static File getFlowOutputFile() throws IOException {
final File flowOutputFile = File.createTempFile(FLOW_XML_PREFIX, GZ_EXTENSION);
flowOutputFile.deleteOnExit();
return flowOutputFile;
}
private static Properties loadProperties(final File propertiesFile) {
final Properties properties = new Properties();
try (final FileReader reader = new FileReader(propertiesFile)) {
properties.load(reader);
} catch (final IOException e) {
final String message = String.format("Failed to read NiFi Properties [%s]", propertiesFile);
throw new UncheckedIOException(message, e);
}
return properties;
}
private static void storeProperties(final File propertiesFile, final String propertiesKey) throws IOException {
final Path propertiesFilePath = propertiesFile.toPath();
final List<String> lines = Files.readAllLines(propertiesFilePath);
final List<String> updatedLines = lines.stream().map(line -> {
if (line.startsWith(SENSITIVE_PROPERTIES_KEY)) {
return SENSITIVE_PROPERTIES_KEY + propertiesKey;
} else {
return line;
}
}).collect(Collectors.toList());
Files.write(propertiesFilePath, updatedLines);
}
private static PropertyEncryptor getPropertyEncryptor(final String propertiesKey, final String propertiesAlgorithm) {
return new PropertyEncryptorBuilder(propertiesKey).setAlgorithm(propertiesAlgorithm).build();
}
private static File getFlowConfigurationFile(final Properties properties) {
return new File(properties.getProperty(CONFIGURATION_FILE));
}
}

95
nifi-commons/nifi-flow-encryptor/src/test/java/org/apache/nifi/flow/encryptor/StandardFlowEncryptorTest.java Normal file
View File

@ -0,0 +1,95 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.nifi.flow.encryptor;
import org.apache.nifi.encrypt.PropertyEncryptor;
import org.apache.nifi.encrypt.PropertyEncryptorBuilder;
import org.apache.nifi.security.util.EncryptionMethod;
import org.junit.Before;
import org.junit.Test;
import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.InputStream;
import java.nio.charset.StandardCharsets;
import java.util.UUID;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
public class StandardFlowEncryptorTest {
private static final String INPUT_KEY = UUID.randomUUID().toString();
private static final String OUTPUT_KEY = UUID.randomUUID().toString();
private static final String ENCRYPTED_FORMAT = "enc{%s}";
private static final Pattern OUTPUT_PATTERN = Pattern.compile("^enc\\{([^}]+?)}$");
private PropertyEncryptor inputEncryptor;
private PropertyEncryptor outputEncryptor;
private StandardFlowEncryptor flowEncryptor;
@Before
public void setEncryptors() {
inputEncryptor = getPropertyEncryptor(INPUT_KEY, EncryptionMethod.MD5_256AES.getAlgorithm());
outputEncryptor = getPropertyEncryptor(OUTPUT_KEY, EncryptionMethod.SHA256_256AES.getAlgorithm());
flowEncryptor = new StandardFlowEncryptor();
}
@Test
public void testProcessEncrypted() {
final String property = StandardFlowEncryptorTest.class.getSimpleName();
final String encryptedProperty = String.format(ENCRYPTED_FORMAT, inputEncryptor.encrypt(property));
final String encryptedRow = String.format("%s%n", encryptedProperty);
final InputStream inputStream = new ByteArrayInputStream(encryptedRow.getBytes(StandardCharsets.UTF_8));
final ByteArrayOutputStream outputStream = new ByteArrayOutputStream();
flowEncryptor.processFlow(inputStream, outputStream, inputEncryptor, outputEncryptor);
final String outputEncrypted = new String(outputStream.toByteArray());
final Matcher matcher = OUTPUT_PATTERN.matcher(outputEncrypted);
assertTrue(String.format("Encrypted Pattern not found [%s]", outputEncrypted), matcher.find());
final String outputEncryptedProperty = matcher.group(1);
final String outputDecrypted = outputEncryptor.decrypt(outputEncryptedProperty);
assertEquals(property, outputDecrypted);
}
@Test
public void testProcessNoEncrypted() {
final String property = String.format("%s%n", StandardFlowEncryptorTest.class.getSimpleName());
final InputStream inputStream = new ByteArrayInputStream(property.getBytes(StandardCharsets.UTF_8));
final ByteArrayOutputStream outputStream = new ByteArrayOutputStream();
flowEncryptor.processFlow(inputStream, outputStream, inputEncryptor, outputEncryptor);
final String outputProperty = new String(outputStream.toByteArray());
assertEquals(property, outputProperty);
}
private PropertyEncryptor getPropertyEncryptor(final String propertiesKey, final String propertiesAlgorithm) {
return new PropertyEncryptorBuilder(propertiesKey).setAlgorithm(propertiesAlgorithm).build();
}
}

118
nifi-commons/nifi-flow-encryptor/src/test/java/org/apache/nifi/flow/encryptor/command/SetSensitivePropertiesKeyTest.java Normal file
View File

@ -0,0 +1,118 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.nifi.flow.encryptor.command;
import org.apache.nifi.stream.io.GZIPOutputStream;
import org.junit.After;
import org.junit.Test;
import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
import java.net.URISyntaxException;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.List;
import java.util.Optional;
import java.util.UUID;
import java.util.stream.Collectors;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
public class SetSensitivePropertiesKeyTest {
private static final String FLOW_CONTENTS = "<property><value>PROPERTY</value></property>";
@After
public void clearProperties() {
System.clearProperty(SetSensitivePropertiesKey.PROPERTIES_FILE_PATH);
}
@Test
public void testMainNoArguments() {
SetSensitivePropertiesKey.main(new String[]{});
}
@Test
public void testMainBlankKeyAndAlgorithm() throws IOException, URISyntaxException {
final Path flowConfiguration = getFlowConfiguration();
final Path propertiesPath = getNiFiProperties(flowConfiguration, "/blank.nifi.properties");
System.setProperty(SetSensitivePropertiesKey.PROPERTIES_FILE_PATH, propertiesPath.toString());
final String sensitivePropertiesKey = UUID.randomUUID().toString();
SetSensitivePropertiesKey.main(new String[]{sensitivePropertiesKey});
assertPropertiesKeyUpdated(propertiesPath, sensitivePropertiesKey);
assertTrue("Flow Configuration not found", flowConfiguration.toFile().exists());
}
@Test
public void testMainPopulatedKeyAndAlgorithm() throws IOException, URISyntaxException {
final Path flowConfiguration = getFlowConfiguration();
final Path propertiesPath = getNiFiProperties(flowConfiguration, "/populated.nifi.properties");
System.setProperty(SetSensitivePropertiesKey.PROPERTIES_FILE_PATH, propertiesPath.toString());
final String sensitivePropertiesKey = UUID.randomUUID().toString();
SetSensitivePropertiesKey.main(new String[]{sensitivePropertiesKey});
assertPropertiesKeyUpdated(propertiesPath, sensitivePropertiesKey);
assertTrue("Flow Configuration not found", flowConfiguration.toFile().exists());
}
private void assertPropertiesKeyUpdated(final Path propertiesPath, final String sensitivePropertiesKey) throws IOException {
final Optional<String> keyProperty = Files.readAllLines(propertiesPath)
.stream()
.filter(line -> line.startsWith(SetSensitivePropertiesKey.PROPS_KEY))
.findFirst();
assertTrue("Sensitive Key Property not found", keyProperty.isPresent());
final String expectedProperty = String.format("%s=%s", SetSensitivePropertiesKey.PROPS_KEY, sensitivePropertiesKey);
assertEquals("Sensitive Key Property not updated", expectedProperty, keyProperty.get());
}
private Path getNiFiProperties(final Path flowConfigurationPath, String propertiesResource) throws IOException, URISyntaxException {
final Path sourcePropertiesPath = Paths.get(SetSensitivePropertiesKey.class.getResource(propertiesResource).toURI());
final List<String> sourceProperties = Files.readAllLines(sourcePropertiesPath);
final List<String> flowProperties = sourceProperties.stream().map(line -> {
if (line.startsWith(SetSensitivePropertiesKey.CONFIGURATION_FILE)) {
return line + flowConfigurationPath.toString();
} else {
return line;
}
}).collect(Collectors.toList());
final Path propertiesPath = Files.createTempFile(SetSensitivePropertiesKey.class.getSimpleName(), ".properties");
propertiesPath.toFile().deleteOnExit();
Files.write(propertiesPath, flowProperties);
return propertiesPath;
}
private Path getFlowConfiguration() throws IOException {
final Path flowConfigurationPath = Files.createTempFile(SetSensitivePropertiesKey.class.getSimpleName(), ".xml.gz");
final File flowConfigurationFile = flowConfigurationPath.toFile();
flowConfigurationFile.deleteOnExit();
try (final GZIPOutputStream outputStream = new GZIPOutputStream(new FileOutputStream(flowConfigurationFile))) {
outputStream.write(FLOW_CONTENTS.getBytes(StandardCharsets.UTF_8));
}
return flowConfigurationPath;
}
}

17
nifi-commons/nifi-flow-encryptor/src/test/resources/blank.nifi.properties Normal file
View File

@ -0,0 +1,17 @@
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
nifi.sensitive.props.key=
nifi.sensitive.props.algorithm=
nifi.flow.configuration.file=

17
nifi-commons/nifi-flow-encryptor/src/test/resources/populated.nifi.properties Normal file
View File

@ -0,0 +1,17 @@
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
nifi.sensitive.props.key=D5E41AC1-EEF8-4A54-930D-593F749AE95C
nifi.sensitive.props.algorithm=NIFI_ARGON2_AES_GCM_256
nifi.flow.configuration.file=

45
nifi-commons/nifi-property-encryptor/pom.xml Normal file
View File

@ -0,0 +1,45 @@
<?xml version="1.0"?>
<!--
Licensed to the Apache Software Foundation (ASF) under one or more
contributor license agreements. See the NOTICE file distributed with
this work for additional information regarding copyright ownership.
The ASF licenses this file to You under the Apache License, Version 2.0
(the "License"); you may not use this file except in compliance with
the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
-->
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>org.apache.nifi</groupId>
<artifactId>nifi-commons</artifactId>
<version>1.14.0-SNAPSHOT</version>
</parent>
<artifactId>nifi-property-encryptor</artifactId>
<dependencies>
<dependency>
<groupId>org.apache.nifi</groupId>
<artifactId>nifi-security-utils</artifactId>
<version>1.14.0-SNAPSHOT</version>
</dependency>
<dependency>
<groupId>org.bouncycastle</groupId>
<artifactId>bcprov-jdk15on</artifactId>
</dependency>
<dependency>
<groupId>org.apache.commons</groupId>
<artifactId>commons-lang3</artifactId>
<version>3.11</version>
</dependency>
<dependency>
<groupId>commons-codec</groupId>
<artifactId>commons-codec</artifactId>
<version>1.14</version>
</dependency>
</dependencies>
</project>

0
nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/main/java/org/apache/nifi/encrypt/CipherPropertyEncryptor.java → nifi-commons/nifi-property-encryptor/src/main/java/org/apache/nifi/encrypt/CipherPropertyEncryptor.java
View File

0
nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/main/java/org/apache/nifi/encrypt/EncryptionException.java → nifi-commons/nifi-property-encryptor/src/main/java/org/apache/nifi/encrypt/EncryptionException.java
View File

0
nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/main/java/org/apache/nifi/encrypt/KeyedCipherPropertyEncryptor.java → nifi-commons/nifi-property-encryptor/src/main/java/org/apache/nifi/encrypt/KeyedCipherPropertyEncryptor.java
View File

0
nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/main/java/org/apache/nifi/encrypt/PasswordBasedCipherPropertyEncryptor.java → nifi-commons/nifi-property-encryptor/src/main/java/org/apache/nifi/encrypt/PasswordBasedCipherPropertyEncryptor.java
View File

0
nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/main/java/org/apache/nifi/encrypt/PropertyEncryptionMethod.java → nifi-commons/nifi-property-encryptor/src/main/java/org/apache/nifi/encrypt/PropertyEncryptionMethod.java
View File

0
nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/main/java/org/apache/nifi/encrypt/PropertyEncryptor.java → nifi-commons/nifi-property-encryptor/src/main/java/org/apache/nifi/encrypt/PropertyEncryptor.java
View File

89
nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/main/java/org/apache/nifi/encrypt/PropertyEncryptorFactory.java → nifi-commons/nifi-property-encryptor/src/main/java/org/apache/nifi/encrypt/PropertyEncryptorBuilder.java
View File

@ -16,70 +16,55 @@
*/
package org.apache.nifi.encrypt;
import org.apache.commons.lang3.StringUtils;
import org.apache.nifi.security.util.EncryptionMethod;
import org.apache.nifi.security.util.crypto.AESKeyedCipherProvider;
import org.apache.nifi.security.util.crypto.KeyedCipherProvider;
import org.apache.nifi.security.util.crypto.PBECipherProvider;
import org.apache.nifi.util.NiFiProperties;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import javax.crypto.SecretKey;
import java.util.Objects;
/**
* Property Encryptor Factory for encapsulating instantiation of Property Encryptors based on various parameters
* Property Encryptor Builder
*/
public class PropertyEncryptorFactory {
private static final Logger LOGGER = LoggerFactory.getLogger(PropertyEncryptorFactory.class);
public class PropertyEncryptorBuilder {
private static final PropertySecretKeyProvider SECRET_KEY_PROVIDER = new StandardPropertySecretKeyProvider();
private static final String DEFAULT_PASSWORD = "nififtw!";
private final String password;
private static final String NOTIFICATION_BORDER = "*";
private static final int NOTIFICATION_WIDTH = 80;
private static final String NOTIFICATION_DELIMITER = StringUtils.repeat(NOTIFICATION_BORDER, NOTIFICATION_WIDTH);
private static final String NOTIFICATION = StringUtils.joinWith(System.lineSeparator(),
System.lineSeparator(),
NOTIFICATION_DELIMITER,
StringUtils.center(String.format("FOUND BLANK SENSITIVE PROPERTIES KEY [%s]", NiFiProperties.SENSITIVE_PROPS_KEY), NOTIFICATION_WIDTH),
StringUtils.center("USING DEFAULT KEY FOR ENCRYPTION", NOTIFICATION_WIDTH),
StringUtils.center(String.format("SET [%s] TO SECURE SENSITIVE PROPERTIES", NiFiProperties.SENSITIVE_PROPS_KEY), NOTIFICATION_WIDTH),
NOTIFICATION_DELIMITER
);
private String algorithm = PropertyEncryptionMethod.NIFI_ARGON2_AES_GCM_256.toString();
/**
* Get Property Encryptor using NiFi Properties
* Property Encryptor Builder with required password
*
* @param password Password required
*/
public PropertyEncryptorBuilder(final String password) {
Objects.requireNonNull(password, "Password required");
this.password = password;
}
/**
* Set Algorithm as either Property Encryption Method or Encryption Method
*
* @param algorithm Algorithm
* @return Property Encryptor Builder
*/
public PropertyEncryptorBuilder setAlgorithm(final String algorithm) {
Objects.requireNonNull(algorithm, "Algorithm required");
this.algorithm = algorithm;
return this;
}
/**
* Build Property Encryptor using current configuration
*
* @param properties NiFi Properties
* @return Property Encryptor
*/
@SuppressWarnings("deprecation")
public static PropertyEncryptor getPropertyEncryptor(final NiFiProperties properties) {
Objects.requireNonNull(properties, "NiFi Properties is required");
final String algorithm = properties.getProperty(NiFiProperties.SENSITIVE_PROPS_ALGORITHM);
String password = properties.getProperty(NiFiProperties.SENSITIVE_PROPS_KEY);
if (StringUtils.isBlank(password)) {
LOGGER.error(NOTIFICATION);
password = DEFAULT_PASSWORD;
}
public PropertyEncryptor build() {
final PropertyEncryptionMethod propertyEncryptionMethod = findPropertyEncryptionAlgorithm(algorithm);
if (propertyEncryptionMethod == null) {
final EncryptionMethod encryptionMethod = findEncryptionMethod(algorithm);
if (encryptionMethod.isPBECipher()) {
final PBECipherProvider cipherProvider = new org.apache.nifi.security.util.crypto.NiFiLegacyCipherProvider();
return new PasswordBasedCipherPropertyEncryptor(cipherProvider, encryptionMethod, password);
} else {
final String message = String.format("Algorithm [%s] not supported for Sensitive Properties", encryptionMethod.getAlgorithm());
throw new UnsupportedOperationException(message);
}
return getPasswordBasedCipherPropertyEncryptor();
} else {
final KeyedCipherProvider keyedCipherProvider = new AESKeyedCipherProvider();
final SecretKey secretKey = SECRET_KEY_PROVIDER.getSecretKey(propertyEncryptionMethod, password);
@ -88,7 +73,19 @@ public class PropertyEncryptorFactory {
}
}
private static PropertyEncryptionMethod findPropertyEncryptionAlgorithm(final String algorithm) {
@SuppressWarnings("deprecation")
private PasswordBasedCipherPropertyEncryptor getPasswordBasedCipherPropertyEncryptor() {
final EncryptionMethod encryptionMethod = findEncryptionMethod(algorithm);
if (encryptionMethod.isPBECipher()) {
final PBECipherProvider cipherProvider = new org.apache.nifi.security.util.crypto.NiFiLegacyCipherProvider();
return new PasswordBasedCipherPropertyEncryptor(cipherProvider, encryptionMethod, password);
} else {
final String message = String.format("Algorithm [%s] not supported for Sensitive Properties", encryptionMethod.getAlgorithm());
throw new UnsupportedOperationException(message);
}
}
private PropertyEncryptionMethod findPropertyEncryptionAlgorithm(final String algorithm) {
PropertyEncryptionMethod foundPropertyEncryptionMethod = null;
for (final PropertyEncryptionMethod propertyEncryptionMethod : PropertyEncryptionMethod.values()) {
@ -101,7 +98,7 @@ public class PropertyEncryptorFactory {
return foundPropertyEncryptionMethod;
}
private static EncryptionMethod findEncryptionMethod(final String algorithm) {
private EncryptionMethod findEncryptionMethod(final String algorithm) {
final EncryptionMethod encryptionMethod = EncryptionMethod.forAlgorithm(algorithm);
if (encryptionMethod == null) {
final String message = String.format("Encryption Method not found for Algorithm [%s]", algorithm);

47
nifi-commons/nifi-property-encryptor/src/main/java/org/apache/nifi/encrypt/PropertyEncryptorFactory.java Normal file
View File

@ -0,0 +1,47 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.nifi.encrypt;
import org.apache.commons.lang3.StringUtils;
import org.apache.nifi.util.NiFiProperties;
import java.util.Objects;
/**
* Property Encryptor Factory for encapsulating instantiation of Property Encryptors based on various parameters
*/
public class PropertyEncryptorFactory {
private static final String KEY_REQUIRED = String.format("NiFi Sensitive Properties Key [%s] is required", NiFiProperties.SENSITIVE_PROPS_KEY);
/**
* Get Property Encryptor using NiFi Properties
*
* @param properties NiFi Properties
* @return Property Encryptor
*/
public static PropertyEncryptor getPropertyEncryptor(final NiFiProperties properties) {
Objects.requireNonNull(properties, "NiFi Properties is required");
final String algorithm = properties.getProperty(NiFiProperties.SENSITIVE_PROPS_ALGORITHM);
String password = properties.getProperty(NiFiProperties.SENSITIVE_PROPS_KEY);
if (StringUtils.isBlank(password)) {
throw new IllegalArgumentException(KEY_REQUIRED);
}
return new PropertyEncryptorBuilder(password).setAlgorithm(algorithm).build();
}
}

0
nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/main/java/org/apache/nifi/encrypt/PropertySecretKeyProvider.java → nifi-commons/nifi-property-encryptor/src/main/java/org/apache/nifi/encrypt/PropertySecretKeyProvider.java
View File

0
nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/main/java/org/apache/nifi/encrypt/StandardPropertySecretKeyProvider.java → nifi-commons/nifi-property-encryptor/src/main/java/org/apache/nifi/encrypt/StandardPropertySecretKeyProvider.java
View File

0
nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/test/java/org/apache/nifi/encrypt/KeyedCipherPropertyEncryptorTest.java → nifi-commons/nifi-property-encryptor/src/test/java/org/apache/nifi/encrypt/KeyedCipherPropertyEncryptorTest.java
View File

0
nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/test/java/org/apache/nifi/encrypt/PasswordBasedCipherPropertyEncryptorTest.java → nifi-commons/nifi-property-encryptor/src/test/java/org/apache/nifi/encrypt/PasswordBasedCipherPropertyEncryptorTest.java
View File

3
nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/test/java/org/apache/nifi/encrypt/PropertyEncryptorFactoryTest.java → nifi-commons/nifi-property-encryptor/src/test/java/org/apache/nifi/encrypt/PropertyEncryptorFactoryTest.java
View File

@ -49,8 +49,7 @@ public class PropertyEncryptorFactoryTest {
properties.setProperty(NiFiProperties.SENSITIVE_PROPS_KEY, StringUtils.EMPTY);
final NiFiProperties niFiProperties = NiFiProperties.createBasicNiFiProperties(null, properties);
final PropertyEncryptor encryptor = PropertyEncryptorFactory.getPropertyEncryptor(niFiProperties);
assertNotNull(encryptor);
assertThrows(IllegalArgumentException.class, () -> PropertyEncryptorFactory.getPropertyEncryptor(niFiProperties));
}
@Test

0
nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/test/java/org/apache/nifi/encrypt/StandardPropertySecretKeyProviderTest.java → nifi-commons/nifi-property-encryptor/src/test/java/org/apache/nifi/encrypt/StandardPropertySecretKeyProviderTest.java
View File

2
nifi-commons/pom.xml
View File

@ -27,11 +27,13 @@
<module>nifi-data-provenance-utils</module>
<module>nifi-expression-language</module>
<module>nifi-flowfile-packager</module>
<module>nifi-flow-encryptor</module>
<module>nifi-hl7-query-language</module>
<module>nifi-json-utils</module>
<module>nifi-logging-utils</module>
<module>nifi-metrics</module>
<module>nifi-parameter</module>
<module>nifi-property-encryptor</module>
<module>nifi-properties</module>
<module>nifi-record</module>
<module>nifi-record-path</module>

12
nifi-docs/src/main/asciidoc/administration-guide.adoc
View File

@ -3926,6 +3926,18 @@ where:
For more information see the <<toolkit-guide.adoc#encrypt_config_tool,Encrypt-Config Tool>> section in the NiFi Toolkit Guide.
==== Updating the Sensitive Properties Key
Starting with version 1.14.0, NiFi requires a value for 'nifi.sensitive.props.key' in _nifi.properties_.
The following command can be used to read an existing _flow.xml.gz_ configuration and set a new sensitive properties key in _nifi.properties_:
```
$ ./bin/nifi.sh set-sensitive-properties-key <sensitivePropertiesKey>
```
The minimum required length for a new sensitive properties key is 12 characters.
=== Start New NiFi
In your new NiFi installation:

5
nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-components/pom.xml
View File

@ -66,6 +66,11 @@
<artifactId>nifi-nar-utils</artifactId>
<version>1.14.0-SNAPSHOT</version>
</dependency>
<dependency>
<groupId>org.apache.nifi</groupId>
<artifactId>nifi-property-encryptor</artifactId>
<version>1.14.0-SNAPSHOT</version>
</dependency>
<dependency>
<groupId>org.apache.nifi.registry</groupId>

18
nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/groovy/org/apache/nifi/controller/serialization/FlowFromDOMFactoryTest.groovy
View File

@ -16,13 +16,8 @@
*/
package org.apache.nifi.controller.serialization
import org.apache.commons.codec.binary.Hex
import org.apache.nifi.encrypt.EncryptionException
import org.apache.nifi.encrypt.PropertyEncryptor
import org.apache.nifi.encrypt.PropertyEncryptorFactory
import org.apache.nifi.security.kms.CryptoUtils
import org.apache.nifi.security.util.EncryptionMethod
import org.bouncycastle.jce.provider.BouncyCastleProvider
import org.junit.BeforeClass
import org.junit.Test
import org.junit.runner.RunWith
@ -30,27 +25,14 @@ import org.junit.runners.JUnit4
import org.slf4j.Logger
import org.slf4j.LoggerFactory
import javax.crypto.Cipher
import javax.crypto.SecretKey
import javax.crypto.SecretKeyFactory
import javax.crypto.spec.PBEKeySpec
import javax.crypto.spec.PBEParameterSpec
import java.security.Security
import static groovy.test.GroovyAssert.shouldFail
@RunWith(JUnit4.class)
class FlowFromDOMFactoryTest {
private static final Logger logger = LoggerFactory.getLogger(FlowFromDOMFactoryTest.class)
private static final String DEFAULT_PASSWORD = "nififtw!"
private static final byte[] DEFAULT_SALT = new byte[8]
private static final int DEFAULT_ITERATION_COUNT = 0
private static final EncryptionMethod DEFAULT_ENCRYPTION_METHOD = EncryptionMethod.MD5_128AES
@BeforeClass
static void setUpOnce() throws Exception {
Security.addProvider(new BouncyCastleProvider())
logger.metaClass.methodMissing = { String name, args ->
logger.info("[${name?.toUpperCase()}] ${(args as List).join(" ")}")
}

98
nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-properties-loader/src/main/java/org/apache/nifi/properties/NiFiPropertiesLoader.java
View File

@ -21,12 +21,19 @@ import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.UncheckedIOException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.security.NoSuchAlgorithmException;
import java.security.SecureRandom;
import java.security.Security;
import java.util.Base64;
import java.util.List;
import java.util.Properties;
import java.util.stream.Collectors;
import java.util.Set;
import javax.crypto.Cipher;
import org.apache.commons.lang3.StringUtils;
import org.apache.nifi.security.kms.CryptoUtils;
import org.apache.nifi.util.NiFiProperties;
import org.bouncycastle.jce.provider.BouncyCastleProvider;
@ -36,7 +43,13 @@ import org.slf4j.LoggerFactory;
public class NiFiPropertiesLoader {
private static final Logger logger = LoggerFactory.getLogger(NiFiPropertiesLoader.class);
private static final Base64.Encoder KEY_ENCODER = Base64.getEncoder().withoutPadding();
private static final int SENSITIVE_PROPERTIES_KEY_LENGTH = 24;
private static final String EMPTY_SENSITIVE_PROPERTIES_KEY = String.format("%s=", NiFiProperties.SENSITIVE_PROPS_KEY);
private static final String MIGRATION_INSTRUCTIONS = "See Admin Guide section [Updating the Sensitive Properties Key]";
private static final String PROPERTIES_KEY_MESSAGE = String.format("Sensitive Properties Key [%s] not found: %s", NiFiProperties.SENSITIVE_PROPS_KEY, MIGRATION_INSTRUCTIONS);
private final String defaultPropertiesFilePath = CryptoUtils.getDefaultFilePath();
private NiFiProperties instance;
private String keyHex;
@ -132,7 +145,7 @@ public class NiFiPropertiesLoader {
}
private NiFiProperties loadDefault() {
return load(CryptoUtils.getDefaultFilePath());
return load(defaultPropertiesFilePath);
}
static String getDefaultProviderKey() {
@ -168,36 +181,22 @@ public class NiFiPropertiesLoader {
throw new IllegalArgumentException("NiFi properties file missing or unreadable");
}
Properties rawProperties = new Properties();
InputStream inStream = null;
try {
inStream = new BufferedInputStream(new FileInputStream(file));
rawProperties.load(inStream);
final Properties rawProperties = new Properties();
try (final InputStream inputStream = new BufferedInputStream(new FileInputStream(file))) {
rawProperties.load(inputStream);
logger.info("Loaded {} properties from {}", rawProperties.size(), file.getAbsolutePath());
Set<String> keys = rawProperties.stringPropertyNames();
final Set<String> keys = rawProperties.stringPropertyNames();
for (final String key : keys) {
String prop = rawProperties.getProperty(key);
rawProperties.setProperty(key, StringUtils.stripEnd(prop, null));
final String property = rawProperties.getProperty(key);
rawProperties.setProperty(key, property.trim());
}
ProtectedNiFiProperties protectedNiFiProperties = new ProtectedNiFiProperties(rawProperties);
return protectedNiFiProperties;
return new ProtectedNiFiProperties(rawProperties);
} catch (final Exception ex) {
logger.error("Cannot load properties file due to " + ex.getLocalizedMessage());
logger.error("Cannot load properties file due to {}", ex.getLocalizedMessage());
throw new RuntimeException("Cannot load properties file due to "
+ ex.getLocalizedMessage(), ex);
} finally {
if (null != inStream) {
try {
inStream.close();
} catch (final Exception ex) {
/**
* do nothing *
*/
}
}
}
}
@ -249,9 +248,58 @@ public class NiFiPropertiesLoader {
*/
public NiFiProperties get() {
if (instance == null) {
instance = loadDefault();
instance = getDefaultProperties();
}
return instance;
}
private NiFiProperties getDefaultProperties() {
NiFiProperties defaultProperties = loadDefault();
if (isKeyGenerationRequired(defaultProperties)) {
if (defaultProperties.isClustered()) {
logger.error("Clustered Configuration Found: Shared Sensitive Properties Key [{}] required for cluster nodes", NiFiProperties.SENSITIVE_PROPS_KEY);
throw new SensitivePropertyProtectionException(PROPERTIES_KEY_MESSAGE);
}
final File flowConfiguration = defaultProperties.getFlowConfigurationFile();
if (flowConfiguration.exists()) {
logger.error("Flow Configuration [{}] Found: Migration Required for blank Sensitive Properties Key [{}]", flowConfiguration, NiFiProperties.SENSITIVE_PROPS_KEY);
throw new SensitivePropertyProtectionException(PROPERTIES_KEY_MESSAGE);
}
setSensitivePropertiesKey();
defaultProperties = loadDefault();
}
return defaultProperties;
}
private void setSensitivePropertiesKey() {
logger.warn("Generating Random Sensitive Properties Key [{}]", NiFiProperties.SENSITIVE_PROPS_KEY);
final SecureRandom secureRandom = new SecureRandom();
final byte[] sensitivePropertiesKeyBinary = new byte[SENSITIVE_PROPERTIES_KEY_LENGTH];
secureRandom.nextBytes(sensitivePropertiesKeyBinary);
final String sensitivePropertiesKey = KEY_ENCODER.encodeToString(sensitivePropertiesKeyBinary);
try {
final File niFiPropertiesFile = new File(defaultPropertiesFilePath);
final Path niFiPropertiesPath = Paths.get(niFiPropertiesFile.toURI());
final List<String> lines = Files.readAllLines(niFiPropertiesPath);
final List<String> updatedLines = lines.stream().map(line -> {
if (line.equals(EMPTY_SENSITIVE_PROPERTIES_KEY)) {
return line + sensitivePropertiesKey;
} else {
return line;
}
}).collect(Collectors.toList());
Files.write(niFiPropertiesPath, updatedLines);
logger.info("NiFi Properties [{}] updated with Sensitive Properties Key", niFiPropertiesPath);
} catch (final IOException e) {
throw new UncheckedIOException("Failed to set Sensitive Properties Key", e);
}
}
private static boolean isKeyGenerationRequired(final NiFiProperties properties) {
final String configuredSensitivePropertiesKey = properties.getProperty(NiFiProperties.SENSITIVE_PROPS_KEY);
return (configuredSensitivePropertiesKey == null || configuredSensitivePropertiesKey.length() == 0);
}
}

2
nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-properties-loader/src/test/groovy/org/apache/nifi/properties/AESSensitivePropertyProviderTest.groovy
View File

@ -447,7 +447,7 @@ class AESSensitivePropertyProviderTest extends GroovyTestCase {
@Test
void testShouldEncryptArbitraryValues() {
// Arrange
def values = ["thisIsABadPassword", "thisIsABadSensitiveKeyPassword", "thisIsABadKeystorePassword", "thisIsABadKeyPassword", "thisIsABadTruststorePassword", "This is an encrypted banner message", "nififtw!"]
def values = ["thisIsABadPassword", "thisIsABadSensitiveKeyPassword", "thisIsABadKeystorePassword", "thisIsABadKeyPassword", "thisIsABadTruststorePassword", "This is an encrypted banner message"]
String key = "2C576A9585DB862F5ECBEE5B4FFFCCA1" //getKeyOfSize(128)
// key = "0" * 64

39
nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-properties-loader/src/test/groovy/org/apache/nifi/properties/NiFiPropertiesLoaderGroovyTest.groovy
View File

@ -18,6 +18,7 @@ package org.apache.nifi.properties
import org.apache.commons.lang3.SystemUtils
import org.apache.nifi.util.NiFiProperties
import org.apache.nifi.util.file.FileUtils
import org.bouncycastle.jce.provider.BouncyCastleProvider
import org.junit.After
import org.junit.AfterClass
@ -201,6 +202,44 @@ class NiFiPropertiesLoaderGroovyTest extends GroovyTestCase {
assert msg =~ "Cannot read from bootstrap.conf"
}
@Test
void testShouldLoadUnprotectedPropertiesFromPathWithGeneratedSensitivePropertiesKey() throws Exception {
// Arrange
final File propertiesFile = File.createTempFile("nifi.without.key", ".properties")
propertiesFile.deleteOnExit()
final OutputStream outputStream = new FileOutputStream(propertiesFile)
final InputStream inputStream = getClass().getResourceAsStream("/conf/nifi.without.key.properties")
FileUtils.copy(inputStream, outputStream)
System.setProperty(NiFiProperties.PROPERTIES_FILE_PATH, propertiesFile.absolutePath);
NiFiPropertiesLoader niFiPropertiesLoader = new NiFiPropertiesLoader()
// Act
NiFiProperties niFiProperties = niFiPropertiesLoader.get()
// Assert
final String sensitivePropertiesKey = niFiProperties.getProperty(NiFiProperties.SENSITIVE_PROPS_KEY)
assert sensitivePropertiesKey.length() == 32
}
@Test
void testShouldNotLoadUnprotectedPropertiesFromPathWithBlankKeyForClusterNode() throws Exception {
// Arrange
final File propertiesFile = File.createTempFile("nifi.without.key", ".properties")
propertiesFile.deleteOnExit()
final OutputStream outputStream = new FileOutputStream(propertiesFile)
final InputStream inputStream = getClass().getResourceAsStream("/conf/nifi.cluster.without.key.properties")
FileUtils.copy(inputStream, outputStream)
System.setProperty(NiFiProperties.PROPERTIES_FILE_PATH, propertiesFile.absolutePath);
NiFiPropertiesLoader niFiPropertiesLoader = new NiFiPropertiesLoader()
// Act
shouldFail(SensitivePropertyProtectionException) {
niFiPropertiesLoader.get()
}
}
@Test
void testShouldNotLoadUnprotectedPropertiesFromNullFile() throws Exception {
// Arrange

95
nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-properties-loader/src/test/resources/conf/nifi.cluster.without.key.properties Normal file
View File

@ -0,0 +1,95 @@
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# Core Properties #
nifi.flow.configuration.file=./target/flow.xml.gz
nifi.flow.configuration.archive.dir=./target/archive/
nifi.flowcontroller.autoResumeState=true
nifi.flowcontroller.graceful.shutdown.period=10 sec
nifi.flowservice.writedelay.interval=2 sec
nifi.administrative.yield.duration=30 sec
nifi.reporting.task.configuration.file=./target/reporting-tasks.xml
nifi.controller.service.configuration.file=./target/controller-services.xml
nifi.templates.directory=./target/templates
nifi.ui.banner.text=UI Banner Text
nifi.ui.autorefresh.interval=30 sec
nifi.nar.library.directory=./target/resources/NiFiProperties/lib/
nifi.nar.library.directory.alt=./target/resources/NiFiProperties/lib2/
nifi.nar.working.directory=./target/work/nar/
# H2 Settings
nifi.database.directory=./target/database_repository
nifi.h2.url.append=;LOCK_TIMEOUT=25000;WRITE_DELAY=0;AUTO_SERVER=FALSE
# FlowFile Repository
nifi.flowfile.repository.directory=./target/test-repo
nifi.flowfile.repository.partitions=1
nifi.flowfile.repository.checkpoint.interval=2 mins
nifi.queue.swap.threshold=20000
nifi.swap.storage.directory=./target/test-repo/swap
nifi.swap.in.period=5 sec
nifi.swap.in.threads=1
nifi.swap.out.period=5 sec
nifi.swap.out.threads=4
# Content Repository
nifi.content.claim.max.appendable.size=10 MB
nifi.content.claim.max.flow.files=100
nifi.content.repository.directory.default=./target/content_repository
# Provenance Repository Properties
nifi.provenance.repository.storage.directory=./target/provenance_repository
nifi.provenance.repository.max.storage.time=24 hours
nifi.provenance.repository.max.storage.size=1 GB
nifi.provenance.repository.rollover.time=30 secs
nifi.provenance.repository.rollover.size=100 MB
# Site to Site properties
nifi.remote.input.socket.port=9990
nifi.remote.input.secure=true
# web properties #
nifi.web.war.directory=./target/lib
nifi.web.http.host=
nifi.web.http.port=8080
nifi.web.https.host=
nifi.web.https.port=
nifi.web.jetty.working.directory=./target/work/jetty
# security properties #
nifi.sensitive.props.key=
nifi.sensitive.props.algorithm=PBEWITHMD5AND256BITAES-CBC-OPENSSL
nifi.sensitive.props.provider=BC
nifi.security.keystore=
nifi.security.keystoreType=
nifi.security.keystorePasswd=
nifi.security.keyPasswd=
nifi.security.truststore=
nifi.security.truststoreType=
nifi.security.truststorePasswd=
nifi.security.user.authorizer=
# cluster common properties (cluster manager and nodes must have same values) #
nifi.cluster.protocol.heartbeat.interval=5 sec
nifi.cluster.protocol.is.secure=false
nifi.cluster.protocol.socket.timeout=30 sec
# cluster node properties (only configure for cluster nodes) #
nifi.cluster.is.node=true
nifi.cluster.node.address=
nifi.cluster.node.protocol.port=
nifi.cluster.node.protocol.threads=2

95
nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-properties-loader/src/test/resources/conf/nifi.without.key.properties Normal file
View File

@ -0,0 +1,95 @@
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# Core Properties #
nifi.flow.configuration.file=./target/flow.xml.gz
nifi.flow.configuration.archive.dir=./target/archive/
nifi.flowcontroller.autoResumeState=true
nifi.flowcontroller.graceful.shutdown.period=10 sec
nifi.flowservice.writedelay.interval=2 sec
nifi.administrative.yield.duration=30 sec
nifi.reporting.task.configuration.file=./target/reporting-tasks.xml
nifi.controller.service.configuration.file=./target/controller-services.xml
nifi.templates.directory=./target/templates
nifi.ui.banner.text=UI Banner Text
nifi.ui.autorefresh.interval=30 sec
nifi.nar.library.directory=./target/resources/NiFiProperties/lib/
nifi.nar.library.directory.alt=./target/resources/NiFiProperties/lib2/
nifi.nar.working.directory=./target/work/nar/
# H2 Settings
nifi.database.directory=./target/database_repository
nifi.h2.url.append=;LOCK_TIMEOUT=25000;WRITE_DELAY=0;AUTO_SERVER=FALSE
# FlowFile Repository
nifi.flowfile.repository.directory=./target/test-repo
nifi.flowfile.repository.partitions=1
nifi.flowfile.repository.checkpoint.interval=2 mins
nifi.queue.swap.threshold=20000
nifi.swap.storage.directory=./target/test-repo/swap
nifi.swap.in.period=5 sec
nifi.swap.in.threads=1
nifi.swap.out.period=5 sec
nifi.swap.out.threads=4
# Content Repository
nifi.content.claim.max.appendable.size=10 MB
nifi.content.claim.max.flow.files=100
nifi.content.repository.directory.default=./target/content_repository
# Provenance Repository Properties
nifi.provenance.repository.storage.directory=./target/provenance_repository
nifi.provenance.repository.max.storage.time=24 hours
nifi.provenance.repository.max.storage.size=1 GB
nifi.provenance.repository.rollover.time=30 secs
nifi.provenance.repository.rollover.size=100 MB
# Site to Site properties
nifi.remote.input.socket.port=9990
nifi.remote.input.secure=true
# web properties #
nifi.web.war.directory=./target/lib
nifi.web.http.host=
nifi.web.http.port=8080
nifi.web.https.host=
nifi.web.https.port=
nifi.web.jetty.working.directory=./target/work/jetty
# security properties #
nifi.sensitive.props.key=
nifi.sensitive.props.algorithm=PBEWITHMD5AND256BITAES-CBC-OPENSSL
nifi.sensitive.props.provider=BC
nifi.security.keystore=
nifi.security.keystoreType=
nifi.security.keystorePasswd=
nifi.security.keyPasswd=
nifi.security.truststore=
nifi.security.truststoreType=
nifi.security.truststorePasswd=
nifi.security.user.authorizer=
# cluster common properties (cluster manager and nodes must have same values) #
nifi.cluster.protocol.heartbeat.interval=5 sec
nifi.cluster.protocol.is.secure=false
nifi.cluster.protocol.socket.timeout=30 sec
# cluster node properties (only configure for cluster nodes) #
nifi.cluster.is.node=false
nifi.cluster.node.address=
nifi.cluster.node.protocol.port=
nifi.cluster.node.protocol.threads=2

14
nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-resources/src/main/resources/bin/nifi.sh
View File

@ -334,6 +334,16 @@ run() {
run_nifi_cmd="exec ${run_nifi_cmd}"
fi
if [ "$1" = "set-sensitive-properties-key" ]; then
run_command="'${JAVA}' -cp '${BOOTSTRAP_CLASSPATH}' '-Dnifi.properties.file.path=${NIFI_HOME}/conf/nifi.properties' 'org.apache.nifi.flow.encryptor.command.SetSensitivePropertiesKey'"
eval "cd ${NIFI_HOME}"
shift
eval "${run_command}" '"$@"'
EXIT_STATUS=$?
echo
return;
fi
if [ "$1" = "stateless" ]; then
STATELESS_JAVA_OPTS="${STATELESS_JAVA_OPTS:=-Xms1024m -Xmx1024m}"
@ -430,7 +440,7 @@ case "$1" in
install "$@"
;;
start|stop|decommission|run|status|is_loaded|dump|diagnostics|env|stateless)
start|stop|decommission|run|status|is_loaded|dump|diagnostics|env|stateless|set-sensitive-properties-key)
main "$@"
;;
@ -440,6 +450,6 @@ case "$1" in
run "start"
;;
*)
echo "Usage nifi {start|stop|decommission|run|restart|status|dump|diagnostics|install|stateless}"
echo "Usage nifi {start|stop|decommission|run|restart|status|dump|diagnostics|install|stateless|set-sensitive-properties-key}"
;;
esac

5
nifi-toolkit/nifi-toolkit-encrypt-config/pom.xml
View File

@ -24,6 +24,11 @@
<artifactId>nifi-toolkit-encrypt-config</artifactId>
<description>Tool to encrypt sensitive configuration values</description>
<dependencies>
<dependency>
<groupId>org.apache.nifi</groupId>
<artifactId>nifi-flow-encryptor</artifactId>
<version>1.14.0-SNAPSHOT</version>
</dependency>
<dependency>
<groupId>org.apache.nifi</groupId>
<artifactId>nifi-properties</artifactId>

219
nifi-toolkit/nifi-toolkit-encrypt-config/src/main/groovy/org/apache/nifi/properties/ConfigEncryptionTool.groovy
View File

@ -27,6 +27,10 @@ import org.apache.commons.cli.Option
import org.apache.commons.cli.Options
import org.apache.commons.cli.ParseException
import org.apache.commons.codec.binary.Hex
import org.apache.nifi.encrypt.PropertyEncryptor
import org.apache.nifi.encrypt.PropertyEncryptorFactory
import org.apache.nifi.flow.encryptor.FlowEncryptor
import org.apache.nifi.flow.encryptor.StandardFlowEncryptor
import org.apache.nifi.security.kms.CryptoUtils
import org.apache.nifi.toolkit.tls.commandLine.CommandLineParseException
import org.apache.nifi.toolkit.tls.commandLine.ExitCode
@ -41,22 +45,17 @@ import org.xml.sax.SAXException
import javax.crypto.BadPaddingException
import javax.crypto.Cipher
import javax.crypto.SecretKey
import javax.crypto.SecretKeyFactory
import javax.crypto.spec.PBEKeySpec
import javax.crypto.spec.PBEParameterSpec
import java.nio.charset.StandardCharsets
import java.nio.file.Path
import java.nio.file.Paths
import java.nio.file.StandardCopyOption;
import java.nio.file.StandardCopyOption
import java.security.KeyException
import java.security.SecureRandom
import java.security.Security
import java.util.regex.Matcher
import java.util.zip.GZIPInputStream
import java.util.zip.GZIPOutputStream
import java.util.zip.ZipException
import java.nio.file.Files;
import java.nio.file.Files
class ConfigEncryptionTool {
private static final Logger logger = LoggerFactory.getLogger(ConfigEncryptionTool.class)
@ -69,7 +68,7 @@ class ConfigEncryptionTool {
public String authorizersPath
public String outputAuthorizersPath
public static flowXmlPath
public outputFlowXmlPath
public String outputFlowXmlPath
private String keyHex
private String migrationKeyHex
@ -124,7 +123,7 @@ class ConfigEncryptionTool {
// Static holder to avoid re-generating the options object multiple times in an invocation
private static Options staticOptions
// Hard-coded fallback value from {@link org.apache.nifi.encrypt.PropertyEncryptorFactory}
// Hard-coded fallback value from historical defaults
private static final String DEFAULT_NIFI_SENSITIVE_PROPS_KEY = "nififtw!"
private static final int MIN_PASSWORD_LENGTH = 12
@ -133,11 +132,6 @@ class ConfigEncryptionTool {
private static final int SCRYPT_N = 2**16
private static final int SCRYPT_R = 8
private static final int SCRYPT_P = 1
static final String CURRENT_SCRYPT_VERSION = "s0"
// Hard-coded values from StandardPBEByteEncryptor which will be removed during refactor of all flow encryption code in NIFI-1465
private static final int DEFAULT_KDF_ITERATIONS = 1000
private static final int DEFAULT_SALT_SIZE_BYTES = 16
private static
final String BOOTSTRAP_KEY_COMMENT = "# Root key in hexadecimal format for encrypted sensitive configuration values"
@ -191,7 +185,6 @@ class ConfigEncryptionTool {
private static final String XML_DECLARATION_REGEX = /<\?xml version="1.0" encoding="UTF-8"\?>/
private static final String WRAPPED_FLOW_XML_CIPHER_TEXT_REGEX = /enc\{[a-fA-F0-9]+?\}/
private static final String DEFAULT_PROVIDER = BouncyCastleProvider.PROVIDER_NAME
private static final String DEFAULT_FLOW_ALGORITHM = "PBEWITHMD5AND256BITAES-CBC-OPENSSL"
private static final Map<String, String> PROPERTY_KEY_MAP = [
@ -682,113 +675,6 @@ class ConfigEncryptionTool {
}
}
/**
* Decrypts a single element encrypted in the flow.xml.gz style (hex-encoded and wrapped with "enc{" and "}").
*
* Example:
* {@code enc{0123456789ABCDEF} } -> "some text"
*
* @param wrappedCipherText the wrapped and hex-encoded cipher text
* @param password the password used to encrypt the content (UTF-8 encoded)
* @param algorithm the encryption and KDF algorithm (defaults to PBEWITHMD5AND256BITAES-CBC-OPENSSL)
* @param provider the security provider (defaults to BC)
* @return the plaintext in UTF-8 encoding
*/
private
static String decryptFlowElement(String wrappedCipherText, String password, String algorithm = DEFAULT_FLOW_ALGORITHM, String provider = DEFAULT_PROVIDER) {
// Drop the "enc{" and closing "}"
if (!(wrappedCipherText =~ WRAPPED_FLOW_XML_CIPHER_TEXT_REGEX)) {
throw new SensitivePropertyProtectionException("The provided cipher text does not match the expected format 'enc{0123456789ABCDEF...}'")
}
String unwrappedCipherText = wrappedCipherText.replaceAll(/enc\{/, "")[0..<-1]
if (unwrappedCipherText.length() % 2 == 1 || unwrappedCipherText.length() == 0) {
throw new SensitivePropertyProtectionException("The provided cipher text must have an even number of hex characters")
}
// Decode the hex
byte[] cipherBytes = Hex.decodeHex(unwrappedCipherText.chars)
/* The structure of each cipher text is 16 bytes of salt || actual cipher text,
* so extract the salt (32 bytes encoded as hex, 16 bytes raw) and combine that
* with the default (and unchanged) iteration count that is hardcoded in
* {@link StandardPBEByteEncryptor}. I am extracting
* these values to magic numbers here so when the refactoring is performed,
* stronger decisions can be implemented here
*/
byte[] saltBytes = cipherBytes[0..<DEFAULT_SALT_SIZE_BYTES]
cipherBytes = cipherBytes[DEFAULT_SALT_SIZE_BYTES..-1]
Cipher decryptionCipher = generateFlowDecryptionCipher(password, saltBytes, algorithm, provider)
byte[] plainBytes = decryptionCipher.doFinal(cipherBytes)
new String(plainBytes, StandardCharsets.UTF_8)
}
/**
* Returns an initialized {@link javax.crypto.Cipher} instance with the extracted salt.
*
* @param password the password (UTF-8 encoding)
* @param saltBytes the salt (raw bytes)
* @param algorithm the KDF/encryption algorithm
* @param provider the security provider
* @return the initialized {@link javax.crypto.Cipher}
*/
private
static Cipher generateFlowDecryptionCipher(String password, byte[] saltBytes, String algorithm = DEFAULT_FLOW_ALGORITHM, String provider = DEFAULT_PROVIDER) {
Cipher decryptCipher = Cipher.getInstance(algorithm, provider)
PBEKeySpec keySpec = new PBEKeySpec(password.chars)
SecretKeyFactory keyFactory = SecretKeyFactory.getInstance(algorithm, provider)
SecretKey pbeKey = keyFactory.generateSecret(keySpec)
PBEParameterSpec parameterSpec = new PBEParameterSpec(saltBytes, DEFAULT_KDF_ITERATIONS)
decryptCipher.init(Cipher.DECRYPT_MODE, pbeKey, parameterSpec)
decryptCipher
}
/**
* Encrypts a single element in the flow.xml.gz style (hex-encoded and wrapped with "enc{" and "}").
*
* Example:
* "some text" -> {@code enc{0123456789ABCDEF} }
*
* @param plaintext the plaintext in UTF-8 encoding
* @param saltBytes the salt to embed in the cipher text to allow key derivation and decryption later in raw format
* @param encryptCipher the configured Cipher instance
* @return the wrapped and hex-encoded cipher text
*/
private static String encryptFlowElement(String plaintext, byte[] saltBytes, Cipher encryptCipher) {
byte[] plainBytes = plaintext?.getBytes(StandardCharsets.UTF_8) ?: new byte[0]
/* The structure of each cipher text is 16 bytes of salt || actual cipher text,
* so extract the salt (32 bytes encoded as hex, 16 bytes raw) and combine that
* with the default (and unchanged) iteration count that is hardcoded in
* {@link StandardPBEByteEncryptor}. I am extracting
* these values to magic numbers here so when the refactoring is performed,
* stronger decisions can be implemented here
*/
if (saltBytes.length != DEFAULT_SALT_SIZE_BYTES) {
throw new SensitivePropertyProtectionException("The salt must be ${DEFAULT_SALT_SIZE_BYTES} bytes")
}
byte[] cipherBytes = encryptCipher.doFinal(plainBytes)
byte[] saltAndCipherBytes = concatByteArrays(saltBytes, cipherBytes)
// Encode the hex
String hexEncodedCipherText = Hex.encodeHexString(saltAndCipherBytes)
"enc{${hexEncodedCipherText}}"
}
/**
* Utility method to quickly concatenate an arbitrary number of byte[].
*
* @param arrays the byte[] arrays
* @returna single byte[] containing the values concatenated
*/
private static byte[] concatByteArrays(byte[] ... arrays) {
ByteArrayOutputStream outputStream = new ByteArrayOutputStream()
arrays.each { byte[] it -> outputStream.write(it) }
outputStream.toByteArray()
}
/**
* Scans XML content and decrypts each encrypted element, then re-encrypts it with the new key, and returns the final XML content.
*
@ -799,57 +685,37 @@ class ConfigEncryptionTool {
* @param existingProvider the {@link java.security.Provider} to use (defaults to BC)
* @return the encrypted XML content as an InputStream
*/
private InputStream migrateFlowXmlContent(InputStream flowXmlContent, String existingFlowPassword, String newFlowPassword, String existingAlgorithm = DEFAULT_FLOW_ALGORITHM, String existingProvider = DEFAULT_PROVIDER, String newAlgorithm = DEFAULT_FLOW_ALGORITHM, String newProvider = DEFAULT_PROVIDER) {
/* For re-encryption, for performance reasons, we will use a fixed salt for all of
* the operations. These values are stored in the same file and the default key is in the
* source code (see NIFI-1465 and NIFI-1277), so the security trade-off is minimal
* but the performance hit is substantial. We can't make this decision for
* decryption because the FlowSerializer still uses PropertyEncryptor which does not
* follow this pattern
*/
byte[] encryptionSalt = new byte[DEFAULT_SALT_SIZE_BYTES]
new SecureRandom().nextBytes(encryptionSalt)
Cipher encryptCipher = generateFlowEncryptionCipher(newFlowPassword, encryptionSalt, newAlgorithm, newProvider)
int elementCount = 0
private InputStream migrateFlowXmlContent(InputStream flowXmlContent, String existingFlowPassword, String newFlowPassword, String existingAlgorithm = DEFAULT_FLOW_ALGORITHM, String newAlgorithm = DEFAULT_FLOW_ALGORITHM) {
File tempFlowXmlFile = new File(getTemporaryFlowXmlFile(outputFlowXmlPath).toString())
BufferedWriter tempFlowXmlWriter = getFlowOutputStream(tempFlowXmlFile, flowXmlContent instanceof GZIPInputStream)
final OutputStream flowOutputStream = getFlowOutputStream(tempFlowXmlFile, flowXmlContent instanceof GZIPInputStream)
// Scan through XML content as a stream, decrypt and re-encrypt fields with a new flow password
final BufferedReader reader = new BufferedReader(new InputStreamReader(flowXmlContent))
String line;
NiFiProperties inputProperties = NiFiProperties.createBasicNiFiProperties("", [
(NiFiProperties.SENSITIVE_PROPS_KEY): existingFlowPassword,
(NiFiProperties.SENSITIVE_PROPS_ALGORITHM): existingAlgorithm
])
while((line = reader.readLine()) != null) {
def matcher = line =~ WRAPPED_FLOW_XML_CIPHER_TEXT_REGEX
if(matcher.find()) {
String plaintext = decryptFlowElement(matcher.getAt(0), existingFlowPassword, existingAlgorithm, existingProvider)
byte[] cipherBytes = encryptCipher.doFinal(plaintext.bytes)
byte[] saltAndCipherBytes = concatByteArrays(encryptionSalt, cipherBytes)
elementCount++
tempFlowXmlWriter.writeLine(line.replaceFirst(WRAPPED_FLOW_XML_CIPHER_TEXT_REGEX, "enc{${Hex.encodeHex(saltAndCipherBytes)}}"))
} else {
tempFlowXmlWriter.writeLine(line)
}
}
tempFlowXmlWriter.flush()
tempFlowXmlWriter.close()
NiFiProperties outputProperties = NiFiProperties.createBasicNiFiProperties("", [
(NiFiProperties.SENSITIVE_PROPS_KEY): newFlowPassword,
(NiFiProperties.SENSITIVE_PROPS_ALGORITHM): newAlgorithm
])
final PropertyEncryptor inputEncryptor = PropertyEncryptorFactory.getPropertyEncryptor(inputProperties)
final PropertyEncryptor outputEncryptor = PropertyEncryptorFactory.getPropertyEncryptor(outputProperties)
final FlowEncryptor flowEncryptor = new StandardFlowEncryptor()
flowEncryptor.processFlow(flowXmlContent, flowOutputStream, inputEncryptor, outputEncryptor)
// Overwrite the original flow file with the migrated flow file
Files.move(tempFlowXmlFile.toPath(), Paths.get(outputFlowXmlPath), StandardCopyOption.ATOMIC_MOVE)
if (isVerbose) {
logger.info("Decrypted and re-encrypted ${elementCount} elements for flow.xml.gz")
}
loadFlowXml(outputFlowXmlPath)
}
private BufferedWriter getFlowOutputStream(File outputFlowXmlPath, boolean isFileGZipped) {
private OutputStream getFlowOutputStream(File outputFlowXmlPath, boolean isFileGZipped) {
OutputStream flowOutputStream = new FileOutputStream(outputFlowXmlPath)
if(isFileGZipped) {
flowOutputStream = new GZIPOutputStream(flowOutputStream)
}
new BufferedWriter(new OutputStreamWriter(flowOutputStream))
return flowOutputStream
}
// Create a temporary output file we can write the stream to
@ -859,35 +725,6 @@ class ConfigEncryptionTool {
Paths.get(originalOutputFlowXmlPath).resolveSibling(migratedFileName)
}
/**
* Returns an initialized encryption cipher for the flow.xml.gz content.
*
* @param newFlowPassword the new encryption password
* @param saltBytes the salt [16 bytes in raw format]
* @param algorithm the KDF/encryption algorithm
* @param provider the security provider
* @return the initialized cipher instance
*/
private
static Cipher generateFlowEncryptionCipher(String newFlowPassword, byte[] saltBytes, String algorithm = DEFAULT_FLOW_ALGORITHM, String provider = DEFAULT_PROVIDER) {
// Use the standard Cipher with the password and algorithm provided
Cipher encryptCipher = Cipher.getInstance(algorithm, provider)
/* For re-encryption, for performance reasons, we will use a fixed salt for all of
* the operations. These values are stored in the same file and the default key is in the
* source code (see NIFI-1465 and NIFI-1277), so the security trade-off is minimal
* but the performance hit is substantial. We can't make this decision for
* decryption because the FlowSerializer still uses PropertyEncryptor which does not
* follow this pattern
*/
PBEKeySpec keySpec = new PBEKeySpec(newFlowPassword.chars)
SecretKeyFactory keyFactory = SecretKeyFactory.getInstance(algorithm, provider)
SecretKey pbeKey = keyFactory.generateSecret(keySpec)
PBEParameterSpec parameterSpec = new PBEParameterSpec(saltBytes, DEFAULT_KDF_ITERATIONS)
encryptCipher.init(Cipher.ENCRYPT_MODE, pbeKey, parameterSpec)
encryptCipher
}
String decryptLoginIdentityProviders(String encryptedXml, String existingKeyHex = keyHex) {
AESSensitivePropertyProvider sensitivePropertyProvider = new AESSensitivePropertyProvider(existingKeyHex)
@ -1619,14 +1456,12 @@ class ConfigEncryptionTool {
// Get the algorithms and providers
NiFiProperties nfp = niFiProperties
String existingAlgorithm = nfp?.getProperty(NiFiProperties.SENSITIVE_PROPS_ALGORITHM) ?: DEFAULT_FLOW_ALGORITHM
String existingProvider = nfp?.getProperty(NiFiProperties.SENSITIVE_PROPS_PROVIDER) ?: DEFAULT_PROVIDER
String newAlgorithm = newFlowAlgorithm ?: existingAlgorithm
String newProvider = newFlowProvider ?: existingProvider
try {
logger.info("Migrating flow.xml file at ${flowXmlPath}. This could take a while if the flow XML is very large.")
migrateFlowXmlContent(flowXmlInputStream, existingFlowPassword, newFlowPassword, existingAlgorithm, existingProvider, newAlgorithm, newProvider)
migrateFlowXmlContent(flowXmlInputStream, existingFlowPassword, newFlowPassword, existingAlgorithm, newAlgorithm)
} catch (Exception e) {
logger.error("Encountered an error: ${e.getLocalizedMessage()}")
if (e instanceof BadPaddingException) {

382
nifi-toolkit/nifi-toolkit-encrypt-config/src/test/groovy/org/apache/nifi/properties/ConfigEncryptionToolTest.groovy
View File

@ -83,16 +83,6 @@ class ConfigEncryptionToolTest extends GroovyTestCase {
private static final String PASSWORD = "thisIsABadPassword"
private static final String ANOTHER_PASSWORD = "thisIsAnotherBadPassword"
private static final String STATIC_SALT = "\$s0\$40801\$ABCDEFGHIJKLMNOPQRSTUV"
private static final String SCRYPT_SALT_PATTERN = /\$\w{2}\$\w{5,}\$[\w\/\=\+]+/
// Hash of "password" with 00 * 16 salt
private static
final String HASHED_PASSWORD = "\$s0\$40801\$AAAAAAAAAAAAAAAAAAAAAA\$gLSh7ChbHdOIMvZ74XGjV6qF65d9qvQ8n75FeGnM8YM"
// Hash of [key derived from "password"] with 00 * 16 salt
private static
final String HASHED_KEY_HEX = "\$s0\$40801\$AAAAAAAAAAAAAAAAAAAAAA\$pJOGA9sPL+pRzynnwt6G2FfVTyLQdbKSbk6W8IKId8E"
// From ConfigEncryptionTool.deriveKeyFromPassword("thisIsABadPassword")
private static
final String PASSWORD_KEY_HEX_256 = "2C576A9585DB862F5ECBEE5B4FFFCCA14B18D8365968D7081651006507AD2BDE"
@ -137,6 +127,7 @@ class ConfigEncryptionToolTest extends GroovyTestCase {
@After
void tearDown() throws Exception {
System.clearProperty(NiFiProperties.PROPERTIES_FILE_PATH)
TestAppender.reset()
}
@ -842,37 +833,6 @@ class ConfigEncryptionToolTest extends GroovyTestCase {
workingFile.deleteOnExit()
}
@Ignore("Setting the Windows file permissions fails in the test harness, so the test does not throw the expected exception")
@Test
void testLoadNiFiPropertiesShouldHandleReadFailureOnWindows() {
// Arrange
Assume.assumeTrue("Test only runs on Windows", SystemUtils.IS_OS_WINDOWS)
File inputPropertiesFile = new File("src/test/resources/nifi_with_sensitive_properties_unprotected.properties")
File workingFile = new File("target/tmp_nifi.properties")
workingFile.delete()
Files.copy(inputPropertiesFile.toPath(), workingFile.toPath())
// Empty set of permissions
workingFile.setReadable(false)
ConfigEncryptionTool tool = new ConfigEncryptionTool()
String[] args = ["-n", workingFile.path, "-k", KEY_HEX]
tool.parse(args)
// Act
def msg = shouldFail(IOException) {
tool.loadNiFiProperties()
logger.info("Read nifi.properties")
}
logger.expected(msg)
// Assert
assert msg == "Cannot load NiFiProperties from [${workingFile.path}]".toString()
workingFile.deleteOnExit()
}
@Test
void testShouldEncryptSensitiveProperties() {
// Arrange
@ -1028,66 +988,6 @@ class ConfigEncryptionToolTest extends GroovyTestCase {
workingFile.deleteOnExit()
}
@Ignore("this test needs to be updated to ensure any created files are done under target")
@Test
void testWriteKeyToBootstrapConfShouldHandleReadFailure() {
// Arrange
File emptyKeyFile = new File("src/test/resources/bootstrap_with_empty_root_key.conf")
File workingFile = new File("target/tmp_bootstrap.conf")
workingFile.delete()
Files.copy(emptyKeyFile.toPath(), workingFile.toPath())
// Empty set of permissions
setFilePermissions(workingFile, [])
logger.info("Set POSIX permissions to ${getFilePermissions(workingFile)}")
ConfigEncryptionTool tool = new ConfigEncryptionTool()
String[] args = ["-b", workingFile.path, "-k", KEY_HEX, "-n", "nifi.properties"]
tool.parse(args)
// Act
def msg = shouldFail(IOException) {
tool.writeKeyToBootstrapConf()
logger.info("Updated bootstrap.conf")
}
logger.expected(msg)
// Assert
assert msg == "The bootstrap.conf file at tmp_bootstrap.conf must exist and be readable and writable by the user running this tool"
workingFile.deleteOnExit()
}
@Ignore("this test needs to be updated to ensure any created files are done under target")
@Test
void testWriteKeyToBootstrapConfShouldHandleWriteFailure() {
// Arrange
File emptyKeyFile = new File("src/test/resources/bootstrap_with_empty_root_key.conf")
File workingFile = new File("target/tmp_bootstrap.conf")
workingFile.delete()
Files.copy(emptyKeyFile.toPath(), workingFile.toPath())
// Read-only set of permissions
setFilePermissions(workingFile, [PosixFilePermission.OWNER_READ, PosixFilePermission.GROUP_READ, PosixFilePermission.OTHERS_READ])
logger.info("Set POSIX permissions to ${getFilePermissions(workingFile)}")
ConfigEncryptionTool tool = new ConfigEncryptionTool()
String[] args = ["-b", workingFile.path, "-k", KEY_HEX, "-n", "nifi.properties"]
tool.parse(args)
// Act
def msg = shouldFail(IOException) {
tool.writeKeyToBootstrapConf()
logger.info("Updated bootstrap.conf")
}
logger.expected(msg)
// Assert
assert msg == "The bootstrap.conf file at tmp_bootstrap.conf must exist and be readable and writable by the user running this tool"
workingFile.deleteOnExit()
}
@Test
void testShouldEncryptNiFiPropertiesWithEmptyProtectionScheme() {
// Arrange
@ -1474,43 +1374,6 @@ class ConfigEncryptionToolTest extends GroovyTestCase {
setupTmpDir()
}
@Ignore("Setting the Windows file permissions fails in the test harness, so the test does not throw the expected exception")
@Test
void testWriteNiFiPropertiesShouldHandleWriteFailureWhenFileDoesNotExistOnWindows() {
// Arrange
Assume.assumeTrue("Test only runs on Windows", SystemUtils.IS_OS_WINDOWS)
File inputPropertiesFile = new File("src/test/resources/nifi_with_sensitive_properties_unprotected.properties")
File tmpDir = new File("target/tmp/")
tmpDir.mkdirs()
File workingFile = new File("target/tmp/tmp_nifi.properties")
workingFile.delete()
// Read-only set of permissions
tmpDir.setWritable(false)
ConfigEncryptionTool tool = new ConfigEncryptionTool()
String[] args = ["-n", inputPropertiesFile.path, "-o", workingFile.path, "-k", KEY_HEX]
tool.parse(args)
NiFiProperties niFiProperties = tool.loadNiFiProperties()
tool.@niFiProperties = niFiProperties
logger.info("Loaded ${niFiProperties.size()} properties from ${inputPropertiesFile.path}")
// Act
def msg = shouldFail(IOException) {
tool.writeNiFiProperties()
logger.info("Wrote to ${workingFile.path}")
}
logger.expected(msg)
// Assert
assert msg == "The nifi.properties file at ${workingFile.path} must be writable by the user running this tool".toString()
workingFile.deleteOnExit()
setFilePermissions(tmpDir, [PosixFilePermission.OWNER_READ, PosixFilePermission.OWNER_WRITE])
tmpDir.deleteOnExit()
}
@Test
void testShouldPerformFullOperation() {
// Arrange
@ -3821,11 +3684,6 @@ class ConfigEncryptionToolTest extends GroovyTestCase {
def updatedFlowCipherTexts = findFieldsInStream(updatedFlowXmlContent, WFXCTR)
logger.info("Updated flow.xml.gz cipher texts: ${updatedFlowCipherTexts}")
assert updatedFlowCipherTexts.size() == CIPHER_TEXT_COUNT
updatedFlowCipherTexts.each {
String decryptedValue = ConfigEncryptionTool.decryptFlowElement(it, newFlowPassword)
logger.info("Decrypted value of migrated ${workingFlowXmlFile.path} was: ${decryptedValue}")
assert decryptedValue == PASSWORD || decryptedValue == ANOTHER_PASSWORD
}
}
})
@ -3927,11 +3785,6 @@ class ConfigEncryptionToolTest extends GroovyTestCase {
def migratedFlowCipherTexts = findFieldsInStream(migratedFlowXmlContent, WFXCTR)
logger.info("Updated flow.xml.gz cipher texts: ${migratedFlowCipherTexts}")
assert migratedFlowCipherTexts.size() == CIPHER_TEXT_COUNT
migratedFlowCipherTexts.each {
String decryptedValue = ConfigEncryptionTool.decryptFlowElement(it, newFlowPassword)
logger.info("Decrypted value of migrated ${workingFlowXmlFile.path} was: ${decryptedValue}")
assert decryptedValue == PASSWORD || decryptedValue == ANOTHER_PASSWORD
}
}
})
@ -4068,11 +3921,6 @@ class ConfigEncryptionToolTest extends GroovyTestCase {
logger.info("Original flow.xml.gz cipher texts: ${originalFlowCipherTexts}")
logger.info("Updated flow.xml.gz cipher texts: ${migratedFlowCipherTexts}")
assert migratedFlowCipherTexts.size() == CIPHER_TEXT_COUNT
migratedFlowCipherTexts.each {
String decryptedValue = ConfigEncryptionTool.decryptFlowElement(it, newFlowPassword)
logger.info("Decrypted value of migrated ${workingFlowXmlFile.path} was: ${decryptedValue}")
assert decryptedValue == PASSWORD || decryptedValue == ANOTHER_PASSWORD
}
}
})
@ -4204,11 +4052,6 @@ class ConfigEncryptionToolTest extends GroovyTestCase {
logger.info("Original " + workingFlowXmlFile.path + " unique cipher texts: ${originalFlowCipherTexts}")
logger.info("Migrated " + workingFlowXmlFile.path + " unique cipher texts: ${migratedFlowCipherTexts}")
assert migratedFlowCipherTexts.size() == CIPHER_TEXT_COUNT
migratedFlowCipherTexts.each {
String decryptedValue = ConfigEncryptionTool.decryptFlowElement(it, newFlowPassword)
logger.info("Decrypted value of migrated ${workingFlowXmlFile.path} was: ${decryptedValue}")
assert decryptedValue == PASSWORD || decryptedValue == ANOTHER_PASSWORD
}
}
})
@ -4362,11 +4205,6 @@ class ConfigEncryptionToolTest extends GroovyTestCase {
def flowCipherTexts = findFieldsInStream(updatedFlowXmlContent, WFXCTR)
logger.info("Updated flow.xml.gz cipher texts: ${flowCipherTexts}")
assert flowCipherTexts.size() == CIPHER_TEXT_COUNT
flowCipherTexts.each {
String decryptedValue = ConfigEncryptionTool.decryptFlowElement(it, newFlowPassword)
logger.info("Decrypted value of migrated ${workingFlowXmlFile.path} was: ${decryptedValue}")
assert decryptedValue == PASSWORD || decryptedValue == ANOTHER_PASSWORD
}
// Update the "original" flow cipher texts for the next run to the current values
originalFlowCipherTexts = flowCipherTexts
@ -4374,178 +4212,6 @@ class ConfigEncryptionToolTest extends GroovyTestCase {
}
}
@Test
void testDecryptFlowXmlContentShouldVerifyPattern() {
// Arrange
String existingFlowPassword = "flowPassword"
String sensitivePropertyValue = "thisIsABadProcessorPassword"
final Map<String, String> properties = new HashMap<>()
properties.put(NiFiProperties.SENSITIVE_PROPS_ALGORITHM, DEFAULT_ENCRYPTION_METHOD.algorithm)
properties.put(NiFiProperties.SENSITIVE_PROPS_PROVIDER, DEFAULT_ENCRYPTION_METHOD.provider)
properties.put(NiFiProperties.SENSITIVE_PROPS_KEY, existingFlowPassword)
final NiFiProperties niFiProperties = NiFiProperties.createBasicNiFiProperties(null, properties)
PropertyEncryptor sanityEncryptor = PropertyEncryptorFactory.getPropertyEncryptor(niFiProperties)
String sanityCipherText = "enc{${sanityEncryptor.encrypt(sensitivePropertyValue)}}"
logger.info("Sanity check value: \t${sensitivePropertyValue} -> ${sanityCipherText}")
def validCipherTexts = (0..4).collect {
"enc{${sanityEncryptor.encrypt(sensitivePropertyValue)}}"
}
logger.info("Generated valid cipher texts: \n${validCipherTexts.join("\n")}")
def invalidCipherTexts = ["enc{}",
"enc{x}",
"encx",
"enc{012}",
"enc{01",
"enc{aBc19+===}",
"enc{aB=c19+}",
"enc{aB@}",
"",
"}",
"\"",
">",
null]
// Act
def successfulResults = validCipherTexts.collect { String cipherText ->
ConfigEncryptionTool.decryptFlowElement(cipherText, existingFlowPassword)
}
def failedResults = invalidCipherTexts.collect { String cipherText ->
def msg = shouldFail(SensitivePropertyProtectionException) {
ConfigEncryptionTool.decryptFlowElement(cipherText, existingFlowPassword)
}
logger.expected(msg)
msg
}
// Assert
assert successfulResults.every { it == sensitivePropertyValue }
assert failedResults.every {
it =~ /The provided cipher text does not match the expected format 'enc\{0123456789ABCDEF\.\.\.\}'/ ||
it == "The provided cipher text must have an even number of hex characters"
}
}
/**
* This test verifies that the crypto logic in the tool is compatible with the default encryptor implementation.
*/
@Test
void testShouldDecryptFlowXmlContent() {
// Arrange
String existingFlowPassword = "nififtw!"
String sensitivePropertyValue = "thisIsABadProcessorPassword"
final Map<String, String> properties = new HashMap<>()
properties.put(NiFiProperties.SENSITIVE_PROPS_ALGORITHM, DEFAULT_ENCRYPTION_METHOD.algorithm)
properties.put(NiFiProperties.SENSITIVE_PROPS_PROVIDER, DEFAULT_ENCRYPTION_METHOD.provider)
properties.put(NiFiProperties.SENSITIVE_PROPS_KEY, existingFlowPassword)
final NiFiProperties niFiProperties = NiFiProperties.createBasicNiFiProperties(null, properties)
PropertyEncryptor sanityEncryptor = PropertyEncryptorFactory.getPropertyEncryptor(niFiProperties)
String sanityCipherText = "enc{${sanityEncryptor.encrypt(sensitivePropertyValue)}}"
logger.info("Sanity check value: \t${sensitivePropertyValue} -> ${sanityCipherText}")
// Act
String decryptedElement = ConfigEncryptionTool.decryptFlowElement(sanityCipherText, existingFlowPassword, DEFAULT_ALGORITHM, DEFAULT_PROVIDER)
logger.info("Decrypted flow element: ${decryptedElement}")
String decryptedElementWithDefaultParameters = ConfigEncryptionTool.decryptFlowElement(sanityCipherText, existingFlowPassword)
logger.info("Decrypted flow element: ${decryptedElementWithDefaultParameters}")
// Assert
assert decryptedElement == sensitivePropertyValue
assert decryptedElementWithDefaultParameters == sensitivePropertyValue
}
/**
* This test verifies that the crypto logic in the tool is compatible with an encrypted value taken from a production flow.xml.gz.
*/
@Test
void testShouldDecryptFlowXmlContentFromLegacyFlow() {
// Arrange
// DEFAULT_SENSITIVE_PROPS_KEY = "nififtw!" at the time this test
// was written and for the encrypted value, but it could change, so don't
// reference transitively here
String existingFlowPassword = DEFAULT_LEGACY_SENSITIVE_PROPS_KEY
final String EXPECTED_PLAINTEXT = "thisIsABadPassword"
final String ENCRYPTED_VALUE_FROM_FLOW = "enc{5d8c45f04790e73cba72e5e3fbee1145f2e18256c3b33c283e17f5281611cb5e5f9e6cc988c5be0e8cca7b5dc8fa7cf7}"
// Act
String decryptedElement = ConfigEncryptionTool.decryptFlowElement(ENCRYPTED_VALUE_FROM_FLOW, existingFlowPassword, DEFAULT_ALGORITHM, DEFAULT_PROVIDER)
logger.info("Decrypted flow element: ${decryptedElement}")
// Assert
assert decryptedElement == EXPECTED_PLAINTEXT
}
@Test
void testShouldEncryptFlowXmlContent() {
// Arrange
String flowPassword = "nififtw!"
String sensitivePropertyValue = "thisIsAnotherBadPassword"
byte[] saltBytes = "thisIsABadSalt..".bytes
final Map<String, String> properties = new HashMap<>()
properties.put(NiFiProperties.SENSITIVE_PROPS_ALGORITHM, DEFAULT_ENCRYPTION_METHOD.algorithm)
properties.put(NiFiProperties.SENSITIVE_PROPS_PROVIDER, DEFAULT_ENCRYPTION_METHOD.provider)
properties.put(NiFiProperties.SENSITIVE_PROPS_KEY, flowPassword)
final NiFiProperties niFiProperties = NiFiProperties.createBasicNiFiProperties(null, properties)
PropertyEncryptor sanityEncryptor = PropertyEncryptorFactory.getPropertyEncryptor(niFiProperties)
Cipher encryptionCipher = generateEncryptionCipher(flowPassword)
// Act
String encryptedElement = ConfigEncryptionTool.encryptFlowElement(sensitivePropertyValue, saltBytes, encryptionCipher)
logger.info("Encrypted flow element: ${encryptedElement}")
// Assert
assert encryptedElement =~ WFXCTR
String sanityPlaintext = sanityEncryptor.decrypt(encryptedElement[4..<-1])
logger.info("Sanity check value: \t${encryptedElement} -> ${sanityPlaintext}")
assert sanityPlaintext == sensitivePropertyValue
}
@Test
void testShouldEncryptAndDecryptFlowXmlContent() {
// Arrange
String flowPassword = "flowPassword"
String sensitivePropertyValue = "thisIsABadProcessorPassword"
byte[] saltBytes = "thisIsABadSalt..".bytes
Cipher encryptionCipher = generateEncryptionCipher(flowPassword)
// Act
String encryptedElement = ConfigEncryptionTool.encryptFlowElement(sensitivePropertyValue, saltBytes, encryptionCipher)
logger.info("Encrypted flow element: ${encryptedElement}")
String decryptedElement = ConfigEncryptionTool.decryptFlowElement(encryptedElement, flowPassword)
logger.info("Decrypted flow element: ${decryptedElement}")
// Assert
assert encryptedElement =~ WFXCTR
assert decryptedElement == sensitivePropertyValue
}
private
static Cipher generateEncryptionCipher(String password, String algorithm = DEFAULT_ALGORITHM, String provider = DEFAULT_PROVIDER) {
Cipher cipher = Cipher.getInstance(algorithm, provider)
PBEKeySpec keySpec = new PBEKeySpec(password.chars)
SecretKeyFactory keyFactory = SecretKeyFactory.getInstance(algorithm, provider)
SecretKey pbeKey = keyFactory.generateSecret(keySpec)
byte[] saltBytes = "thisIsABadSalt..".bytes
PBEParameterSpec parameterSpec = new PBEParameterSpec(saltBytes, 1000)
cipher.init(Cipher.ENCRYPT_MODE, pbeKey, parameterSpec)
cipher
}
@Test
void testShouldMigrateFlowXmlContent() {
// Arrange
@ -4584,11 +4250,6 @@ class ConfigEncryptionToolTest extends GroovyTestCase {
def migratedCipherTexts = findFieldsInStream(migratedFlowXmlFile, WFXCTR)
assert migratedCipherTexts.size() == cipherTextCount
migratedCipherTexts.each {
String decryptedValue = ConfigEncryptionTool.decryptFlowElement(it, newFlowPassword)
logger.info("Decrypted value of migrated " + workingFile.path + " was: " + decryptedValue)
assert decryptedValue == PASSWORD || decryptedValue == ANOTHER_PASSWORD
}
// Ensure that everything else is identical
assert flowXmlFile.text.replaceAll(WFXCTR, "") ==
@ -4642,11 +4303,6 @@ class ConfigEncryptionToolTest extends GroovyTestCase {
assert newCipherTexts.size() == ORIGINAL_CIPHER_TEXT_COUNT
newCipherTexts.each {
String decryptedValue = ConfigEncryptionTool.decryptFlowElement(it, newFlowPassword)
assert decryptedValue == PASSWORD || decryptedValue == ANOTHER_PASSWORD
}
// Ensure that everything else is identical
assert new File(workingFile.path).text.replaceAll(WFXCTR, "") ==
flowXmlFile.text.replaceAll(WFXCTR, "")
@ -4699,42 +4355,6 @@ class ConfigEncryptionToolTest extends GroovyTestCase {
}
}
/**
* This test is tightly scoped to the migration of the flow XML content to ensure the expected exception type is thrown.
*/
@Test
void testMigrateFlowXmlContentWithIncorrectExistingPasswordShouldFailWithBadPaddingException() {
// Arrange
String flowXmlPath = "src/test/resources/flow.xml"
File flowXmlFile = new File(flowXmlPath)
File tmpDir = setupTmpDir()
File workingFile = new File("target/tmp/tmp-flow.xml")
workingFile.delete()
Files.copy(flowXmlFile.toPath(), workingFile.toPath())
ConfigEncryptionTool tool = new ConfigEncryptionTool()
tool.isVerbose = true
tool.flowXmlPath = workingFile.path
tool.outputFlowXmlPath = workingFile.path
// Use the wrong existing password
String wrongExistingFlowPassword = DEFAULT_LEGACY_SENSITIVE_PROPS_KEY.reverse()
String newFlowPassword = FLOW_PASSWORD
InputStream xmlContent = new ByteArrayInputStream(workingFile.bytes)
// Act
def message = shouldFail(BadPaddingException) {
InputStream migratedXmlContent = tool.migrateFlowXmlContent(xmlContent, wrongExistingFlowPassword, newFlowPassword)
logger.info("Migrated flow.xml.")
}
logger.expected(message)
// Assert
assert message =~ "pad block corrupted"
}
/**
* This test is scoped to the higher-level method to ensure that if a bad padding exception is thrown, the right errors are displayed.
*/