mirror of https://github.com/apache/nifi.git
NIFI-9711 Added support for flow.json.gz in SetSensitivePropertiesKey
Signed-off-by: Joe Gresock <jgresock@gmail.com> This closes #5783.
This commit is contained in:
exceptionfactory
Joe Gresock
parent
d0a23bc26b
commit
45f8795177
|
|
@ -33,6 +33,7 @@ import java.io.UncheckedIOException;
|
|||
import java.nio.file.Files;
|
||||
import java.nio.file.Path;
|
||||
import java.nio.file.StandardCopyOption;
|
||||
import java.util.Arrays;
|
||||
import java.util.List;
|
||||
import java.util.Properties;
|
||||
import java.util.stream.Collectors;
|
||||
|
|
@ -51,9 +52,13 @@ public class SetSensitivePropertiesKey {
|
|||
|
||||
protected static final String CONFIGURATION_FILE = "nifi.flow.configuration.file";
|
||||
|
||||
protected static final String CONFIGURATION_JSON_FILE = "nifi.flow.configuration.json.file";
|
||||
|
||||
private static final List<String> CONFIGURATION_FILES = Arrays.asList(CONFIGURATION_FILE, CONFIGURATION_JSON_FILE);
|
||||
|
||||
private static final int MINIMUM_REQUIRED_LENGTH = 12;
|
||||
|
||||
private static final String FLOW_XML_PREFIX = "flow.xml.";
|
||||
private static final String FLOW_PREFIX = "nifi.flow.";
|
||||
|
||||
private static final String GZ_EXTENSION = ".gz";
|
||||
|
||||
|
|
@ -82,7 +87,6 @@ public class SetSensitivePropertiesKey {
|
|||
final File propertiesFile = new File(propertiesFilePath);
|
||||
final Properties properties = loadProperties(propertiesFile);
|
||||
|
||||
final File flowConfigurationFile = getFlowConfigurationFile(properties);
|
||||
try {
|
||||
storeProperties(propertiesFile, outputPropertiesKey);
|
||||
System.out.printf("NiFi Properties Processed [%s]%n", propertiesFilePath);
|
||||
|
|
@ -91,15 +95,27 @@ public class SetSensitivePropertiesKey {
|
|||
throw new UncheckedIOException(message, e);
|
||||
}
|
||||
|
||||
if (flowConfigurationFile.exists()) {
|
||||
final String algorithm = getAlgorithm(properties);
|
||||
final PropertyEncryptor outputEncryptor = getPropertyEncryptor(outputPropertiesKey, algorithm);
|
||||
processFlowConfiguration(properties, outputEncryptor);
|
||||
processFlowConfigurationFiles(properties, outputPropertiesKey);
|
||||
}
|
||||
|
||||
private static void processFlowConfigurationFiles(final Properties properties, final String outputPropertiesKey) {
|
||||
final String algorithm = getAlgorithm(properties);
|
||||
final PropertyEncryptor outputEncryptor = getPropertyEncryptor(outputPropertiesKey, algorithm);
|
||||
|
||||
for (final String configurationFilePropertyName : CONFIGURATION_FILES) {
|
||||
final String configurationFileProperty = properties.getProperty(configurationFilePropertyName);
|
||||
if (configurationFileProperty == null || configurationFileProperty.isEmpty()) {
|
||||
System.out.printf("Flow Configuration Property not specified [%s]%n", configurationFileProperty);
|
||||
} else {
|
||||
final File configurationFile = new File(configurationFileProperty);
|
||||
if (configurationFile.exists()) {
|
||||
processFlowConfiguration(configurationFile, properties, outputEncryptor);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private static void processFlowConfiguration(final Properties properties, final PropertyEncryptor outputEncryptor) {
|
||||
final File flowConfigurationFile = getFlowConfigurationFile(properties);
|
||||
private static void processFlowConfiguration(final File flowConfigurationFile, final Properties properties, final PropertyEncryptor outputEncryptor) {
|
||||
try (final InputStream flowInputStream = new GZIPInputStream(new FileInputStream(flowConfigurationFile))) {
|
||||
final File flowOutputFile = getFlowOutputFile();
|
||||
final Path flowOutputPath = flowOutputFile.toPath();
|
||||
|
|
@ -115,7 +131,7 @@ public class SetSensitivePropertiesKey {
|
|||
final Path flowConfigurationPath = flowConfigurationFile.toPath();
|
||||
Files.move(flowOutputPath, flowConfigurationPath, StandardCopyOption.REPLACE_EXISTING);
|
||||
System.out.printf("Flow Configuration Processed [%s]%n", flowConfigurationPath);
|
||||
} catch (final IOException|RuntimeException e) {
|
||||
} catch (final IOException | RuntimeException e) {
|
||||
System.err.printf("Failed to process Flow Configuration [%s]%n", flowConfigurationFile);
|
||||
e.printStackTrace();
|
||||
}
|
||||
|
|
@ -138,7 +154,7 @@ public class SetSensitivePropertiesKey {
|
|||
}
|
||||
|
||||
private static File getFlowOutputFile() throws IOException {
|
||||
final File flowOutputFile = File.createTempFile(FLOW_XML_PREFIX, GZ_EXTENSION);
|
||||
final File flowOutputFile = File.createTempFile(FLOW_PREFIX, GZ_EXTENSION);
|
||||
flowOutputFile.deleteOnExit();
|
||||
return flowOutputFile;
|
||||
}
|
||||
|
|
@ -170,8 +186,4 @@ public class SetSensitivePropertiesKey {
|
|||
private static PropertyEncryptor getPropertyEncryptor(final String propertiesKey, final String propertiesAlgorithm) {
|
||||
return new PropertyEncryptorBuilder(propertiesKey).setAlgorithm(propertiesAlgorithm).build();
|
||||
}
|
||||
|
||||
private static File getFlowConfigurationFile(final Properties properties) {
|
||||
return new File(properties.getProperty(CONFIGURATION_FILE));
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -21,9 +21,11 @@ import org.junit.jupiter.api.AfterEach;
|
|||
import org.junit.jupiter.api.Test;
|
||||
|
||||
import java.io.File;
|
||||
import java.io.FileNotFoundException;
|
||||
import java.io.FileOutputStream;
|
||||
import java.io.IOException;
|
||||
import java.net.URISyntaxException;
|
||||
import java.net.URL;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.nio.file.Files;
|
||||
import java.nio.file.Path;
|
||||
|
|
@ -33,11 +35,27 @@ import java.util.Optional;
|
|||
import java.util.UUID;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
import static org.junit.Assert.assertEquals;
|
||||
import static org.junit.Assert.assertTrue;
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
import static org.junit.jupiter.api.Assertions.assertTrue;
|
||||
|
||||
public class SetSensitivePropertiesKeyTest {
|
||||
private static final String FLOW_CONTENTS = "<property><value>PROPERTY</value></property>";
|
||||
private static final String TEMP_FILE_PREFIX = SetSensitivePropertiesKeyTest.class.getSimpleName();
|
||||
|
||||
private static final String FLOW_CONTENTS_JSON = "{\"property\":\"value\"}";
|
||||
|
||||
private static final String FLOW_CONTENTS_XML = "<property><value>PROPERTY</value></property>";
|
||||
|
||||
private static final String JSON_GZ = ".json.gz";
|
||||
|
||||
private static final String XML_GZ = ".xml.gz";
|
||||
|
||||
private static final String PROPERTIES_EXTENSION = ".properties";
|
||||
|
||||
private static final String BLANK_PROPERTIES = "/blank.nifi.properties";
|
||||
|
||||
private static final String POPULATED_PROPERTIES = "/populated.nifi.properties";
|
||||
|
||||
private static final String LEGACY_BLANK_PROPERTIES = "/legacy-blank.nifi.properties";
|
||||
|
||||
@AfterEach
|
||||
public void clearProperties() {
|
||||
|
|
@ -51,8 +69,9 @@ public class SetSensitivePropertiesKeyTest {
|
|||
|
||||
@Test
|
||||
public void testMainBlankKeyAndAlgorithm() throws IOException, URISyntaxException {
|
||||
final Path flowConfiguration = getFlowConfiguration();
|
||||
final Path propertiesPath = getNiFiProperties(flowConfiguration, "/blank.nifi.properties");
|
||||
final Path flowConfiguration = getFlowConfiguration(FLOW_CONTENTS_XML, XML_GZ);
|
||||
final Path flowConfigurationJson = getFlowConfiguration(FLOW_CONTENTS_JSON, JSON_GZ);
|
||||
final Path propertiesPath = getNiFiProperties(flowConfiguration, flowConfigurationJson, BLANK_PROPERTIES);
|
||||
|
||||
System.setProperty(SetSensitivePropertiesKey.PROPERTIES_FILE_PATH, propertiesPath.toString());
|
||||
|
||||
|
|
@ -60,13 +79,26 @@ public class SetSensitivePropertiesKeyTest {
|
|||
SetSensitivePropertiesKey.main(new String[]{sensitivePropertiesKey});
|
||||
|
||||
assertPropertiesKeyUpdated(propertiesPath, sensitivePropertiesKey);
|
||||
assertTrue("Flow Configuration not found", flowConfiguration.toFile().exists());
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testMainPopulatedKeyAndAlgorithm() throws IOException, URISyntaxException {
|
||||
final Path flowConfiguration = getFlowConfiguration();
|
||||
final Path propertiesPath = getNiFiProperties(flowConfiguration, "/populated.nifi.properties");
|
||||
final Path flowConfiguration = getFlowConfiguration(FLOW_CONTENTS_XML, XML_GZ);
|
||||
final Path flowConfigurationJson = getFlowConfiguration(FLOW_CONTENTS_JSON, JSON_GZ);
|
||||
final Path propertiesPath = getNiFiProperties(flowConfiguration, flowConfigurationJson, POPULATED_PROPERTIES);
|
||||
|
||||
System.setProperty(SetSensitivePropertiesKey.PROPERTIES_FILE_PATH, propertiesPath.toString());
|
||||
|
||||
final String sensitivePropertiesKey = UUID.randomUUID().toString();
|
||||
SetSensitivePropertiesKey.main(new String[]{sensitivePropertiesKey});
|
||||
|
||||
assertPropertiesKeyUpdated(propertiesPath, sensitivePropertiesKey);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testMainLegacyBlankKeyAndAlgorithm() throws IOException, URISyntaxException {
|
||||
final Path flowConfiguration = getFlowConfiguration(FLOW_CONTENTS_XML, XML_GZ);
|
||||
final Path propertiesPath = getNiFiProperties(flowConfiguration, null, LEGACY_BLANK_PROPERTIES);
|
||||
|
||||
System.setProperty(SetSensitivePropertiesKey.PROPERTIES_FILE_PATH, propertiesPath.toString());
|
||||
|
||||
|
|
@ -74,7 +106,6 @@ public class SetSensitivePropertiesKeyTest {
|
|||
SetSensitivePropertiesKey.main(new String[]{sensitivePropertiesKey});
|
||||
|
||||
assertPropertiesKeyUpdated(propertiesPath, sensitivePropertiesKey);
|
||||
assertTrue("Flow Configuration not found", flowConfiguration.toFile().exists());
|
||||
}
|
||||
|
||||
private void assertPropertiesKeyUpdated(final Path propertiesPath, final String sensitivePropertiesKey) throws IOException {
|
||||
|
|
@ -82,37 +113,51 @@ public class SetSensitivePropertiesKeyTest {
|
|||
.stream()
|
||||
.filter(line -> line.startsWith(SetSensitivePropertiesKey.PROPS_KEY))
|
||||
.findFirst();
|
||||
assertTrue("Sensitive Key Property not found", keyProperty.isPresent());
|
||||
assertTrue(keyProperty.isPresent(), "Sensitive Key Property not found");
|
||||
|
||||
final String expectedProperty = String.format("%s=%s", SetSensitivePropertiesKey.PROPS_KEY, sensitivePropertiesKey);
|
||||
assertEquals("Sensitive Key Property not updated", expectedProperty, keyProperty.get());
|
||||
assertEquals(expectedProperty, keyProperty.get(), "Sensitive Key Property not updated");
|
||||
}
|
||||
|
||||
private Path getNiFiProperties(final Path flowConfigurationPath, String propertiesResource) throws IOException, URISyntaxException {
|
||||
final Path sourcePropertiesPath = Paths.get(SetSensitivePropertiesKey.class.getResource(propertiesResource).toURI());
|
||||
private Path getNiFiProperties(
|
||||
final Path flowConfigurationPath,
|
||||
final Path flowConfigurationJsonPath,
|
||||
String propertiesResource
|
||||
) throws IOException, URISyntaxException {
|
||||
final Path sourcePropertiesPath = Paths.get(getResourceUrl(propertiesResource).toURI());
|
||||
final List<String> sourceProperties = Files.readAllLines(sourcePropertiesPath);
|
||||
final List<String> flowProperties = sourceProperties.stream().map(line -> {
|
||||
if (line.startsWith(SetSensitivePropertiesKey.CONFIGURATION_FILE)) {
|
||||
return line + flowConfigurationPath.toString();
|
||||
return line + flowConfigurationPath;
|
||||
} else if (line.startsWith(SetSensitivePropertiesKey.CONFIGURATION_JSON_FILE)) {
|
||||
return flowConfigurationJsonPath == null ? line : line + flowConfigurationJsonPath;
|
||||
} else {
|
||||
return line;
|
||||
}
|
||||
}).collect(Collectors.toList());
|
||||
|
||||
final Path propertiesPath = Files.createTempFile(SetSensitivePropertiesKey.class.getSimpleName(), ".properties");
|
||||
final Path propertiesPath = Files.createTempFile(TEMP_FILE_PREFIX, PROPERTIES_EXTENSION);
|
||||
propertiesPath.toFile().deleteOnExit();
|
||||
Files.write(propertiesPath, flowProperties);
|
||||
return propertiesPath;
|
||||
}
|
||||
|
||||
private Path getFlowConfiguration() throws IOException {
|
||||
final Path flowConfigurationPath = Files.createTempFile(SetSensitivePropertiesKey.class.getSimpleName(), ".xml.gz");
|
||||
private Path getFlowConfiguration(final String contents, final String extension) throws IOException {
|
||||
final Path flowConfigurationPath = Files.createTempFile(TEMP_FILE_PREFIX, extension);
|
||||
final File flowConfigurationFile = flowConfigurationPath.toFile();
|
||||
flowConfigurationFile.deleteOnExit();
|
||||
|
||||
try (final GZIPOutputStream outputStream = new GZIPOutputStream(new FileOutputStream(flowConfigurationFile))) {
|
||||
outputStream.write(FLOW_CONTENTS.getBytes(StandardCharsets.UTF_8));
|
||||
outputStream.write(contents.getBytes(StandardCharsets.UTF_8));
|
||||
}
|
||||
return flowConfigurationPath;
|
||||
}
|
||||
|
||||
private URL getResourceUrl(String resource) throws FileNotFoundException {
|
||||
final URL resourceUrl = SetSensitivePropertiesKey.class.getResource(resource);
|
||||
if (resourceUrl == null) {
|
||||
throw new FileNotFoundException(String.format("Resource [%s] not found", resource));
|
||||
}
|
||||
return resourceUrl;
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -15,3 +15,4 @@
|
|||
nifi.sensitive.props.key=
|
||||
nifi.sensitive.props.algorithm=
|
||||
nifi.flow.configuration.file=
|
||||
nifi.flow.configuration.json.file=
|
||||
|
|
|
|||
|
|
@ -0,0 +1,17 @@
|
|||
# Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
# contributor license agreements. See the NOTICE file distributed with
|
||||
# this work for additional information regarding copyright ownership.
|
||||
# The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
# (the "License"); you may not use this file except in compliance with
|
||||
# the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
nifi.sensitive.props.key=
|
||||
nifi.sensitive.props.algorithm=
|
||||
nifi.flow.configuration.file=
|
||||
|
|
@ -15,3 +15,4 @@
|
|||
nifi.sensitive.props.key=D5E41AC1-EEF8-4A54-930D-593F749AE95C
|
||||
nifi.sensitive.props.algorithm=NIFI_ARGON2_AES_GCM_256
|
||||
nifi.flow.configuration.file=
|
||||
nifi.flow.configuration.json.file=
|
||||
|
|
|
|||
|
|
@ -4334,14 +4334,21 @@ For more information see the <<toolkit-guide.adoc#encrypt_config_tool,Encrypt-Co
|
|||
|
||||
==== Updating the Sensitive Properties Key
|
||||
|
||||
Starting with version 1.14.0, NiFi requires a value for 'nifi.sensitive.props.key' in _nifi.properties_.
|
||||
Starting with version 1.14.0, NiFi requires a value for `nifi.sensitive.props.key` in _nifi.properties_.
|
||||
|
||||
The following command can be used to read an existing _flow.json.gz_ configuration and set a new sensitive properties key in _nifi.properties_:
|
||||
The following command can be used to read an existing flow configuration and set a new sensitive properties key in _nifi.properties_:
|
||||
|
||||
```
|
||||
$ ./bin/nifi.sh set-sensitive-properties-key <sensitivePropertiesKey>
|
||||
```
|
||||
|
||||
The command reads the following flow configuration file properties from _nifi.properties_:
|
||||
|
||||
- `nifi.flow.configuration.file`
|
||||
- `nifi.flow.configuration.json.file`
|
||||
|
||||
The command checks for the existence of each file and updates the sensitive property values found.
|
||||
|
||||
The minimum required length for a new sensitive properties key is 12 characters.
|
||||
|
||||
=== Start New NiFi
|
||||
|
|
|
|||
Loading…
Reference in New Issue