Apache
/
nifi
mirror of https://github.com/apache/nifi.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

NIFI-9412: Autogenerate sensitive key in MiNiFi if not present 

This closes #5558

Signed-off-by: David Handermann <exceptionfactory@apache.org>
This commit is contained in:
Matthew Burgess 2021-11-29 20:54:54 -05:00 committed by exceptionfactory
parent e6b573e79e
commit 07c4a05cdf
No known key found for this signature in database
GPG Key ID: 29B6A52D2AAE8DBA
7 changed files with 53 additions and 50 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
minifi/minifi-bootstrap/pom.xml
View File

@ -60,6 +60,11 @@ limitations under the License.
<groupId>org.apache.nifi</groupId>
<artifactId>nifi-utils</artifactId>
</dependency>
<dependency>
<groupId>org.apache.nifi</groupId>
<artifactId>nifi-properties</artifactId>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>org.yaml</groupId>
<artifactId>snakeyaml</artifactId>

52
minifi/minifi-bootstrap/src/main/java/org/apache/nifi/minifi/bootstrap/util/ConfigTransformer.java
View File

@ -17,7 +17,6 @@
package org.apache.nifi.minifi.bootstrap.util;
import org.apache.commons.io.output.ByteArrayOutputStream;
import org.apache.commons.lang3.StringUtils;
import org.apache.nifi.minifi.bootstrap.configuration.ConfigurationChangeException;
@ -46,6 +45,7 @@ import org.apache.nifi.minifi.commons.schema.common.ConvertableSchema;
import org.apache.nifi.minifi.commons.schema.common.Schema;
import org.apache.nifi.minifi.commons.schema.common.StringUtil;
import org.apache.nifi.minifi.commons.schema.serialization.SchemaLoader;
import org.apache.nifi.util.NiFiProperties;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.w3c.dom.DOMException;
@ -69,7 +69,9 @@ import java.io.OutputStream;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.security.SecureRandom;
import java.util.Arrays;
import java.util.Base64;
import java.util.Collection;
import java.util.HashMap;
import java.util.List;
@ -80,9 +82,10 @@ import java.util.zip.GZIPOutputStream;
public final class ConfigTransformer {
// Underlying version of NIFI will be using
public static final String NIFI_VERSION = "1.8.0";
public static final String ROOT_GROUP = "Root-Group";
public static final String NIFI_VERSION_KEY = "nifi.version";
private static final Base64.Encoder KEY_ENCODER = Base64.getEncoder().withoutPadding();
private static final int SENSITIVE_PROPERTIES_KEY_LENGTH = 24;
public static final Logger logger = LoggerFactory.getLogger(ConfigTransformer.class);
@ -178,8 +181,7 @@ public final class ConfigTransformer {
ProvenanceRepositorySchema provenanceRepositorySchema = configSchema.getProvenanceRepositorySchema();
OrderedProperties orderedProperties = new OrderedProperties();
orderedProperties.setProperty(NIFI_VERSION_KEY, NIFI_VERSION, "# Core Properties #" + System.lineSeparator());
orderedProperties.setProperty("nifi.flow.configuration.file", "./conf/flow.xml.gz");
orderedProperties.setProperty("nifi.flow.configuration.file", "./conf/flow.xml.gz", "# Core Properties #" + System.lineSeparator());
orderedProperties.setProperty("nifi.flow.configuration.archive.enabled", "false");
orderedProperties.setProperty("nifi.flow.configuration.archive.dir", "./conf/archive/");
orderedProperties.setProperty("nifi.flowcontroller.autoResumeState", "true");
@ -258,7 +260,17 @@ public final class ConfigTransformer {
orderedProperties.setProperty("nifi.web.jetty.threads", "200");
final String sensitivePropertiesKey = sensitiveProperties.getKey();
final String notnullSensitivePropertiesKey = sensitivePropertiesKey != null ? sensitivePropertiesKey : "";
final String notnullSensitivePropertiesKey;
// Auto-generate the sensitive properties key if not provided, NiFi security libraries require it
if (StringUtil.isNullOrEmpty(sensitivePropertiesKey)) {
logger.warn("Generating Random Sensitive Properties Key [{}]", NiFiProperties.SENSITIVE_PROPS_KEY);
final SecureRandom secureRandom = new SecureRandom();
final byte[] sensitivePropertiesKeyBinary = new byte[SENSITIVE_PROPERTIES_KEY_LENGTH];
secureRandom.nextBytes(sensitivePropertiesKeyBinary);
notnullSensitivePropertiesKey = KEY_ENCODER.encodeToString(sensitivePropertiesKeyBinary);
} else {
notnullSensitivePropertiesKey = sensitivePropertiesKey;
}
orderedProperties.setProperty("nifi.sensitive.props.key", notnullSensitivePropertiesKey, System.lineSeparator() + "# security properties #");
orderedProperties.setProperty("nifi.sensitive.props.algorithm", sensitiveProperties.getAlgorithm());
@ -719,19 +731,19 @@ public final class ConfigTransformer {
public static final String PROPERTIES_FILE_APACHE_2_0_LICENSE =
" Licensed to the Apache Software Foundation (ASF) under one or more\n" +
"# contributor license agreements. See the NOTICE file distributed with\n" +
"# this work for additional information regarding copyright ownership.\n" +
"# The ASF licenses this file to You under the Apache License, Version 2.0\n" +
"# (the \"License\"); you may not use this file except in compliance with\n" +
"# the License. You may obtain a copy of the License at\n" +
"#\n" +
"# http://www.apache.org/licenses/LICENSE-2.0\n" +
"#\n" +
"# Unless required by applicable law or agreed to in writing, software\n" +
"# distributed under the License is distributed on an \"AS IS\" BASIS,\n" +
"# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n" +
"# See the License for the specific language governing permissions and\n" +
"# limitations under the License.\n" +
"\n";
"# contributor license agreements. See the NOTICE file distributed with\n" +
"# this work for additional information regarding copyright ownership.\n" +
"# The ASF licenses this file to You under the Apache License, Version 2.0\n" +
"# (the \"License\"); you may not use this file except in compliance with\n" +
"# the License. You may obtain a copy of the License at\n" +
"#\n" +
"# http://www.apache.org/licenses/LICENSE-2.0\n" +
"#\n" +
"# Unless required by applicable law or agreed to in writing, software\n" +
"# distributed under the License is distributed on an \"AS IS\" BASIS,\n" +
"# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n" +
"# See the License for the specific language governing permissions and\n" +
"# limitations under the License.\n" +
"\n";
}

33
minifi/minifi-bootstrap/src/test/java/org/apache/nifi/minifi/bootstrap/util/ConfigTransformerTest.java
View File

@ -33,6 +33,7 @@ import org.apache.nifi.minifi.commons.schema.ReportingSchema;
import org.apache.nifi.minifi.commons.schema.common.StringUtil;
import org.apache.nifi.minifi.commons.schema.exception.SchemaLoaderException;
import org.apache.nifi.minifi.commons.schema.serialization.SchemaLoader;
import org.apache.nifi.util.StringUtils;
import org.junit.Before;
import org.junit.Rule;
import org.junit.Test;
@ -78,7 +79,6 @@ public class ConfigTransformerTest {
public static final Map<String, Integer> PG_ELEMENT_ORDER_MAP = generateOrderMap(
Arrays.asList("processor", "inputPort", "outputPort", "funnel", "processGroup", "remoteProcessGroup", "connection"));
private XPathFactory xPathFactory;
private Document document;
private Element config;
private DocumentBuilder documentBuilder;
@ -88,7 +88,7 @@ public class ConfigTransformerTest {
@Before
public void setup() throws ParserConfigurationException {
documentBuilder = DocumentBuilderFactory.newInstance().newDocumentBuilder();
document = documentBuilder.newDocument();
final Document document = documentBuilder.newDocument();
config = document.createElement("config");
xPathFactory = XPathFactory.newInstance();
}
@ -169,8 +169,6 @@ public class ConfigTransformerTest {
try (InputStream pre216PropertiesStream = ConfigTransformerTest.class.getClassLoader().getResourceAsStream("MINIFI-216/nifi.properties.before")) {
pre216Properties.load(pre216PropertiesStream);
}
pre216Properties.setProperty(ConfigTransformer.NIFI_VERSION_KEY, ConfigTransformer.NIFI_VERSION);
ByteArrayOutputStream outputStream = new ByteArrayOutputStream();
try (InputStream configStream = ConfigTransformerTest.class.getClassLoader().getResourceAsStream("MINIFI-216/config.yml")) {
ConfigTransformer.writeNiFiProperties(SchemaLoader.loadConfigSchemaFromYaml(configStream), outputStream);
@ -189,8 +187,6 @@ public class ConfigTransformerTest {
try (InputStream pre216PropertiesStream = ConfigTransformerTest.class.getClassLoader().getResourceAsStream("MINIFI-216/nifi.properties.before")) {
pre216Properties.load(pre216PropertiesStream);
}
pre216Properties.setProperty(ConfigTransformer.NIFI_VERSION_KEY, ConfigTransformer.NIFI_VERSION);
ByteArrayOutputStream outputStream = new ByteArrayOutputStream();
try (InputStream configStream = ConfigTransformerTest.class.getClassLoader().getResourceAsStream("MINIFI-216/configOverrides.yml")) {
ConfigSchema configSchema = SchemaLoader.loadConfigSchemaFromYaml(configStream);
@ -214,8 +210,6 @@ public class ConfigTransformerTest {
try (InputStream pre216PropertiesStream = ConfigTransformerTest.class.getClassLoader().getResourceAsStream("MINIFI-277/nifi.properties")) {
initialProperties.load(pre216PropertiesStream);
}
initialProperties.setProperty(ConfigTransformer.NIFI_VERSION_KEY, ConfigTransformer.NIFI_VERSION);
ByteArrayOutputStream outputStream = new ByteArrayOutputStream();
try (InputStream configStream = ConfigTransformerTest.class.getClassLoader().getResourceAsStream("MINIFI-277/config.yml")) {
ConfigSchema configSchema = SchemaLoader.loadConfigSchemaFromYaml(configStream);
@ -421,7 +415,7 @@ public class ConfigTransformerTest {
File inputFile = new File("./src/test/resources/config-invalid.yml");
ConfigTransformer.transformConfigFile(new FileInputStream(inputFile), "./target/", null);
fail("Invalid configuration file was not detected.");
} catch (SchemaLoaderException e){
} catch (SchemaLoaderException e) {
assertEquals("Provided YAML configuration is not a Map", e.getMessage());
}
}
@ -432,7 +426,7 @@ public class ConfigTransformerTest {
File inputFile = new File("./src/test/resources/config-malformed-field.yml");
ConfigTransformer.transformConfigFile(new FileInputStream(inputFile), "./target/", null);
fail("Invalid configuration file was not detected.");
} catch (InvalidConfigurationException e){
} catch (InvalidConfigurationException e) {
assertEquals("Failed to transform config file due to:['threshold' in section 'Swap' because it is found but could not be parsed as a Number]", e.getMessage());
}
}
@ -443,7 +437,7 @@ public class ConfigTransformerTest {
File inputFile = new File("./src/test/resources/config-empty.yml");
ConfigTransformer.transformConfigFile(new FileInputStream(inputFile), "./target/", null);
fail("Invalid configuration file was not detected.");
} catch (SchemaLoaderException e){
} catch (SchemaLoaderException e) {
assertEquals("Provided YAML configuration is not a Map", e.getMessage());
}
}
@ -454,7 +448,7 @@ public class ConfigTransformerTest {
File inputFile = new File("./src/test/resources/config-missing-required-field.yml");
ConfigTransformer.transformConfigFile(new FileInputStream(inputFile), "./target/", null);
fail("Invalid configuration file was not detected.");
} catch (InvalidConfigurationException e){
} catch (InvalidConfigurationException e) {
assertEquals("Failed to transform config file due to:['class' in section 'Processors' because it was not found and it is required]", e.getMessage());
}
}
@ -465,7 +459,7 @@ public class ConfigTransformerTest {
File inputFile = new File("./src/test/resources/config-multiple-problems.yml");
ConfigTransformer.transformConfigFile(new FileInputStream(inputFile), "./target/", null);
fail("Invalid configuration file was not detected.");
} catch (InvalidConfigurationException e){
} catch (InvalidConfigurationException e) {
assertEquals("Failed to transform config file due to:['class' in section 'Processors' because it was not found and it is required], " +
"['scheduling strategy' in section 'Provenance Reporting' because it is not a valid scheduling strategy], " +
"['source name' in section 'Connections' because it was not found and it is required]", e.getMessage());
@ -699,8 +693,6 @@ public class ConfigTransformerTest {
try (InputStream pre216PropertiesStream = ConfigTransformerTest.class.getClassLoader().getResourceAsStream("MINIFI-245/nifi.properties.before")) {
pre216Properties.load(pre216PropertiesStream);
}
pre216Properties.setProperty(ConfigTransformer.NIFI_VERSION_KEY, ConfigTransformer.NIFI_VERSION);
ByteArrayOutputStream outputStream = new ByteArrayOutputStream();
try (InputStream configStream = ConfigTransformerTest.class.getClassLoader().getResourceAsStream("MINIFI-245/config.yml")) {
ConfigTransformer.writeNiFiProperties(SchemaLoader.loadConfigSchemaFromYaml(configStream), outputStream);
@ -710,7 +702,7 @@ public class ConfigTransformerTest {
for (String name : pre216Properties.stringPropertyNames()) {
// Verify the Content Repo property was overridden
if("nifi.content.repository.implementation".equals(name)) {
if ("nifi.content.repository.implementation".equals(name)) {
assertNotEquals("Property key " + name + " was not overridden.", pre216Properties.getProperty(name), properties.getProperty(name));
} else {
assertEquals("Property key " + name + " doesn't match.", pre216Properties.getProperty(name), properties.getProperty(name));
@ -724,8 +716,6 @@ public class ConfigTransformerTest {
try (InputStream pre216PropertiesStream = ConfigTransformerTest.class.getClassLoader().getResourceAsStream("NIFI-8753/nifi.properties.before")) {
pre216Properties.load(pre216PropertiesStream);
}
pre216Properties.setProperty(ConfigTransformer.NIFI_VERSION_KEY, ConfigTransformer.NIFI_VERSION);
ByteArrayOutputStream outputStream = new ByteArrayOutputStream();
try (InputStream configStream = ConfigTransformerTest.class.getClassLoader().getResourceAsStream("NIFI-8753/config.yml")) {
ConfigTransformer.writeNiFiProperties(SchemaLoader.loadConfigSchemaFromYaml(configStream), outputStream);
@ -735,7 +725,7 @@ public class ConfigTransformerTest {
for (String name : pre216Properties.stringPropertyNames()) {
// Verify the Content Repo property was overridden
if("nifi.flowfile.repository.implementation".equals(name)) {
if ("nifi.flowfile.repository.implementation".equals(name)) {
assertNotEquals("Property key " + name + " was not overridden.", pre216Properties.getProperty(name), properties.getProperty(name));
} else {
assertEquals("Property key " + name + " doesn't match.", pre216Properties.getProperty(name), properties.getProperty(name));
@ -751,7 +741,8 @@ public class ConfigTransformerTest {
}
final Properties properties = new Properties();
properties.load(new ByteArrayInputStream(outputStream.toByteArray()));
assertEquals("", properties.getProperty("nifi.sensitive.props.key"));
// The property should not be empty/null as it is auto-generated when missing
assertTrue(StringUtils.isNotEmpty(properties.getProperty("nifi.sensitive.props.key")));
}
private String getText(Element element, String path) throws XPathExpressionException {
@ -780,7 +771,7 @@ public class ConfigTransformerTest {
if (index != null) {
if (elementOrderList > index) {
fail("Found " + nodeName + " after " + lastOrderedElementName + "; expected all " + nodeName + " elements to come before the following elements: " + orderMap.entrySet().stream()
.filter(e -> e.getValue() > index ).sorted(Comparator.comparingInt(e -> e.getValue())).map(e -> e.getKey()).collect(Collectors.joining(", ")));
.filter(e -> e.getValue() > index).sorted(Comparator.comparingInt(e -> e.getValue())).map(e -> e.getKey()).collect(Collectors.joining(", ")));
}
lastOrderedElementName = nodeName;
elementOrderList = index;

3
minifi/minifi-bootstrap/src/test/resources/MINIFI-216/nifi.properties.before
View File

@ -15,7 +15,6 @@
# Core Properties #
nifi.version=1.1.0
nifi.flow.configuration.file=./conf/flow.xml.gz
nifi.flow.configuration.archive.enabled=false
nifi.flow.configuration.archive.dir=./conf/archive/
@ -92,7 +91,7 @@ nifi.web.jetty.working.directory=./work/jetty
nifi.web.jetty.threads=200
# security properties #
nifi.sensitive.props.key=
# This needs to be ignored during unit testing: nifi.sensitive.props.key=
nifi.sensitive.props.algorithm=PBEWITHMD5AND256BITAES-CBC-OPENSSL
nifi.security.keystore=/tmp/ssl/localhost-ks.jks

3
minifi/minifi-bootstrap/src/test/resources/MINIFI-245/nifi.properties.before
View File

@ -15,7 +15,6 @@
# Core Properties #
nifi.version=1.1.0
nifi.flow.configuration.file=./conf/flow.xml.gz
nifi.flow.configuration.archive.enabled=false
nifi.flow.configuration.archive.dir=./conf/archive/
@ -92,7 +91,7 @@ nifi.web.jetty.working.directory=./work/jetty
nifi.web.jetty.threads=200
# security properties #
nifi.sensitive.props.key=
# This needs to be ignored during unit testing: nifi.sensitive.props.key=
nifi.sensitive.props.algorithm=PBEWITHMD5AND256BITAES-CBC-OPENSSL
nifi.security.keystore=/tmp/ssl/localhost-ks.jks

4
minifi/minifi-bootstrap/src/test/resources/MINIFI-277/nifi.properties
View File

@ -15,8 +15,6 @@
# Core Properties #
nifi.version=1.1.0
nifi.flow.configuration.file=./conf/flow.xml.gz
nifi.flow.configuration.archive.enabled=false
nifi.flow.configuration.archive.dir=./conf/archive/
@ -94,7 +92,7 @@ nifi.web.jetty.working.directory=./work/jetty
nifi.web.jetty.threads=200
# security properties #
nifi.sensitive.props.key=
# This needs to be ignored during unit testing: nifi.sensitive.props.key=
nifi.sensitive.props.algorithm=PBEWITHMD5AND256BITAES-CBC-OPENSSL
nifi.security.keystore=/tmp/ssl/localhost-ks.jks

3
minifi/minifi-bootstrap/src/test/resources/NIFI-8753/nifi.properties.before
View File

@ -15,7 +15,6 @@
# Core Properties #
nifi.version=1.1.0
nifi.flow.configuration.file=./conf/flow.xml.gz
nifi.flow.configuration.archive.enabled=false
nifi.flow.configuration.archive.dir=./conf/archive/
@ -92,7 +91,7 @@ nifi.web.jetty.working.directory=./work/jetty
nifi.web.jetty.threads=200
# security properties #
nifi.sensitive.props.key=
# This needs to be ignored during unit testing: nifi.sensitive.props.key=
nifi.sensitive.props.algorithm=PBEWITHMD5AND256BITAES-CBC-OPENSSL
nifi.security.keystore=/tmp/ssl/localhost-ks.jks