Apache
/
nifi
mirror of https://github.com/apache/nifi.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

NIFI-1831 Added internal logic and command-line tool to allow AES-encrypted sensitive configuration values in nifi.properties. 

This closes #834.
This commit is contained in:
Andy LoPresto 2016-08-15 20:18:47 -07:00
parent 7a4fed189c
commit c638191a47
No known key found for this signature in database
GPG Key ID: 3C6EF65B2F7DEF69
115 changed files with 10687 additions and 364 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

22
LICENSE
View File

@ -583,3 +583,25 @@ This product bundles 'jsonlint' which is available under an MIT license.
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
THE SOFTWARE.
This product bundles source from 'AbstractingTheJavaConsole'. The source is available under an MIT LICENSE.
Copyright (C) 2010 McDowell
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in
all copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
THE SOFTWARE.

2
NOTICE
View File

@ -13,6 +13,8 @@ Copyright 2012, 2013 Willi Ballenthin william.ballenthin@mandiant.com
while at Mandiant http://www.mandiant.com
The derived work is adapted from Evtx/Evtx.py, Evtx/BinaryParser.py, Evtx/Nodes.py, Evtx/Views.py and can be found in the org.apache.nifi.processors.evtx.parser package.
This includes derived works from the Apache Storm (ASLv2 licensed) project (https://github.com/apache/storm):
Copyright 2015 The Apache Software Foundation
The derived work is adapted from

1
nifi-api/src/main/java/org/apache/nifi/processor/ProcessorInitializationContext.java
View File

@ -52,5 +52,4 @@ public interface ProcessorInitializationContext extends KerberosContext {
* type of this NiFi instance.
*/
NodeTypeProvider getNodeTypeProvider();
}

2
nifi-assembly/src/main/assembly/common.xml
View File

@ -28,6 +28,8 @@
<include>slf4j-api</include>
<include>logback-classic</include>
<include>nifi-api</include>
<include>commons-lang3</include>
<include>bcprov-jdk15on</include>
</includes>
</dependencySet>

109
nifi-bootstrap/src/main/java/org/apache/nifi/bootstrap/RunNiFi.java
View File

@ -53,7 +53,6 @@ import java.util.concurrent.TimeUnit;
import java.util.concurrent.locks.Condition;
import java.util.concurrent.locks.Lock;
import java.util.concurrent.locks.ReentrantLock;
import org.apache.commons.lang3.StringUtils;
import org.apache.nifi.bootstrap.notification.NotificationType;
import org.apache.nifi.util.file.FileUtils;
@ -61,7 +60,6 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
/**
*
* <p>
* The class which bootstraps Apache NiFi. This class looks for the
* bootstrap.conf file by looking in the following places (in order):</p>
@ -73,7 +71,7 @@ import org.slf4j.LoggerFactory;
* <li>./conf/bootstrap.conf, where {@code ./} represents the working
* directory.</li>
* </ol>
*
* <p>
* If the {@code bootstrap.conf} file cannot be found, throws a {@code FileNotFoundException}.
*/
public class RunNiFi {
@ -98,6 +96,7 @@ public class RunNiFi {
public static final String NIFI_PID_FILE_NAME = "nifi.pid";
public static final String NIFI_STATUS_FILE_NAME = "nifi.status";
public static final String NIFI_LOCK_FILE_NAME = "nifi.lock";
public static final String NIFI_BOOTSTRAP_SENSITIVE_KEY = "nifi.bootstrap.sensitive.key";
public static final String PID_KEY = "pid";
@ -332,7 +331,7 @@ public class RunNiFi {
}
private File getBootstrapFile(final Logger logger, String directory, String defaultDirectory, String fileName) throws IOException{
private File getBootstrapFile(final Logger logger, String directory, String defaultDirectory, String fileName) throws IOException {
final File confDir = bootstrapConfigFile.getParentFile();
final File nifiHome = confDir.getParentFile();
@ -341,9 +340,9 @@ public class RunNiFi {
final File fileDir;
if(confFileDir != null){
if (confFileDir != null) {
fileDir = new File(confFileDir.trim());
} else{
} else {
fileDir = new File(nifiHome, defaultDirectory);
}
@ -353,19 +352,19 @@ public class RunNiFi {
return statusFile;
}
File getPidFile(final Logger logger) throws IOException{
return getBootstrapFile(logger,NIFI_PID_DIR_PROP,DEFAULT_PID_DIR,NIFI_PID_FILE_NAME);
File getPidFile(final Logger logger) throws IOException {
return getBootstrapFile(logger, NIFI_PID_DIR_PROP, DEFAULT_PID_DIR, NIFI_PID_FILE_NAME);
}
File getStatusFile(final Logger logger) throws IOException{
return getBootstrapFile(logger,NIFI_PID_DIR_PROP,DEFAULT_PID_DIR,NIFI_STATUS_FILE_NAME);
File getStatusFile(final Logger logger) throws IOException {
return getBootstrapFile(logger, NIFI_PID_DIR_PROP, DEFAULT_PID_DIR, NIFI_STATUS_FILE_NAME);
}
File getLockFile(final Logger logger) throws IOException{
return getBootstrapFile(logger,NIFI_PID_DIR_PROP,DEFAULT_PID_DIR,NIFI_LOCK_FILE_NAME);
File getLockFile(final Logger logger) throws IOException {
return getBootstrapFile(logger, NIFI_PID_DIR_PROP, DEFAULT_PID_DIR, NIFI_LOCK_FILE_NAME);
}
File getStatusFile() throws IOException{
File getStatusFile() throws IOException {
return getStatusFile(defaultLogger);
}
@ -388,8 +387,8 @@ public class RunNiFi {
return props;
}
private synchronized void saveProperties(final Properties nifiProps, final Logger logger) throws IOException {
final String pid = nifiProps.getProperty(PID_KEY);
private synchronized void savePidProperties(final Properties pidProperties, final Logger logger) throws IOException {
final String pid = pidProperties.getProperty(PID_KEY);
if (!StringUtils.isBlank(pid)) {
writePidFile(pid, logger);
}
@ -410,16 +409,16 @@ public class RunNiFi {
Files.setPosixFilePermissions(statusFile.toPath(), perms);
} catch (final Exception e) {
logger.warn("Failed to set permissions so that only the owner can read status file {}; "
+ "this may allows others to have access to the key needed to communicate with NiFi. "
+ "Permissions should be changed so that only the owner can read this file", statusFile);
+ "this may allows others to have access to the key needed to communicate with NiFi. "
+ "Permissions should be changed so that only the owner can read this file", statusFile);
}
try (final FileOutputStream fos = new FileOutputStream(statusFile)) {
nifiProps.store(fos, null);
pidProperties.store(fos, null);
fos.getFD().sync();
}
logger.debug("Saved Properties {} to {}", new Object[]{nifiProps, statusFile});
logger.debug("Saved Properties {} to {}", new Object[]{pidProperties, statusFile});
}
private synchronized void writePidFile(final String pid, final Logger logger) throws IOException {
@ -519,8 +518,8 @@ public class RunNiFi {
boolean running = false;
String line;
try (final InputStream in = proc.getInputStream();
final Reader streamReader = new InputStreamReader(in);
final BufferedReader reader = new BufferedReader(streamReader)) {
final Reader streamReader = new InputStreamReader(in);
final BufferedReader reader = new BufferedReader(streamReader)) {
while ((line = reader.readLine()) != null) {
if (line.trim().startsWith(pid)) {
@ -578,7 +577,7 @@ public class RunNiFi {
return new Status(port, pid, true, true);
}
final boolean alive = (pid == null) ? false : isProcessRunning(pid, logger);
final boolean alive = pid != null && isProcessRunning(pid, logger);
return new Status(port, pid, pingSuccess, alive);
}
@ -587,7 +586,7 @@ public class RunNiFi {
final Status status = getStatus(logger);
if (status.isRespondingToPing()) {
logger.info("Apache NiFi is currently running, listening to Bootstrap on port {}, PID={}",
new Object[]{status.getPort(), status.getPid() == null ? "unknown" : status.getPid()});
new Object[]{status.getPort(), status.getPid() == null ? "unknown" : status.getPid()});
return;
}
@ -608,7 +607,7 @@ public class RunNiFi {
}
}
public void env(){
public void env() {
final Logger logger = cmdLogger;
final Status status = getStatus(logger);
if (status.getPid() == null) {
@ -641,19 +640,19 @@ public class RunNiFi {
return;
}
try{
try {
final Method getSystemPropertiesMethod = virtualMachine.getClass().getMethod("getSystemProperties");
final Properties sysProps = (Properties)getSystemPropertiesMethod.invoke(virtualMachine);
final Properties sysProps = (Properties) getSystemPropertiesMethod.invoke(virtualMachine);
for (Entry<Object, Object> syspropEntry : sysProps.entrySet()) {
logger.info(syspropEntry.getKey().toString() + " = " +syspropEntry.getValue().toString());
logger.info(syspropEntry.getKey().toString() + " = " + syspropEntry.getValue().toString());
}
} catch (Throwable t) {
throw new RuntimeException(t);
} finally {
try {
detachMethod.invoke(virtualMachine);
} catch (final Exception e){
} catch (final Exception e) {
logger.warn("Caught exception detaching from process", e);
}
}
@ -721,7 +720,7 @@ public class RunNiFi {
}
serviceManager.notify(NotificationType.NIFI_STOPPED, "NiFi Stopped on Host " + hostname,
"Hello,\n\nApache NiFi has been told to initiate a shutdown on host " + hostname + " at " + now + " by user " + user);
"Hello,\n\nApache NiFi has been told to initiate a shutdown on host " + hostname + " at " + now + " by user " + user);
}
public void stop() throws IOException {
@ -824,9 +823,9 @@ public class RunNiFi {
} catch (final IOException ioe) {
if (pid == null) {
logger.error("Failed to send shutdown command to port {} due to {}. No PID found for the NiFi process, so unable to kill process; "
+ "the process should be killed manually.", new Object[] {port, ioe.toString()});
+ "the process should be killed manually.", new Object[]{port, ioe.toString()});
} else {
logger.error("Failed to send shutdown command to port {} due to {}. Will kill the NiFi Process with PID {}.", new Object[] {port, ioe.toString(), pid});
logger.error("Failed to send shutdown command to port {} due to {}. Will kill the NiFi Process with PID {}.", port, ioe.toString(), pid);
notifyStop();
killProcessTree(pid, logger);
if (statusFile.exists() && !statusFile.delete()) {
@ -844,7 +843,7 @@ public class RunNiFi {
final Process proc = Runtime.getRuntime().exec(new String[]{"ps", "-o", "pid", "--no-headers", "--ppid", ppid});
final List<String> childPids = new ArrayList<>();
try (final InputStream in = proc.getInputStream();
final BufferedReader reader = new BufferedReader(new InputStreamReader(in))) {
final BufferedReader reader = new BufferedReader(new InputStreamReader(in))) {
String line;
while ((line = reader.readLine()) != null) {
@ -900,7 +899,7 @@ public class RunNiFi {
}
final File prevLockFile = getLockFile(cmdLogger);
if (prevLockFile.exists() && !prevLockFile.delete()){
if (prevLockFile.exists() && !prevLockFile.delete()) {
cmdLogger.warn("Failed to delete previous lock file {}; this file should be cleaned up manually", prevLockFile);
}
@ -931,7 +930,7 @@ public class RunNiFi {
builder.directory(workingDir);
}
final String nifiLogDir = replaceNull(System.getProperty("org.apache.nifi.bootstrap.config.log.dir"),DEFAULT_LOG_DIR).trim();
final String nifiLogDir = replaceNull(System.getProperty("org.apache.nifi.bootstrap.config.log.dir"), DEFAULT_LOG_DIR).trim();
final String libFilename = replaceNull(props.get("lib.dir"), "./lib").trim();
File libDir = getFile(libFilename, workingDir);
@ -1001,7 +1000,7 @@ public class RunNiFi {
if (javaHome != null) {
String fileExtension = isWindows() ? ".exe" : "";
File javaFile = new File(javaHome + File.separatorChar + "bin"
+ File.separatorChar + "java" + fileExtension);
+ File.separatorChar + "java" + fileExtension);
if (javaFile.exists() && javaFile.canExecute()) {
javaCmd = javaFile.getAbsolutePath();
}
@ -1020,14 +1019,22 @@ public class RunNiFi {
cmd.add("-Dnifi.properties.file.path=" + nifiPropsFilename);
cmd.add("-Dnifi.bootstrap.listen.port=" + listenPort);
cmd.add("-Dapp=NiFi");
cmd.add("-Dorg.apache.nifi.bootstrap.config.log.dir="+nifiLogDir);
cmd.add("-Dorg.apache.nifi.bootstrap.config.log.dir=" + nifiLogDir);
cmd.add("org.apache.nifi.NiFi");
if (props.containsKey(NIFI_BOOTSTRAP_SENSITIVE_KEY) && props.get(NIFI_BOOTSTRAP_SENSITIVE_KEY) != null) {
cmd.add("-k " + props.get(NIFI_BOOTSTRAP_SENSITIVE_KEY));
}
builder.command(cmd);
final StringBuilder cmdBuilder = new StringBuilder();
for (final String s : cmd) {
cmdBuilder.append(s).append(" ");
// Mask the key
if (s.startsWith("-k ")) {
cmdBuilder.append("-k ****");
} else {
cmdBuilder.append(s).append(" ");
}
}
cmdLogger.info("Starting Apache NiFi...");
@ -1044,12 +1051,12 @@ public class RunNiFi {
gracefulShutdownSeconds = Integer.parseInt(gracefulShutdown);
} catch (final NumberFormatException nfe) {
throw new NumberFormatException("The '" + GRACEFUL_SHUTDOWN_PROP + "' property in Bootstrap Config File "
+ bootstrapConfigAbsoluteFile.getAbsolutePath() + " has an invalid value. Must be a non-negative integer");
+ bootstrapConfigAbsoluteFile.getAbsolutePath() + " has an invalid value. Must be a non-negative integer");
}
if (gracefulShutdownSeconds < 0) {
throw new NumberFormatException("The '" + GRACEFUL_SHUTDOWN_PROP + "' property in Bootstrap Config File "
+ bootstrapConfigAbsoluteFile.getAbsolutePath() + " has an invalid value. Must be a non-negative integer");
+ bootstrapConfigAbsoluteFile.getAbsolutePath() + " has an invalid value. Must be a non-negative integer");
}
Process process = builder.start();
@ -1057,9 +1064,9 @@ public class RunNiFi {
Long pid = getPid(process, cmdLogger);
if (pid != null) {
nifiPid = pid;
final Properties nifiProps = new Properties();
nifiProps.setProperty(PID_KEY, String.valueOf(nifiPid));
saveProperties(nifiProps, cmdLogger);
final Properties pidProperties = new Properties();
pidProperties.setProperty(PID_KEY, String.valueOf(nifiPid));
savePidProperties(pidProperties, cmdLogger);
}
shutdownHook = new ShutdownHook(process, this, secretKey, gracefulShutdownSeconds, loggingExecutor);
@ -1098,7 +1105,7 @@ public class RunNiFi {
return;
}
final File lockFile = getLockFile(defaultLogger);
final File lockFile = getLockFile(defaultLogger);
if (lockFile.exists()) {
defaultLogger.info("A shutdown was initiated. Will not restart NiFi");
return;
@ -1119,9 +1126,9 @@ public class RunNiFi {
pid = getPid(process, defaultLogger);
if (pid != null) {
nifiPid = pid;
final Properties nifiProps = new Properties();
nifiProps.setProperty(PID_KEY, String.valueOf(nifiPid));
saveProperties(nifiProps, defaultLogger);
final Properties pidProperties = new Properties();
pidProperties.setProperty(PID_KEY, String.valueOf(nifiPid));
savePidProperties(pidProperties, defaultLogger);
}
shutdownHook = new ShutdownHook(process, this, secretKey, gracefulShutdownSeconds, loggingExecutor);
@ -1134,14 +1141,14 @@ public class RunNiFi {
// We are expected to restart nifi, so send a notification that it died. If we are not restarting nifi,
// then this means that we are intentionally stopping the service.
serviceManager.notify(NotificationType.NIFI_DIED, "NiFi Died on Host " + hostname,
"Hello,\n\nIt appears that Apache NiFi has died on host " + hostname + " at " + now + "; automatically restarting NiFi");
"Hello,\n\nIt appears that Apache NiFi has died on host " + hostname + " at " + now + "; automatically restarting NiFi");
} else {
defaultLogger.error("Apache NiFi does not appear to have started");
// We are expected to restart nifi, so send a notification that it died. If we are not restarting nifi,
// then this means that we are intentionally stopping the service.
serviceManager.notify(NotificationType.NIFI_DIED, "NiFi Died on Host " + hostname,
"Hello,\n\nIt appears that Apache NiFi has died on host " + hostname + " at " + now +
". Attempted to restart NiFi but the services does not appear to have restarted!");
"Hello,\n\nIt appears that Apache NiFi has died on host " + hostname + " at " + now +
". Attempted to restart NiFi but the services does not appear to have restarted!");
}
} else {
return;
@ -1261,7 +1268,7 @@ public class RunNiFi {
this.autoRestartNiFi = restart;
}
void setNiFiCommandControlPort(final int port, final String secretKey) throws IOException{
void setNiFiCommandControlPort(final int port, final String secretKey) throws IOException {
this.ccPort = port;
this.secretKey = secretKey;
@ -1279,7 +1286,7 @@ public class RunNiFi {
nifiProps.setProperty("secret.key", secretKey);
try {
saveProperties(nifiProps, defaultLogger);
savePidProperties(nifiProps, defaultLogger);
} catch (final IOException ioe) {
defaultLogger.warn("Apache NiFi has started but failed to persist NiFi Port information to {} due to {}", new Object[]{statusFile.getAbsolutePath(), ioe});
}

24
nifi-commons/nifi-properties/src/main/java/org/apache/nifi/util/NiFiProperties.java
View File

@ -38,7 +38,7 @@ import java.util.Set;
* values to be available at runtime. It is strongly tied to the startup
* properties needed and is often refer to as the 'nifi.properties' file. The
* properties contains keys and values. Great care should be taken in leveraging
* this class or passing it along. It's use should be refactored and minimized
* this class or passing it along. Its use should be refactored and minimized
* over time.
*/
public abstract class NiFiProperties {
@ -247,9 +247,9 @@ public abstract class NiFiProperties {
public static final String DEFAULT_KERBEROS_AUTHENTICATION_EXPIRATION = "12 hours";
/**
* Retrieves the property value for the given property key
* Retrieves the property value for the given property key.
*
* @param key the key of property value to lookup.
* @param key the key of property value to lookup
* @return value of property at given key or null if not found
*/
public abstract String getProperty(String key);
@ -257,7 +257,7 @@ public abstract class NiFiProperties {
/**
* Retrieves all known property keys.
*
* @return all known property keys.
* @return all known property keys
*/
public abstract Set<String> getPropertyKeys();
@ -375,11 +375,7 @@ public abstract class NiFiProperties {
public Boolean isSiteToSiteSecure() {
final String secureVal = getProperty(SITE_TO_SITE_SECURE, "true");
if ("false".equalsIgnoreCase(secureVal)) {
return false;
} else {
return true;
}
return !"false".equalsIgnoreCase(secureVal);
}
@ -389,11 +385,7 @@ public abstract class NiFiProperties {
public Boolean isSiteToSiteHttpEnabled() {
final String remoteInputHttpEnabled = getProperty(SITE_TO_SITE_HTTP_ENABLED, "false");
if ("true".equalsIgnoreCase(remoteInputHttpEnabled)) {
return true;
} else {
return false;
}
return "true".equalsIgnoreCase(remoteInputHttpEnabled);
}
@ -769,8 +761,8 @@ public abstract class NiFiProperties {
* Returns true if client certificates are required for REST API. Determined
* if the following conditions are all true:
*
* - login identity provider is not populated - Kerberos service support is
* not enabled
* - login identity provider is not populated
* - Kerberos service support is not enabled
*
* @return true if client certificates are required for access to the REST
* API

36
nifi-commons/nifi-properties/src/main/java/org/apache/nifi/util/StringUtils.java
View File

@ -16,6 +16,8 @@
*/
package org.apache.nifi.util;
import java.util.Collection;
/**
* String Utils based on the Apache Commons Lang String Utils.
* These simple util methods here allow us to avoid a dependency in the core
@ -63,4 +65,38 @@ public class StringUtils {
}
return str.substring(pos + separator.length());
}
public static String join(final Collection collection, String delimiter) {
if (collection == null || collection.size() == 0) {
return EMPTY;
}
final StringBuilder sb = new StringBuilder(collection.size() * 16);
for (Object element : collection) {
sb.append((String) element);
sb.append(delimiter);
}
return sb.toString().substring(0, sb.lastIndexOf(delimiter));
}
public static String padLeft(final String source, int length, char padding) {
if (source != null) {
StringBuilder sb = new StringBuilder(source).reverse();
while (sb.length() < length) {
sb.append(padding);
}
return sb.reverse().toString();
}
return null;
}
public static String padRight(final String source, int length, char padding) {
if (source != null) {
StringBuilder sb = new StringBuilder(source);
while (sb.length() < length) {
sb.append(padding);
}
return sb.toString();
}
return null;
}
}

6
nifi-commons/nifi-properties/src/test/java/org/apache/nifi/util/NiFiPropertiesTest.java
View File

@ -80,11 +80,11 @@ public class NiFiPropertiesTest {
}
private NiFiProperties loadNiFiProperties(final String propsPath) {
private NiFiProperties loadNiFiProperties(final String propsPath){
String realPath = null;
try {
try{
realPath = NiFiPropertiesTest.class.getResource(propsPath).toURI().getPath();
} catch (final URISyntaxException ex) {
}catch(final URISyntaxException ex){
throw new RuntimeException(ex);
}
return NiFiProperties.createBasicNiFiProperties(realPath, null);

12
nifi-framework-api/src/main/java/org/apache/nifi/controller/queue/SortColumn.java
View File

@ -27,7 +27,7 @@ public enum SortColumn implements Comparator<FlowFileSummary> {
/**
* Sort based on the current position in the queue
*/
QUEUE_POSITION (new Comparator<FlowFileSummary>() {
QUEUE_POSITION(new Comparator<FlowFileSummary>() {
@Override
public int compare(final FlowFileSummary o1, final FlowFileSummary o2) {
return Integer.compare(o1.getPosition(), o2.getPosition());
@ -37,7 +37,7 @@ public enum SortColumn implements Comparator<FlowFileSummary> {
/**
* Sort based on the UUID of the FlowFile
*/
FLOWFILE_UUID (new Comparator<FlowFileSummary>() {
FLOWFILE_UUID(new Comparator<FlowFileSummary>() {
@Override
public int compare(final FlowFileSummary o1, final FlowFileSummary o2) {
return o1.getUuid().compareTo(o2.getUuid());
@ -47,7 +47,7 @@ public enum SortColumn implements Comparator<FlowFileSummary> {
/**
* Sort based on the 'filename' attribute of the FlowFile
*/
FILENAME (new Comparator<FlowFileSummary>() {
FILENAME(new Comparator<FlowFileSummary>() {
@Override
public int compare(final FlowFileSummary o1, final FlowFileSummary o2) {
return o1.getFilename().compareTo(o2.getFilename());
@ -67,7 +67,7 @@ public enum SortColumn implements Comparator<FlowFileSummary> {
/**
* Sort based on how long the FlowFile has been sitting in the queue
*/
QUEUED_DURATION (new Comparator<FlowFileSummary>() {
QUEUED_DURATION(new Comparator<FlowFileSummary>() {
@Override
public int compare(final FlowFileSummary o1, final FlowFileSummary o2) {
return -Long.compare(o1.getLastQueuedTime(), o2.getLastQueuedTime());
@ -78,7 +78,7 @@ public enum SortColumn implements Comparator<FlowFileSummary> {
* Sort based on the age of the FlowFile. I.e., the time at which the FlowFile's
* "greatest ancestor" entered the flow
*/
FLOWFILE_AGE (new Comparator<FlowFileSummary>() {
FLOWFILE_AGE(new Comparator<FlowFileSummary>() {
@Override
public int compare(final FlowFileSummary o1, final FlowFileSummary o2) {
return Long.compare(o1.getLineageStartDate(), o2.getLineageStartDate());
@ -88,7 +88,7 @@ public enum SortColumn implements Comparator<FlowFileSummary> {
/**
* Sort based on when the FlowFile's penalization ends
*/
PENALIZATION (new Comparator<FlowFileSummary>() {
PENALIZATION(new Comparator<FlowFileSummary>() {
@Override
public int compare(final FlowFileSummary o1, final FlowFileSummary o2) {
return Boolean.compare(o1.isPenalized(), o2.isPenalized());

83
nifi-nar-bundles/nifi-amqp-bundle/nifi-amqp-processors/src/main/java/org/apache/nifi/amqp/processors/AMQPUtils.java
View File

@ -32,7 +32,6 @@ import com.rabbitmq.client.AMQP.BasicProperties;
/**
* Utility helper class simplify interactions with target AMQP API and NIFI API.
*
*/
abstract class AMQPUtils {
@ -43,20 +42,20 @@ abstract class AMQPUtils {
private final static Logger logger = LoggerFactory.getLogger(AMQPUtils.class);
public enum PropertyNames {
CONTENT_TYPE (AMQP_PROP_PREFIX + "contentType"),
CONTENT_ENCODING (AMQP_PROP_PREFIX + "contentEncoding"),
HEADERS (AMQP_PROP_PREFIX + "headers"),
DELIVERY_MODE (AMQP_PROP_PREFIX + "deliveryMode"),
PRIORITY (AMQP_PROP_PREFIX + "priority"),
CORRELATION_ID (AMQP_PROP_PREFIX + "correlationId"),
REPLY_TO (AMQP_PROP_PREFIX + "replyTo"),
EXPIRATION (AMQP_PROP_PREFIX + "expiration"),
MESSAGE_ID (AMQP_PROP_PREFIX + "messageId"),
TIMESTAMP (AMQP_PROP_PREFIX + "timestamp"),
TYPE (AMQP_PROP_PREFIX + "type"),
USER_ID (AMQP_PROP_PREFIX + "userId"),
APP_ID (AMQP_PROP_PREFIX + "appId"),
CLUSTER_ID (AMQP_PROP_PREFIX + "clusterId");
CONTENT_TYPE(AMQP_PROP_PREFIX + "contentType"),
CONTENT_ENCODING(AMQP_PROP_PREFIX + "contentEncoding"),
HEADERS(AMQP_PROP_PREFIX + "headers"),
DELIVERY_MODE(AMQP_PROP_PREFIX + "deliveryMode"),
PRIORITY(AMQP_PROP_PREFIX + "priority"),
CORRELATION_ID(AMQP_PROP_PREFIX + "correlationId"),
REPLY_TO(AMQP_PROP_PREFIX + "replyTo"),
EXPIRATION(AMQP_PROP_PREFIX + "expiration"),
MESSAGE_ID(AMQP_PROP_PREFIX + "messageId"),
TIMESTAMP(AMQP_PROP_PREFIX + "timestamp"),
TYPE(AMQP_PROP_PREFIX + "type"),
USER_ID(AMQP_PROP_PREFIX + "userId"),
APP_ID(AMQP_PROP_PREFIX + "appId"),
CLUSTER_ID(AMQP_PROP_PREFIX + "clusterId");
PropertyNames(String value) {
this.value = value;
@ -71,7 +70,7 @@ abstract class AMQPUtils {
}
static {
for(PropertyNames propertyNames : PropertyNames.values()) {
for (PropertyNames propertyNames : PropertyNames.values()) {
lookup.put(propertyNames.getValue(), propertyNames);
}
}
@ -89,15 +88,12 @@ abstract class AMQPUtils {
/**
* Updates {@link FlowFile} with attributes representing AMQP properties
*
* @param amqpProperties
* instance of {@link BasicProperties}
* @param flowFile
* instance of target {@link FlowFile}
* @param processSession
* instance of {@link ProcessSession}
* @param amqpProperties instance of {@link BasicProperties}
* @param flowFile instance of target {@link FlowFile}
* @param processSession instance of {@link ProcessSession}
*/
public static FlowFile updateFlowFileAttributesWithAmqpProperties(BasicProperties amqpProperties, FlowFile flowFile, ProcessSession processSession) {
if (amqpProperties != null){
if (amqpProperties != null) {
try {
Method[] methods = BasicProperties.class.getDeclaredMethods();
Map<String, String> attributes = new HashMap<>();
@ -126,8 +122,7 @@ abstract class AMQPUtils {
/**
* Will validate if provided name corresponds to valid AMQP property.
*
* @param name
* the name of the property
* @param name the name of the property
* @return 'true' if valid otherwise 'false'
*/
public static boolean isValidAmqpPropertyName(String name) {
@ -147,11 +142,10 @@ abstract class AMQPUtils {
* Will validate if provided amqpPropValue can be converted to a {@link Map}.
* Should be passed in the format: amqp$headers=key=value,key=value etc.
*
* @param amqpPropValue
* the value of the property
* @param amqpPropValue the value of the property
* @return {@link Map} if valid otherwise null
*/
public static Map<String, Object> validateAMQPHeaderProperty(String amqpPropValue){
public static Map<String, Object> validateAMQPHeaderProperty(String amqpPropValue) {
String[] strEntries = amqpPropValue.split(",");
Map<String, Object> headers = new HashMap<>();
for (String strEntry : strEntries) {
@ -170,11 +164,10 @@ abstract class AMQPUtils {
* Will validate if provided amqpPropValue can be converted to an {@link Integer}, and that its
* value is 1 or 2.
*
* @param amqpPropValue
* the value of the property
* @param amqpPropValue the value of the property
* @return {@link Integer} if valid otherwise null
*/
public static Integer validateAMQPDeliveryModeProperty(String amqpPropValue){
public static Integer validateAMQPDeliveryModeProperty(String amqpPropValue) {
Integer deliveryMode = toInt(amqpPropValue);
if (deliveryMode == null || !(deliveryMode == 1 || deliveryMode == 2)) {
@ -187,14 +180,13 @@ abstract class AMQPUtils {
* Will validate if provided amqpPropValue can be converted to an {@link Integer} and that its
* value is between 0 and 9 (inclusive).
*
* @param amqpPropValue
* the value of the property
* @param amqpPropValue the value of the property
* @return {@link Integer} if valid otherwise null
*/
public static Integer validateAMQPPriorityProperty(String amqpPropValue){
public static Integer validateAMQPPriorityProperty(String amqpPropValue) {
Integer priority = toInt(amqpPropValue);
if (priority == null || !(priority >= 0 && priority <= 9)){
if (priority == null || !(priority >= 0 && priority <= 9)) {
logger.warn("Invalid value for AMQP priority property: " + amqpPropValue);
}
return priority;
@ -203,14 +195,13 @@ abstract class AMQPUtils {
/**
* Will validate if provided amqpPropValue can be converted to a {@link Date}.
*
* @param amqpPropValue
* the value of the property
* @param amqpPropValue the value of the property
* @return {@link Date} if valid otherwise null
*/
public static Date validateAMQPTimestampProperty(String amqpPropValue){
public static Date validateAMQPTimestampProperty(String amqpPropValue) {
Long timestamp = toLong(amqpPropValue);
if (timestamp == null){
if (timestamp == null) {
logger.warn("Invalid value for AMQP timestamp property: " + amqpPropValue);
return null;
}
@ -222,14 +213,13 @@ abstract class AMQPUtils {
/**
* Takes a {@link String} and tries to convert to an {@link Integer}.
*
* @param strVal
* the value to be converted
* @param strVal the value to be converted
* @return {@link Integer} if valid otherwise null
*/
private static Integer toInt(String strVal){
private static Integer toInt(String strVal) {
try {
return Integer.parseInt(strVal);
} catch (NumberFormatException aE){
} catch (NumberFormatException aE) {
return null;
}
}
@ -237,14 +227,13 @@ abstract class AMQPUtils {
/**
* Takes a {@link String} and tries to convert to a {@link Long}.
*
* @param strVal
* the value to be converted
* @param strVal the value to be converted
* @return {@link Long} if valid otherwise null
*/
private static Long toLong(String strVal){
private static Long toLong(String strVal) {
try {
return Long.parseLong(strVal);
} catch (NumberFormatException aE){
} catch (NumberFormatException aE) {
return null;
}
}

2
nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-cluster/src/main/java/org/apache/nifi/cluster/coordination/heartbeat/ClusterProtocolHeartbeatMonitor.java
View File

@ -25,10 +25,8 @@ import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentMap;
import java.util.function.Function;
import java.util.stream.Collectors;
import javax.xml.bind.JAXBContext;
import javax.xml.bind.Unmarshaller;
import org.apache.nifi.cluster.coordination.ClusterCoordinator;
import org.apache.nifi.cluster.coordination.node.NodeConnectionState;
import org.apache.nifi.cluster.coordination.node.NodeConnectionStatus;

1
nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-cluster/src/main/java/org/apache/nifi/cluster/coordination/http/endpoints/StatusHistoryEndpointMerger.java
View File

@ -26,7 +26,6 @@ import java.util.Map;
import java.util.Set;
import java.util.TreeMap;
import java.util.regex.Pattern;
import org.apache.nifi.cluster.coordination.http.EndpointResponseMerger;
import org.apache.nifi.cluster.manager.NodeResponse;
import org.apache.nifi.cluster.protocol.NodeIdentifier;

1
nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-cluster/src/main/java/org/apache/nifi/cluster/coordination/node/CuratorNodeProtocolSender.java
View File

@ -19,7 +19,6 @@ package org.apache.nifi.cluster.coordination.node;
import java.io.IOException;
import java.net.InetSocketAddress;
import java.nio.charset.StandardCharsets;
import org.apache.curator.RetryPolicy;
import org.apache.curator.framework.CuratorFramework;
import org.apache.curator.framework.CuratorFrameworkFactory;

1
nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-cluster/src/main/java/org/apache/nifi/cluster/coordination/node/NodeClusterCoordinator.java
View File

@ -31,7 +31,6 @@ import java.util.concurrent.atomic.AtomicLong;
import java.util.function.Supplier;
import java.util.regex.Pattern;
import java.util.stream.Collectors;
import org.apache.commons.collections4.queue.CircularFifoQueue;
import org.apache.commons.lang3.StringUtils;
import org.apache.nifi.cluster.coordination.ClusterCoordinator;

17
nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-cluster/src/test/java/org/apache/nifi/cluster/coordination/http/replication/TestThreadPoolRequestReplicator.java
View File

@ -22,6 +22,13 @@ import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertNull;
import static org.junit.Assert.assertTrue;
import com.sun.jersey.api.client.Client;
import com.sun.jersey.api.client.ClientHandlerException;
import com.sun.jersey.api.client.ClientResponse;
import com.sun.jersey.api.client.ClientResponse.Status;
import com.sun.jersey.api.client.WebResource;
import com.sun.jersey.core.header.InBoundHeaders;
import com.sun.jersey.core.header.OutBoundHeaders;
import java.io.ByteArrayInputStream;
import java.net.SocketTimeoutException;
import java.net.URI;
@ -34,9 +41,7 @@ import java.util.Map;
import java.util.Set;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicInteger;
import javax.ws.rs.HttpMethod;
import org.apache.commons.collections4.map.MultiValueMap;
import org.apache.nifi.cluster.coordination.ClusterCoordinator;
import org.apache.nifi.cluster.coordination.node.NodeConnectionState;
@ -57,14 +62,6 @@ import org.mockito.internal.util.reflection.Whitebox;
import org.mockito.invocation.InvocationOnMock;
import org.mockito.stubbing.Answer;
import com.sun.jersey.api.client.Client;
import com.sun.jersey.api.client.ClientHandlerException;
import com.sun.jersey.api.client.ClientResponse;
import com.sun.jersey.api.client.ClientResponse.Status;
import com.sun.jersey.api.client.WebResource;
import com.sun.jersey.core.header.InBoundHeaders;
import com.sun.jersey.core.header.OutBoundHeaders;
public class TestThreadPoolRequestReplicator {
@BeforeClass

1
nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-cluster/src/test/java/org/apache/nifi/cluster/coordination/node/TestNodeClusterCoordinator.java
View File

@ -32,7 +32,6 @@ import java.util.Map;
import java.util.Set;
import java.util.concurrent.atomic.AtomicReference;
import java.util.stream.Collectors;
import org.apache.nifi.cluster.manager.exception.IllegalNodeDisconnectionException;
import org.apache.nifi.cluster.protocol.ConnectionRequest;
import org.apache.nifi.cluster.protocol.ConnectionResponse;

4
nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-cluster/src/test/java/org/apache/nifi/cluster/integration/Node.java
View File

@ -28,7 +28,6 @@ import java.util.Set;
import java.util.UUID;
import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.TimeUnit;
import org.apache.commons.lang3.builder.HashCodeBuilder;
import org.apache.nifi.authorization.Authorizer;
import org.apache.nifi.cluster.ReportedEvent;
@ -65,7 +64,6 @@ import org.apache.nifi.registry.VariableRegistry;
import org.apache.nifi.reporting.BulletinRepository;
import org.apache.nifi.reporting.Severity;
import org.apache.nifi.util.NiFiProperties;
import org.apache.nifi.web.Revision;
import org.apache.nifi.web.revision.RevisionManager;
import org.junit.Assert;
import org.mockito.Mockito;
@ -117,7 +115,7 @@ public class Node {
};
revisionManager = Mockito.mock(RevisionManager.class);
Mockito.when(revisionManager.getAllRevisions()).thenReturn(Collections.<Revision> emptyList());
Mockito.when(revisionManager.getAllRevisions()).thenReturn(Collections.emptyList());
electionManager = new CuratorLeaderElectionManager(4, nodeProperties);
}

4
nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/pom.xml
View File

@ -28,6 +28,10 @@
<groupId>org.apache.nifi</groupId>
<artifactId>nifi-framework-core-api</artifactId>
</dependency>
<dependency>
<groupId>org.apache.nifi</groupId>
<artifactId>nifi-properties-loader</artifactId>
</dependency>
<dependency>
<groupId>org.apache.nifi</groupId>
<artifactId>nifi-api</artifactId>

23
nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/connectable/StandardConnection.java
View File

@ -16,6 +16,17 @@
*/
package org.apache.nifi.connectable;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.UUID;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.atomic.AtomicLong;
import java.util.concurrent.atomic.AtomicReference;
import java.util.stream.Collectors;
import org.apache.commons.collections4.CollectionUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.lang3.builder.EqualsBuilder;
@ -41,18 +52,6 @@ import org.apache.nifi.processor.FlowFileFilter;
import org.apache.nifi.processor.Relationship;
import org.apache.nifi.provenance.ProvenanceEventRepository;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.UUID;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.atomic.AtomicLong;
import java.util.concurrent.atomic.AtomicReference;
import java.util.stream.Collectors;
/**
* Models a connection between connectable components. A connection may contain
* one or more relationships that map the source component to the destination

7
nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/FileSystemSwapManager.java
View File

@ -37,7 +37,6 @@ import java.util.List;
import java.util.Map;
import java.util.UUID;
import java.util.regex.Pattern;
import org.apache.nifi.controller.queue.FlowFileQueue;
import org.apache.nifi.controller.queue.QueueSize;
import org.apache.nifi.controller.repository.FlowFileRecord;
@ -351,7 +350,7 @@ public class FileSystemSwapManager implements FlowFileSwapManager {
out.flush();
}
logger.info("Successfully swapped out {} FlowFiles from {} to Swap File {}", new Object[]{toSwap.size(), queue, swapLocation});
logger.info("Successfully swapped out {} FlowFiles from {} to Swap File {}", toSwap.size(), queue, swapLocation);
return toSwap.size();
}
@ -399,8 +398,8 @@ public class FileSystemSwapManager implements FlowFileSwapManager {
}
} catch (final EOFException eof) {
final QueueSize queueSize = new QueueSize(numRecords, contentSize);
final SwapSummary summary = new StandardSwapSummary(queueSize, maxRecordId, Collections.<ResourceClaim>emptyList());
final SwapContents partialContents = new StandardSwapContents(summary, Collections.<FlowFileRecord>emptyList());
final SwapSummary summary = new StandardSwapSummary(queueSize, maxRecordId, Collections.emptyList());
final SwapContents partialContents = new StandardSwapContents(summary, Collections.emptyList());
throw new IncompleteSwapFileException(swapLocation, partialContents);
}

2
nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/cluster/ClusterProtocolHeartbeater.java
View File

@ -17,11 +17,9 @@
package org.apache.nifi.controller.cluster;
import java.io.IOException;
import java.util.List;
import java.util.Map;
import java.util.stream.Collectors;
import org.apache.nifi.cluster.coordination.ClusterCoordinator;
import org.apache.nifi.cluster.coordination.node.ClusterRoles;
import org.apache.nifi.cluster.coordination.node.NodeConnectionStatus;

1
nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/leader/election/CuratorLeaderElectionManager.java
View File

@ -18,7 +18,6 @@ package org.apache.nifi.controller.leader.election;
import java.util.HashMap;
import java.util.Map;
import org.apache.commons.lang3.StringUtils;
import org.apache.curator.RetryPolicy;
import org.apache.curator.framework.CuratorFramework;

2
nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/encrypt/StringEncryptor.java
View File

@ -17,8 +17,8 @@
package org.apache.nifi.encrypt;
import java.security.Security;
import org.apache.nifi.util.NiFiProperties;
import org.apache.commons.lang3.StringUtils;
import org.apache.nifi.util.NiFiProperties;
import org.bouncycastle.jce.provider.BouncyCastleProvider;
import org.jasypt.encryption.pbe.StandardPBEStringEncryptor;
import org.jasypt.exceptions.EncryptionInitializationException;

55
nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/remote/StandardRemoteProcessGroup.java
View File

@ -16,10 +16,34 @@
*/
package org.apache.nifi.remote;
import static java.util.Objects.requireNonNull;
import com.sun.jersey.api.client.ClientHandlerException;
import com.sun.jersey.api.client.ClientResponse;
import com.sun.jersey.api.client.ClientResponse.Status;
import com.sun.jersey.api.client.UniformInterfaceException;
import java.io.File;
import java.io.IOException;
import java.net.URI;
import java.net.URISyntaxException;
import java.util.ArrayList;
import java.util.Date;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator;
import java.util.LinkedHashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicReference;
import java.util.concurrent.locks.Lock;
import java.util.concurrent.locks.ReadWriteLock;
import java.util.concurrent.locks.ReentrantReadWriteLock;
import javax.net.ssl.SSLContext;
import javax.ws.rs.core.Response;
import org.apache.nifi.authorization.Resource;
import org.apache.nifi.authorization.resource.Authorizable;
import org.apache.nifi.authorization.resource.ResourceFactory;
@ -52,31 +76,6 @@ import org.apache.nifi.web.api.dto.PortDTO;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import javax.net.ssl.SSLContext;
import javax.ws.rs.core.Response;
import java.io.File;
import java.io.IOException;
import java.net.URI;
import java.net.URISyntaxException;
import java.util.ArrayList;
import java.util.Date;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator;
import java.util.LinkedHashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicReference;
import java.util.concurrent.locks.Lock;
import java.util.concurrent.locks.ReadWriteLock;
import java.util.concurrent.locks.ReentrantReadWriteLock;
import static java.util.Objects.requireNonNull;
/**
* Represents the Root Process Group of a remote NiFi Instance. Holds
* information about that remote instance, as well as {@link IncomingPort}s and
@ -837,7 +836,7 @@ public class StandardRemoteProcessGroup implements RemoteProcessGroup {
// perform the request
final ControllerDTO dto;
try (
final SiteToSiteRestApiClient apiClient = getSiteToSiteRestApiClient();) {
final SiteToSiteRestApiClient apiClient = getSiteToSiteRestApiClient()) {
dto = apiClient.getController();
} catch (IOException e) {
writeLock.lock();
@ -1202,8 +1201,8 @@ public class StandardRemoteProcessGroup implements RemoteProcessGroup {
if (Response.Status.Family.SUCCESSFUL.equals(requestAccountResponse.getStatusInfo().getFamily())) {
logger.info("{} Issued a Request to communicate with remote instance", this);
} else {
logger.error("{} Failed to request account: got unexpected response code of {}:{}", new Object[]{
this, requestAccountResponse.getStatus(), requestAccountResponse.getStatusInfo().getReasonPhrase()});
logger.error("{} Failed to request account: got unexpected response code of {}:{}", this,
requestAccountResponse.getStatus(), requestAccountResponse.getStatusInfo().getReasonPhrase());
}
} catch (final Exception re) {
logger.error("{} Failed to request account due to {}", this, re.toString());

12
nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/resources/nifi-context.xml
View File

@ -16,18 +16,10 @@
<beans default-lazy-init="true"
xmlns="http://www.springframework.org/schema/beans"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xmlns:util="http://www.springframework.org/schema/util"
xmlns:context="http://www.springframework.org/schema/context"
xmlns:aop="http://www.springframework.org/schema/aop"
xsi:schemaLocation="http://www.springframework.org/schema/beans http://www.springframework.org/schema/beans/spring-beans-3.1.xsd
http://www.springframework.org/schema/util http://www.springframework.org/schema/util/spring-util-3.1.xsd
http://www.springframework.org/schema/context http://www.springframework.org/schema/context/spring-context-3.1.xsd
http://www.springframework.org/schema/aop http://www.springframework.org/schema/aop/spring-aop-3.1.xsd">
xsi:schemaLocation="http://www.springframework.org/schema/beans http://www.springframework.org/schema/beans/spring-beans-3.1.xsd">
<!-- nifi properties created via getInstance using a file path specified as a system property -->
<bean id="nifiProperties" class="org.apache.nifi.util.NiFiProperties" factory-method="createBasicNiFiProperties">
<constructor-arg index="0"><null /></constructor-arg>
<constructor-arg index="1"><null /></constructor-arg>
<bean id="nifiProperties" class="org.apache.nifi.properties.NiFiPropertiesLoader" factory-method="loadDefaultWithKeyFromBootstrap">
</bean>
<!-- variable registry -->

33
nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/controller/TestFlowController.java
View File

@ -16,6 +16,21 @@
*/
package org.apache.nifi.controller;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertNotEquals;
import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertTrue;
import static org.mockito.Mockito.when;
import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
import java.nio.charset.StandardCharsets;
import java.util.HashMap;
import java.util.LinkedHashSet;
import java.util.Map;
import java.util.Set;
import org.apache.commons.io.IOUtils;
import org.apache.nifi.admin.service.AuditService;
import org.apache.nifi.authorization.AbstractPolicyBasedAuthorizer;
@ -35,29 +50,13 @@ import org.apache.nifi.processor.Relationship;
import org.apache.nifi.provenance.MockProvenanceRepository;
import org.apache.nifi.registry.VariableRegistry;
import org.apache.nifi.reporting.BulletinRepository;
import org.apache.nifi.util.FileBasedVariableRegistry;
import org.apache.nifi.util.NiFiProperties;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
import org.mockito.Mockito;
import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
import java.nio.charset.StandardCharsets;
import java.util.HashMap;
import java.util.LinkedHashSet;
import java.util.Map;
import java.util.Set;
import org.apache.nifi.util.FileBasedVariableRegistry;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertNotEquals;
import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertTrue;
import static org.mockito.Mockito.when;
public class TestFlowController {
private FlowController controller;

1
nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/controller/scheduling/TestStandardProcessScheduler.java
View File

@ -29,7 +29,6 @@ import java.util.concurrent.Executors;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicInteger;
import org.apache.nifi.annotation.lifecycle.OnDisabled;
import org.apache.nifi.annotation.lifecycle.OnEnabled;
import org.apache.nifi.annotation.lifecycle.OnScheduled;

9
nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/controller/service/TestStandardControllerServiceProvider.java
View File

@ -29,7 +29,6 @@ import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.UUID;
import org.apache.nifi.components.state.StateManager;
import org.apache.nifi.components.state.StateManagerProvider;
import org.apache.nifi.controller.FlowController;
@ -430,7 +429,7 @@ public class TestStandardControllerServiceProvider {
E.setProperty(ServiceA.OTHER_SERVICE.getName(), "A");
E.setProperty(ServiceA.OTHER_SERVICE_2.getName(), "F");
provider.enableControllerServices(Arrays.asList(new ControllerServiceNode[]{A, B, C, D, E, F}));
provider.enableControllerServices(Arrays.asList(A, B, C, D, E, F));
assertTrue(A.isActive());
assertTrue(B.isActive());
@ -473,7 +472,7 @@ public class TestStandardControllerServiceProvider {
F.setProperty(ServiceA.OTHER_SERVICE.getName(), "D");
D.setProperty(ServiceA.OTHER_SERVICE.getName(), "C");
provider.enableControllerServices(Arrays.asList(new ControllerServiceNode[]{C, F, A, B, D}));
provider.enableControllerServices(Arrays.asList(C, F, A, B, D));
assertTrue(A.isActive());
assertTrue(B.isActive());
@ -516,7 +515,7 @@ public class TestStandardControllerServiceProvider {
serviceNode7.setProperty(ServiceC.REQ_SERVICE_2.getName(), "3");
provider.enableControllerServices(Arrays.asList(
new ControllerServiceNode[]{serviceNode1, serviceNode2, serviceNode3, serviceNode4, serviceNode5, serviceNode7}));
serviceNode1, serviceNode2, serviceNode3, serviceNode4, serviceNode5, serviceNode7));
assertFalse(serviceNode1.isActive());
assertFalse(serviceNode2.isActive());
assertFalse(serviceNode3.isActive());
@ -526,7 +525,7 @@ public class TestStandardControllerServiceProvider {
provider.enableControllerService(serviceNode6);
provider.enableControllerServices(Arrays.asList(
new ControllerServiceNode[]{serviceNode1, serviceNode2, serviceNode3, serviceNode4, serviceNode5}));
serviceNode1, serviceNode2, serviceNode3, serviceNode4, serviceNode5));
assertTrue(serviceNode1.isActive());
assertTrue(serviceNode2.isActive());

16
nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-nar-utils/src/main/java/org/apache/nifi/nar/NarThreadContextClassLoader.java
View File

@ -173,14 +173,14 @@ public class NarThreadContextClassLoader extends URLClassLoader {
* constructor or a constructor which takes a NiFiProperties object
* (preferred).
*
* @param <T> type
* @param implementationClassName class
* @param typeDefinition def
* @param nifiProperties props
* @param <T> the type to create an instance for
* @param implementationClassName the implementation class name
* @param typeDefinition the type definition
* @param nifiProperties the NiFiProperties instance
* @return constructed instance
* @throws InstantiationException ex
* @throws IllegalAccessException ex
* @throws ClassNotFoundException ex
* @throws InstantiationException if there is an error instantiating the class
* @throws IllegalAccessException if there is an error accessing the type
* @throws ClassNotFoundException if the class cannot be found
*/
public static <T> T createInstance(final String implementationClassName, final Class<T> typeDefinition, final NiFiProperties nifiProperties)
throws InstantiationException, IllegalAccessException, ClassNotFoundException {
@ -199,7 +199,7 @@ public class NarThreadContextClassLoader extends URLClassLoader {
Thread.currentThread().setContextClassLoader(detectedClassLoaderForType);
final Class<?> desiredClass = rawClass.asSubclass(typeDefinition);
if (nifiProperties == null) {
if(nifiProperties == null){
return typeDefinition.cast(desiredClass.newInstance());
}
Constructor<?> constructor = null;

53
nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-properties-loader/pom.xml Normal file
View File

@ -0,0 +1,53 @@
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<!--
Licensed to the Apache Software Foundation (ASF) under one or more
contributor license agreements. See the NOTICE file distributed with
this work for additional information regarding copyright ownership.
The ASF licenses this file to You under the Apache License, Version 2.0
(the "License"); you may not use this file except in compliance with
the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
-->
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>org.apache.nifi</groupId>
<artifactId>nifi-framework</artifactId>
<version>1.0.0-SNAPSHOT</version>
</parent>
<artifactId>nifi-properties-loader</artifactId>
<name>NiFi Properties Loader</name>
<description>Handles the loading of the nifi.properties file to an instance of NiFiProperties, and transparently
performs any decryption/retrieval of sensitive configuration properties.
</description>
<packaging>jar</packaging>
<dependencies>
<dependency>
<groupId>org.apache.nifi</groupId>
<artifactId>nifi-properties</artifactId>
</dependency>
<dependency>
<groupId>org.bouncycastle</groupId>
<artifactId>bcprov-jdk15on</artifactId>
</dependency>
<dependency>
<groupId>org.slf4j</groupId>
<artifactId>jul-to-slf4j</artifactId>
</dependency>
<dependency>
<groupId>ch.qos.logback</groupId>
<artifactId>logback-classic</artifactId>
</dependency>
<dependency>
<groupId>org.apache.commons</groupId>
<artifactId>commons-lang3</artifactId>
</dependency>
</dependencies>
</project>

251
nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-properties-loader/src/main/java/org/apache/nifi/properties/AESSensitivePropertyProvider.java Normal file
View File

@ -0,0 +1,251 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.nifi.properties;
import java.nio.charset.StandardCharsets;
import java.security.InvalidAlgorithmParameterException;
import java.security.InvalidKeyException;
import java.security.NoSuchAlgorithmException;
import java.security.NoSuchProviderException;
import java.security.SecureRandom;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import java.util.stream.Collectors;
import javax.crypto.BadPaddingException;
import javax.crypto.Cipher;
import javax.crypto.IllegalBlockSizeException;
import javax.crypto.NoSuchPaddingException;
import javax.crypto.SecretKey;
import javax.crypto.spec.IvParameterSpec;
import javax.crypto.spec.SecretKeySpec;
import org.apache.commons.lang3.StringUtils;
import org.bouncycastle.util.encoders.Base64;
import org.bouncycastle.util.encoders.DecoderException;
import org.bouncycastle.util.encoders.EncoderException;
import org.bouncycastle.util.encoders.Hex;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
public class AESSensitivePropertyProvider implements SensitivePropertyProvider {
private static final Logger logger = LoggerFactory.getLogger(AESSensitivePropertyProvider.class);
private static final String IMPLEMENTATION_NAME = "AES Sensitive Property Provider";
private static final String IMPLEMENTATION_KEY = "aes/gcm/";
private static final String ALGORITHM = "AES/GCM/NoPadding";
private static final String PROVIDER = "BC";
private static final String DELIMITER = "||"; // "|" is not a valid Base64 character, so ensured not to be present in cipher text
private static final int IV_LENGTH = 12;
private static final int MIN_CIPHER_TEXT_LENGTH = IV_LENGTH * 4 / 3 + DELIMITER.length() + 1;
private Cipher cipher;
private final SecretKey key;
public AESSensitivePropertyProvider(String keyHex) throws NoSuchPaddingException, NoSuchAlgorithmException, NoSuchProviderException {
byte[] key = validateKey(keyHex);
try {
cipher = Cipher.getInstance(ALGORITHM, PROVIDER);
// Only store the key if the cipher was initialized successfully
this.key = new SecretKeySpec(key, "AES");
} catch (NoSuchAlgorithmException | NoSuchProviderException | NoSuchPaddingException e) {
logger.error("Encountered an error initializing the {}: {}", IMPLEMENTATION_NAME, e.getMessage());
throw new SensitivePropertyProtectionException("Error initializing the protection cipher", e);
}
}
private byte[] validateKey(String keyHex) {
if (keyHex == null || StringUtils.isBlank(keyHex)) {
throw new SensitivePropertyProtectionException("The key cannot be empty");
}
keyHex = formatHexKey(keyHex);
if (!isHexKeyValid(keyHex)) {
throw new SensitivePropertyProtectionException("The key must be a valid hexadecimal key");
}
byte[] key = Hex.decode(keyHex);
final List<Integer> validKeyLengths = getValidKeyLengths();
if (!validKeyLengths.contains(key.length * 8)) {
List<String> validKeyLengthsAsStrings = validKeyLengths.stream().map(i -> Integer.toString(i)).collect(Collectors.toList());
throw new SensitivePropertyProtectionException("The key (" + key.length * 8 + " bits) must be a valid length: " + StringUtils.join(validKeyLengthsAsStrings, ", "));
}
return key;
}
public AESSensitivePropertyProvider(byte[] key) throws NoSuchPaddingException, NoSuchAlgorithmException, NoSuchProviderException {
this(key == null ? "" : Hex.toHexString(key));
}
private static String formatHexKey(String input) {
if (input == null || StringUtils.isBlank(input)) {
return "";
}
return input.replaceAll("[^0-9a-fA-F]", "").toLowerCase();
}
private static boolean isHexKeyValid(String key) {
if (key == null || StringUtils.isBlank(key)) {
return false;
}
// Key length is in "nibbles" (i.e. one hex char = 4 bits)
return getValidKeyLengths().contains(key.length() * 4) && key.matches("^[0-9a-fA-F]*$");
}
private static List<Integer> getValidKeyLengths() {
List<Integer> validLengths = new ArrayList<>();
validLengths.add(128);
try {
if (Cipher.getMaxAllowedKeyLength("AES") > 128) {
validLengths.add(192);
validLengths.add(256);
} else {
logger.warn("JCE Unlimited Strength Cryptography Jurisdiction policies are not available, so the max key length is 128 bits");
}
} catch (NoSuchAlgorithmException e) {
logger.warn("Encountered an error determining the max key length", e);
}
return validLengths;
}
/**
* Returns the name of the underlying implementation.
*
* @return the name of this sensitive property provider
*/
@Override
public String getName() {
return IMPLEMENTATION_NAME;
}
/**
* Returns the key used to identify the provider implementation in {@code nifi.properties}.
*
* @return the key to persist in the sibling property
*/
@Override
public String getIdentifierKey() {
return IMPLEMENTATION_KEY + Collections.max(getValidKeyLengths()).toString();
}
/**
* Returns the encrypted cipher text.
*
* @param unprotectedValue the sensitive value
* @return the value to persist in the {@code nifi.properties} file
* @throws SensitivePropertyProtectionException if there is an exception encrypting the value
*/
@Override
public String protect(String unprotectedValue) throws SensitivePropertyProtectionException {
if (unprotectedValue == null || unprotectedValue.trim().length() == 0) {
throw new IllegalArgumentException("Cannot encrypt an empty value");
}
// Generate IV
byte[] iv = generateIV();
if (iv.length < IV_LENGTH) {
throw new IllegalArgumentException("The IV (" + iv.length + " bytes) must be at least " + IV_LENGTH + " bytes");
}
try {
// Initialize cipher for encryption
cipher.init(Cipher.ENCRYPT_MODE, this.key, new IvParameterSpec(iv));
byte[] plainBytes = unprotectedValue.getBytes(StandardCharsets.UTF_8);
byte[] cipherBytes = cipher.doFinal(plainBytes);
logger.info(getName() + " encrypted a sensitive value successfully");
return base64Encode(iv) + DELIMITER + base64Encode(cipherBytes);
// return Base64.toBase64String(iv) + DELIMITER + Base64.toBase64String(cipherBytes);
} catch (BadPaddingException | IllegalBlockSizeException | EncoderException | InvalidAlgorithmParameterException | InvalidKeyException e) {
final String msg = "Error encrypting a protected value";
logger.error(msg, e);
throw new SensitivePropertyProtectionException(msg, e);
}
}
private String base64Encode(byte[] input) {
return Base64.toBase64String(input).replaceAll("=", "");
}
/**
* Generates a new random IV of 12 bytes using {@link java.security.SecureRandom}.
*
* @return the IV
*/
private byte[] generateIV() {
byte[] iv = new byte[IV_LENGTH];
new SecureRandom().nextBytes(iv);
return iv;
}
/**
* Returns the decrypted plaintext.
*
* @param protectedValue the cipher text read from the {@code nifi.properties} file
* @return the raw value to be used by the application
* @throws SensitivePropertyProtectionException if there is an error decrypting the cipher text
*/
@Override
public String unprotect(String protectedValue) throws SensitivePropertyProtectionException {
if (protectedValue == null || protectedValue.trim().length() < MIN_CIPHER_TEXT_LENGTH) {
throw new IllegalArgumentException("Cannot decrypt a cipher text shorter than " + MIN_CIPHER_TEXT_LENGTH + " chars");
}
if (!protectedValue.contains(DELIMITER)) {
throw new IllegalArgumentException("The cipher text does not contain the delimiter " + DELIMITER + " -- it should be of the form Base64(IV) || Base64(cipherText)");
}
final String IV_B64 = protectedValue.substring(0, protectedValue.indexOf(DELIMITER));
byte[] iv = Base64.decode(IV_B64);
if (iv.length < IV_LENGTH) {
throw new IllegalArgumentException("The IV (" + iv.length + " bytes) must be at least " + IV_LENGTH + " bytes");
}
String CIPHERTEXT_B64 = protectedValue.substring(protectedValue.indexOf(DELIMITER) + 2);
// Restore the = padding if necessary to reconstitute the GCM MAC check
if (CIPHERTEXT_B64.length() % 4 != 0) {
final int paddedLength = CIPHERTEXT_B64.length() + 4 - (CIPHERTEXT_B64.length() % 4);
CIPHERTEXT_B64 = StringUtils.rightPad(CIPHERTEXT_B64, paddedLength, '=');
}
try {
byte[] cipherBytes = Base64.decode(CIPHERTEXT_B64);
cipher.init(Cipher.DECRYPT_MODE, this.key, new IvParameterSpec(iv));
byte[] plainBytes = cipher.doFinal(cipherBytes);
logger.info(getName() + " decrypted a sensitive value successfully");
return new String(plainBytes, StandardCharsets.UTF_8);
} catch (BadPaddingException | IllegalBlockSizeException | DecoderException | InvalidAlgorithmParameterException | InvalidKeyException e) {
final String msg = "Error decrypting a protected value";
logger.error(msg, e);
throw new SensitivePropertyProtectionException(msg, e);
}
}
public static int getIvLength() {
return IV_LENGTH;
}
public static int getMinCipherTextLength() {
return MIN_CIPHER_TEXT_LENGTH;
}
public static String getDelimiter() {
return DELIMITER;
}
}

53
nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-properties-loader/src/main/java/org/apache/nifi/properties/AESSensitivePropertyProviderFactory.java Normal file
View File

@ -0,0 +1,53 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.nifi.properties;
import java.security.NoSuchAlgorithmException;
import java.security.NoSuchProviderException;
import javax.crypto.NoSuchPaddingException;
import org.apache.commons.lang3.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
public class AESSensitivePropertyProviderFactory implements SensitivePropertyProviderFactory {
private static final Logger logger = LoggerFactory.getLogger(AESSensitivePropertyProviderFactory.class);
private String keyHex;
public AESSensitivePropertyProviderFactory(String keyHex) {
this.keyHex = keyHex;
}
public SensitivePropertyProvider getProvider() throws SensitivePropertyProtectionException {
try {
if (keyHex != null && !StringUtils.isBlank(keyHex)) {
return new AESSensitivePropertyProvider(keyHex);
} else {
throw new SensitivePropertyProtectionException("The provider factory cannot generate providers without a key");
}
} catch (NoSuchAlgorithmException | NoSuchProviderException | NoSuchPaddingException e) {
String msg = "Error creating AES Sensitive Property Provider";
logger.warn(msg, e);
throw new SensitivePropertyProtectionException(msg, e);
}
}
@Override
public String toString() {
return "SensitivePropertyProviderFactory for creating AESSensitivePropertyProviders";
}
}

128
nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-properties-loader/src/main/java/org/apache/nifi/properties/MultipleSensitivePropertyProtectionException.java Normal file
View File

@ -0,0 +1,128 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.nifi.properties;
import java.util.Collection;
import java.util.HashSet;
import java.util.Set;
import org.apache.commons.lang3.StringUtils;
public class MultipleSensitivePropertyProtectionException extends SensitivePropertyProtectionException {
private Set<String> failedKeys;
/**
* Constructs a new throwable with {@code null} as its detail message.
* The cause is not initialized, and may subsequently be initialized by a
* call to {@link #initCause}.
* <p>
* <p>The {@link #fillInStackTrace()} method is called to initialize
* the stack trace data in the newly created throwable.
*/
public MultipleSensitivePropertyProtectionException() {
}
/**
* Constructs a new throwable with the specified detail message. The
* cause is not initialized, and may subsequently be initialized by
* a call to {@link #initCause}.
* <p>
* <p>The {@link #fillInStackTrace()} method is called to initialize
* the stack trace data in the newly created throwable.
*
* @param message the detail message. The detail message is saved for
* later retrieval by the {@link #getMessage()} method.
*/
public MultipleSensitivePropertyProtectionException(String message) {
super(message);
}
/**
* Constructs a new throwable with the specified detail message and
* cause. <p>Note that the detail message associated with
* {@code cause} is <i>not</i> automatically incorporated in
* this throwable's detail message.
* <p>
* <p>The {@link #fillInStackTrace()} method is called to initialize
* the stack trace data in the newly created throwable.
*
* @param message the detail message (which is saved for later retrieval
* by the {@link #getMessage()} method).
* @param cause the cause (which is saved for later retrieval by the
* {@link #getCause()} method). (A {@code null} value is
* permitted, and indicates that the cause is nonexistent or
* unknown.)
* @since 1.4
*/
public MultipleSensitivePropertyProtectionException(String message, Throwable cause) {
super(message, cause);
}
/**
* Constructs a new throwable with the specified cause and a detail
* message of {@code (cause==null ? null : cause.toString())} (which
* typically contains the class and detail message of {@code cause}).
* This constructor is useful for throwables that are little more than
* wrappers for other throwables (for example, PrivilegedActionException).
* <p>
* <p>The {@link #fillInStackTrace()} method is called to initialize
* the stack trace data in the newly created throwable.
*
* @param cause the cause (which is saved for later retrieval by the
* {@link #getCause()} method). (A {@code null} value is
* permitted, and indicates that the cause is nonexistent or
* unknown.)
* @since 1.4
*/
public MultipleSensitivePropertyProtectionException(Throwable cause) {
super(cause);
}
/**
* Constructs a new exception with the provided message and a unique set of the keys that caused the error.
*
* @param message the message
* @param failedKeys any failed keys
*/
public MultipleSensitivePropertyProtectionException(String message, Collection<String> failedKeys) {
this(message, failedKeys, null);
}
/**
* Constructs a new exception with the provided message and a unique set of the keys that caused the error.
*
* @param message the message
* @param failedKeys any failed keys
* @param cause the cause (which is saved for later retrieval by the
* {@link #getCause()} method). (A {@code null} value is
* permitted, and indicates that the cause is nonexistent or
* unknown.)
*/
public MultipleSensitivePropertyProtectionException(String message, Collection<String> failedKeys, Throwable cause) {
super(message, cause);
this.failedKeys = new HashSet<>(failedKeys);
}
public Set<String> getFailedKeys() {
return this.failedKeys;
}
@Override
public String toString() {
return "SensitivePropertyProtectionException for [" + StringUtils.join(this.failedKeys, ", ") + "]: " + getLocalizedMessage();
}
}

254
nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-properties-loader/src/main/java/org/apache/nifi/properties/NiFiPropertiesLoader.java Normal file
View File

@ -0,0 +1,254 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.nifi.properties;
import java.io.BufferedInputStream;
import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.nio.file.Files;
import java.nio.file.Paths;
import java.security.NoSuchAlgorithmException;
import java.security.Security;
import java.util.Optional;
import java.util.Properties;
import java.util.stream.Stream;
import javax.crypto.Cipher;
import org.apache.nifi.util.NiFiProperties;
import org.bouncycastle.jce.provider.BouncyCastleProvider;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
public class NiFiPropertiesLoader {
private static final Logger logger = LoggerFactory.getLogger(NiFiPropertiesLoader.class);
private static final String RELATIVE_PATH = "conf/nifi.properties";
private static final String BOOTSTRAP_KEY_PREFIX = "nifi.bootstrap.sensitive.key=";
private NiFiProperties instance;
private String keyHex;
// Future enhancement: allow for external registration of new providers
private static SensitivePropertyProviderFactory sensitivePropertyProviderFactory;
public NiFiPropertiesLoader() {
}
/**
* Returns an instance of the loader configured with the key.
*
* @param keyHex the key used to encrypt any sensitive properties
* @return the configured loader
*/
public static NiFiPropertiesLoader withKey(String keyHex) {
NiFiPropertiesLoader loader = new NiFiPropertiesLoader();
loader.setKeyHex(keyHex);
return loader;
}
/**
* Sets the hexadecimal key used to unprotect properties encrypted with
* {@link AESSensitivePropertyProvider}. If the key has already been set,
* calling this method will throw a {@link RuntimeException}.
*
* @param keyHex the key in hexadecimal format
*/
public void setKeyHex(String keyHex) {
if (this.keyHex == null || this.keyHex.trim().isEmpty()) {
this.keyHex = keyHex;
} else {
throw new RuntimeException("Cannot overwrite an existing key");
}
}
/**
* Returns a {@link NiFiProperties} instance with any encrypted properties
* decrypted using the key from the {@code conf/bootstrap.conf} file. This
* method is exposed to allow Spring factory-method loading at application
* startup.
*
* @return the populated and decrypted NiFiProperties instance
* @throws IOException if there is a problem reading from the bootstrap.conf or nifi.properties files
*/
public static NiFiProperties loadDefaultWithKeyFromBootstrap() throws IOException {
try {
String keyHex = extractKeyFromBootstrapFile();
return NiFiPropertiesLoader.withKey(keyHex).loadDefault();
} catch (IOException e) {
logger.error("Encountered an exception loading the default nifi.properties file {} with the key provided in bootstrap.conf", getDefaultFilePath(), e);
throw e;
}
}
private static String extractKeyFromBootstrapFile() throws IOException {
// Guess at location of bootstrap.conf file from nifi.properties file
String defaultNiFiPropertiesPath = getDefaultFilePath();
File propertiesFile = new File(defaultNiFiPropertiesPath);
File confDir = new File(propertiesFile.getParent());
if (confDir.exists() && confDir.canRead()) {
File expectedBootstrapFile = new File(confDir, "bootstrap.conf");
if (expectedBootstrapFile.exists() && expectedBootstrapFile.canRead()) {
try (Stream<String> stream = Files.lines(Paths.get(expectedBootstrapFile.getAbsolutePath()))) {
Optional<String> keyLine = stream.filter(l -> l.startsWith(BOOTSTRAP_KEY_PREFIX)).findFirst();
if (keyLine.isPresent()) {
return keyLine.get().split("=", 2)[1];
} else {
logger.warn("No encryption key present in the bootstrap.conf file at {}", expectedBootstrapFile.getAbsolutePath());
return "";
}
} catch (IOException e) {
logger.error("Cannot read from bootstrap.conf file at {} to extract encryption key", expectedBootstrapFile.getAbsolutePath());
throw new IOException("Cannot read from bootstrap.conf", e);
}
} else {
logger.error("Cannot read from bootstrap.conf file at {} to extract encryption key -- file is missing or permissions are incorrect", expectedBootstrapFile.getAbsolutePath());
throw new IOException("Cannot read from bootstrap.conf");
}
} else {
logger.error("Cannot read from bootstrap.conf file at {} to extract encryption key -- conf/ directory is missing or permissions are incorrect", confDir.getAbsolutePath());
throw new IOException("Cannot read from bootstrap.conf");
}
}
private static String getDefaultFilePath() {
String systemPath = System.getProperty(NiFiProperties.PROPERTIES_FILE_PATH);
if (systemPath == null || systemPath.trim().isEmpty()) {
logger.warn("The system variable {} is not set, so it is being set to '{}'", NiFiProperties.PROPERTIES_FILE_PATH, RELATIVE_PATH);
System.setProperty(NiFiProperties.PROPERTIES_FILE_PATH, RELATIVE_PATH);
systemPath = RELATIVE_PATH;
}
logger.info("Determined default nifi.properties path to be '{}'", systemPath);
return systemPath;
}
private NiFiProperties loadDefault() {
return load(getDefaultFilePath());
}
private static String getDefaultProviderKey() {
try {
return "aes/gcm/" + (Cipher.getMaxAllowedKeyLength("AES") > 128 ? "256" : "128");
} catch (NoSuchAlgorithmException e) {
return "aes/gcm/128";
}
}
private void initializeSensitivePropertyProviderFactory() {
if (sensitivePropertyProviderFactory == null) {
sensitivePropertyProviderFactory = new AESSensitivePropertyProviderFactory(keyHex);
}
}
private SensitivePropertyProvider getSensitivePropertyProvider() {
initializeSensitivePropertyProviderFactory();
return sensitivePropertyProviderFactory.getProvider();
}
/**
* Returns a {@link ProtectedNiFiProperties} instance loaded from the serialized
* form in the file. Responsible for actually reading from disk and deserializing
* the properties. Returns a protected instance to allow for decryption operations.
*
* @param file the file containing serialized properties
* @return the ProtectedNiFiProperties instance
*/
ProtectedNiFiProperties readProtectedPropertiesFromDisk(File file) {
if (file == null || !file.exists() || !file.canRead()) {
String path = (file == null ? "missing file" : file.getAbsolutePath());
logger.error("Cannot read from '{}' -- file is missing or not readable", path);
throw new IllegalArgumentException("NiFi properties file missing or unreadable");
}
Properties rawProperties = new Properties();
InputStream inStream = null;
try {
inStream = new BufferedInputStream(new FileInputStream(file));
rawProperties.load(inStream);
logger.info("Loaded {} properties from {}", rawProperties.size(), file.getAbsolutePath());
ProtectedNiFiProperties protectedNiFiProperties = new ProtectedNiFiProperties(rawProperties);
return protectedNiFiProperties;
} catch (final Exception ex) {
logger.error("Cannot load properties file due to " + ex.getLocalizedMessage());
throw new RuntimeException("Cannot load properties file due to "
+ ex.getLocalizedMessage(), ex);
} finally {
if (null != inStream) {
try {
inStream.close();
} catch (final Exception ex) {
/**
* do nothing *
*/
}
}
}
}
/**
* Returns an instance of {@link NiFiProperties} loaded from the provided
* {@link File}. If any properties are protected, will attempt to use the
* appropriate {@link SensitivePropertyProvider} to unprotect them transparently.
*
* @param file the File containing the serialized properties
* @return the NiFiProperties instance
*/
public NiFiProperties load(File file) {
ProtectedNiFiProperties protectedNiFiProperties = readProtectedPropertiesFromDisk(file);
if (protectedNiFiProperties.hasProtectedKeys()) {
Security.addProvider(new BouncyCastleProvider());
protectedNiFiProperties.addSensitivePropertyProvider(getSensitivePropertyProvider());
}
return protectedNiFiProperties.getUnprotectedProperties();
}
/**
* Returns an instance of {@link NiFiProperties}. If the path is empty, this
* will load the default properties file as specified by
* {@code NiFiProperties.PROPERTY_FILE_PATH}.
*
* @param path the path of the serialized properties file
* @return the NiFiProperties instance
* @see NiFiPropertiesLoader#load(File)
*/
public NiFiProperties load(String path) {
if (path != null && !path.trim().isEmpty()) {
return load(new File(path));
} else {
return loadDefault();
}
}
/**
* Returns the loaded {@link NiFiProperties} instance. If none is currently loaded, attempts to load the default instance.
*
* @return the current NiFiProperties instance
*/
public NiFiProperties get() {
if (instance == null) {
instance = loadDefault();
}
return instance;
}
}

521
nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-properties-loader/src/main/java/org/apache/nifi/properties/ProtectedNiFiProperties.java Normal file
View File

@ -0,0 +1,521 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.nifi.properties;
import static java.util.Arrays.asList;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Properties;
import java.util.Set;
import org.apache.commons.lang3.StringUtils;
import org.apache.nifi.util.NiFiProperties;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
/**
* Decorator class for intermediate phase when {@link NiFiPropertiesLoader} loads the
* raw properties file and performs unprotection activities before returning a clean
* implementation of {@link NiFiProperties}, likely {@link StandardNiFiProperties}.
* This encapsulates the sensitive property access logic from external consumers
* of {@code NiFiProperties}.
*/
class ProtectedNiFiProperties extends StandardNiFiProperties {
private static final Logger logger = LoggerFactory.getLogger(ProtectedNiFiProperties.class);
private NiFiProperties niFiProperties;
private Map<String, SensitivePropertyProvider> localProviderCache = new HashMap<>();
// Additional "sensitive" property key
public static final String ADDITIONAL_SENSITIVE_PROPERTIES_KEY = "nifi.sensitive.props.additional.keys";
// Default list of "sensitive" property keys
public static final List<String> DEFAULT_SENSITIVE_PROPERTIES = new ArrayList<>(asList(SECURITY_KEY_PASSWD,
SECURITY_KEYSTORE_PASSWD, SECURITY_TRUSTSTORE_PASSWD, SENSITIVE_PROPS_KEY));
public ProtectedNiFiProperties() {
this(new StandardNiFiProperties());
}
/**
* Creates an instance containing the provided {@link NiFiProperties}.
*
* @param props the NiFiProperties to contain
*/
public ProtectedNiFiProperties(NiFiProperties props) {
this.niFiProperties = props;
logger.debug("Loaded {} properties (including {} protection schemes) into ProtectedNiFiProperties", getPropertyKeysIncludingProtectionSchemes().size(), getProtectedPropertyKeys().size());
}
/**
* Creates an instance containing the provided raw {@link Properties}.
*
* @param rawProps the Properties to contain
*/
public ProtectedNiFiProperties(Properties rawProps) {
this(new StandardNiFiProperties(rawProps));
}
/**
* Retrieves the property value for the given property key.
*
* @param key the key of property value to lookup
* @return value of property at given key or null if not found
*/
@Override
public String getProperty(String key) {
return getInternalNiFiProperties().getProperty(key);
}
/**
* Retrieves all known property keys.
*
* @return all known property keys
*/
@Override
public Set<String> getPropertyKeys() {
Set<String> filteredKeys = getPropertyKeysIncludingProtectionSchemes();
filteredKeys.removeIf(p -> p.endsWith(".protected"));
return filteredKeys;
}
/**
* Returns the internal representation of the {@link NiFiProperties} -- protected
* or not as determined by the current state. No guarantee is made to the
* protection state of these properties. If the internal reference is null, a new
* {@link StandardNiFiProperties} instance is created.
*
* @return the internal properties
*/
NiFiProperties getInternalNiFiProperties() {
if (this.niFiProperties == null) {
this.niFiProperties = new StandardNiFiProperties();
}
return this.niFiProperties;
}
/**
* Returns the number of properties, excluding protection scheme properties.
* <p>
* Example:
* <p>
* key: E(value, key)
* key.protected: aes/gcm/256
* key2: value2
* <p>
* would return size 2
*
* @return the count of real properties
*/
@Override
public int size() {
return getPropertyKeys().size();
}
/**
* Returns the complete set of property keys, including any protection keys (i.e. 'x.y.z.protected').
*
* @return the set of property keys
*/
Set<String> getPropertyKeysIncludingProtectionSchemes() {
return getInternalNiFiProperties().getPropertyKeys();
}
/**
* Splits a single string containing multiple property keys into a List. Delimited by ',' or ';' and ignores leading and trailing whitespace around delimiter.
*
* @param multipleProperties a single String containing multiple properties, i.e. "nifi.property.1; nifi.property.2, nifi.property.3"
* @return a List containing the split and trimmed properties
*/
private static List<String> splitMultipleProperties(String multipleProperties) {
if (multipleProperties == null || multipleProperties.trim().isEmpty()) {
return new ArrayList<>(0);
} else {
List<String> properties = new ArrayList<>(asList(multipleProperties.split("\\s*[,;]\\s*")));
for (int i = 0; i < properties.size(); i++) {
properties.set(i, properties.get(i).trim());
}
return properties;
}
}
/**
* Returns a list of the keys identifying "sensitive" properties. There is a default list,
* and additional keys can be provided in the {@code nifi.sensitive.props.additional.keys} property in {@code nifi.properties}.
*
* @return the list of sensitive property keys
*/
public List<String> getSensitivePropertyKeys() {
String additionalPropertiesString = getProperty(ADDITIONAL_SENSITIVE_PROPERTIES_KEY);
if (additionalPropertiesString == null || additionalPropertiesString.trim().isEmpty()) {
return DEFAULT_SENSITIVE_PROPERTIES;
} else {
List<String> additionalProperties = splitMultipleProperties(additionalPropertiesString);
/* Remove this key if it was accidentally provided as a sensitive key
* because we cannot protect it and read from it
*/
if (additionalProperties.contains(ADDITIONAL_SENSITIVE_PROPERTIES_KEY)) {
logger.warn("The key '{}' contains itself. This is poor practice and should be removed", ADDITIONAL_SENSITIVE_PROPERTIES_KEY);
additionalProperties.remove(ADDITIONAL_SENSITIVE_PROPERTIES_KEY);
}
additionalProperties.addAll(DEFAULT_SENSITIVE_PROPERTIES);
return additionalProperties;
}
}
/**
* Returns true if any sensitive keys are protected.
*
* @return true if any key is protected; false otherwise
*/
public boolean hasProtectedKeys() {
List<String> sensitiveKeys = getSensitivePropertyKeys();
for (String k : sensitiveKeys) {
if (isPropertyProtected(k)) {
return true;
}
}
return false;
}
/**
* Returns a Map of the keys identifying "sensitive" properties that are currently protected and the "protection" key for each. This may or may not include all properties marked as sensitive.
*
* @return the Map of protected property keys and the protection identifier for each
*/
public Map<String, String> getProtectedPropertyKeys() {
List<String> sensitiveKeys = getSensitivePropertyKeys();
// This is the Java 8 way, but can likely be optimized (and not sure of correctness)
// Map<String, String> protectedProperties = sensitiveKeys.stream().filter(key ->
// getProperty(getProtectionKey(key)) != null).collect(Collectors.toMap(Function.identity(), key ->
// getProperty(getProtectionKey(key))));
// Groovy
// Map<String, String> groovyProtectedProperties = sensitiveKeys.collectEntries { key ->
// [(key): getProperty(getProtectionKey(key))] }.findAll { k, v -> v }
// Traditional way
Map<String, String> traditionalProtectedProperties = new HashMap<>();
for (String key : sensitiveKeys) {
String protection = getProperty(getProtectionKey(key));
if (!StringUtils.isBlank(protection)) {
traditionalProtectedProperties.put(key, protection);
}
}
return traditionalProtectedProperties;
}
/**
* Returns the unique set of all protection schemes currently in use for this instance.
*
* @return the set of protection schemes
*/
public Set<String> getProtectionSchemes() {
return new HashSet<>(getProtectedPropertyKeys().values());
}
/**
* Returns a percentage of the total number of properties marked as sensitive that are currently protected.
*
* @return the percent of sensitive properties marked as protected
*/
public int getPercentOfSensitivePropertiesProtected() {
return (int) Math.round(getProtectedPropertyKeys().size() / ((double) getSensitivePropertyKeys().size()) * 100);
}
/**
* Returns true if the property identified by this key is considered sensitive in this instance of {@code NiFiProperties}.
* Some properties are sensitive by default, while others can be specified by
* {@link ProtectedNiFiProperties#ADDITIONAL_SENSITIVE_PROPERTIES_KEY}.
*
* @param key the key
* @return true if it is sensitive
* @see ProtectedNiFiProperties#getSensitivePropertyKeys()
*/
public boolean isPropertySensitive(String key) {
// If the explicit check for ADDITIONAL_SENSITIVE_PROPERTIES_KEY is not here, this will loop infinitely
return key != null && !key.equals(ADDITIONAL_SENSITIVE_PROPERTIES_KEY) && getSensitivePropertyKeys().contains(key.trim());
}
/**
* Returns true if the property identified by this key is considered protected in this instance of {@code NiFiProperties}.
* The property value is protected if the key is sensitive and the sibling key of key.protected is present.
*
* @param key the key
* @return true if it is currently marked as protected
* @see ProtectedNiFiProperties#getSensitivePropertyKeys()
*/
public boolean isPropertyProtected(String key) {
return key != null && isPropertySensitive(key) && !StringUtils.isBlank(getProperty(getProtectionKey(key)));
}
/**
* Returns the sibling property key which specifies the protection scheme for this key.
* <p>
* Example:
* <p>
* nifi.sensitive.key=ABCXYZ
* nifi.sensitive.key.protected=aes/gcm/256
* <p>
* nifi.sensitive.key -> nifi.sensitive.key.protected
*
* @param key the key identifying the sensitive property
* @return the key identifying the protection scheme for the sensitive property
*/
public String getProtectionKey(String key) {
if (key == null || key.isEmpty()) {
throw new IllegalArgumentException("Cannot find protection key for null key");
}
return key + ".protected";
}
/**
* Returns the unprotected {@link NiFiProperties} instance. If none of the properties
* loaded are marked as protected, it will simply pass through the internal instance.
* If any are protected, it will drop the protection scheme keys and translate each
* protected value (encrypted, HSM-retrieved, etc.) into the raw value and store it
* under the original key.
* <p>
* If any property fails to unprotect, it will save that key and continue. After
* attempting all properties, it will throw an exception containing all failed
* properties. This is necessary because the order is not enforced, so all failed
* properties should be gathered together.
*
* @return the NiFiProperties instance with all raw values
* @throws SensitivePropertyProtectionException if there is a problem unprotecting one or more keys
*/
public NiFiProperties getUnprotectedProperties() throws SensitivePropertyProtectionException {
if (hasProtectedKeys()) {
logger.info("There are {} protected properties of {} sensitive properties ({}%)",
getProtectedPropertyKeys().size(),
getSensitivePropertyKeys().size(),
getPercentOfSensitivePropertiesProtected());
Properties rawProperties = new Properties();
Set<String> failedKeys = new HashSet<>();
for (String key : getPropertyKeys()) {
/* Three kinds of keys
* 1. protection schemes -- skip
* 2. protected keys -- unprotect and copy
* 3. normal keys -- copy over
*/
if (key.endsWith(".protected")) {
// Do nothing
} else if (isPropertyProtected(key)) {
try {
rawProperties.setProperty(key, unprotectValue(key, getProperty(key)));
} catch (SensitivePropertyProtectionException e) {
logger.warn("Failed to unprotect '{}'", key, e);
failedKeys.add(key);
}
} else {
rawProperties.setProperty(key, getProperty(key));
}
}
if (!failedKeys.isEmpty()) {
if (failedKeys.size() > 1) {
logger.warn("Combining {} failed keys [{}] into single exception", failedKeys.size(), StringUtils.join(failedKeys, ", "));
throw new MultipleSensitivePropertyProtectionException("Failed to unprotect keys", failedKeys);
} else {
throw new SensitivePropertyProtectionException("Failed to unprotect key " + failedKeys.iterator().next());
}
}
NiFiProperties unprotected = new StandardNiFiProperties(rawProperties);
return unprotected;
} else {
logger.debug("No protected properties");
return getInternalNiFiProperties();
}
}
/**
* Registers a new {@link SensitivePropertyProvider}. This method will throw a {@link UnsupportedOperationException} if a provider is already registered for the protection scheme.
*
* @param sensitivePropertyProvider the provider
*/
void addSensitivePropertyProvider(SensitivePropertyProvider sensitivePropertyProvider) {
if (sensitivePropertyProvider == null) {
throw new IllegalArgumentException("Cannot add null SensitivePropertyProvider");
}
if (getSensitivePropertyProviders().containsKey(sensitivePropertyProvider.getIdentifierKey())) {
throw new UnsupportedOperationException("Cannot overwrite existing sensitive property provider registered for " + sensitivePropertyProvider.getIdentifierKey());
}
getSensitivePropertyProviders().put(sensitivePropertyProvider.getIdentifierKey(), sensitivePropertyProvider);
}
private String getDefaultProtectionScheme() {
if (!getSensitivePropertyProviders().isEmpty()) {
List<String> schemes = new ArrayList<>(getSensitivePropertyProviders().keySet());
Collections.sort(schemes);
return schemes.get(0);
} else {
throw new IllegalStateException("No registered protection schemes");
}
}
/**
* Returns a new instance of {@link NiFiProperties} with all populated sensitive values protected by the default protection scheme. Plain non-sensitive values are copied directly.
*
* @return the protected properties in a {@link StandardNiFiProperties} object
* @throws IllegalStateException if no protection schemes are registered
*/
NiFiProperties protectPlainProperties() {
try {
return protectPlainProperties(getDefaultProtectionScheme());
} catch (IllegalStateException e) {
final String msg = "Cannot protect properties with default scheme if no protection schemes are registered";
logger.warn(msg);
throw new IllegalStateException(msg, e);
}
}
/**
* Returns a new instance of {@link NiFiProperties} with all populated sensitive values protected by the provided protection scheme. Plain non-sensitive values are copied directly.
*
* @param protectionScheme the identifier key of the {@link SensitivePropertyProvider} to use
* @return the protected properties in a {@link StandardNiFiProperties} object
*/
NiFiProperties protectPlainProperties(String protectionScheme) {
SensitivePropertyProvider spp = getSensitivePropertyProvider(protectionScheme);
// Make a new holder (settable)
Properties protectedProperties = new Properties();
// Copy over the plain keys
Set<String> plainKeys = getPropertyKeys();
plainKeys.removeAll(getSensitivePropertyKeys());
for (String key : plainKeys) {
protectedProperties.setProperty(key, getInternalNiFiProperties().getProperty(key));
}
// Add the protected keys and the protection schemes
for (String key : getSensitivePropertyKeys()) {
final String plainValue = getInternalNiFiProperties().getProperty(key);
if (plainValue == null || plainValue.trim().isEmpty()) {
protectedProperties.setProperty(key, plainValue);
} else {
final String protectedValue = spp.protect(plainValue);
protectedProperties.setProperty(key, protectedValue);
protectedProperties.setProperty(getProtectionKey(key), protectionScheme);
}
}
return new StandardNiFiProperties(protectedProperties);
}
/**
* Returns the number of properties that are marked as protected in the provided {@link NiFiProperties} instance without requiring external creation of a {@link ProtectedNiFiProperties} instance.
*
* @param plainProperties the instance to count protected properties
* @return the number of protected properties
*/
public static int countProtectedProperties(NiFiProperties plainProperties) {
return new ProtectedNiFiProperties(plainProperties).getProtectedPropertyKeys().size();
}
/**
* Returns the number of properties that are marked as sensitive in the provided {@link NiFiProperties} instance without requiring external creation of a {@link ProtectedNiFiProperties} instance.
*
* @param plainProperties the instance to count sensitive properties
* @return the number of sensitive properties
*/
public static int countSensitiveProperties(NiFiProperties plainProperties) {
return new ProtectedNiFiProperties(plainProperties).getSensitivePropertyKeys().size();
}
@Override
public String toString() {
final Set<String> providers = getSensitivePropertyProviders().keySet();
return new StringBuilder("ProtectedNiFiProperties instance with ")
.append(size()).append(" properties (")
.append(getProtectedPropertyKeys().size())
.append(" protected) and ")
.append(providers.size())
.append(" sensitive property providers: ")
.append(StringUtils.join(providers, ", "))
.toString();
}
/**
* Returns the local provider cache (null-safe) as a Map of protection schemes -> implementations.
*
* @return the map
*/
private Map<String, SensitivePropertyProvider> getSensitivePropertyProviders() {
if (localProviderCache == null) {
localProviderCache = new HashMap<>();
}
return localProviderCache;
}
private SensitivePropertyProvider getSensitivePropertyProvider(String protectionScheme) {
if (isProviderAvailable(protectionScheme)) {
return getSensitivePropertyProviders().get(protectionScheme);
} else {
throw new SensitivePropertyProtectionException("No provider available for " + protectionScheme);
}
}
private boolean isProviderAvailable(String protectionScheme) {
return getSensitivePropertyProviders().containsKey(protectionScheme);
}
/**
* If the value is protected, unprotects it and returns it. If not, returns the original value.
*
* @param key the retrieved property key
* @param retrievedValue the retrieved property value
* @return the unprotected value
*/
private String unprotectValue(String key, String retrievedValue) {
// Checks if the key is sensitive and marked as protected
if (isPropertyProtected(key)) {
final String protectionScheme = getProperty(getProtectionKey(key));
// No provider registered for this scheme, so just return the value
if (!isProviderAvailable(protectionScheme)) {
logger.warn("No provider available for {} so passing the protected {} value back", protectionScheme, key);
return retrievedValue;
}
try {
SensitivePropertyProvider sensitivePropertyProvider = getSensitivePropertyProvider(protectionScheme);
return sensitivePropertyProvider.unprotect(retrievedValue);
} catch (SensitivePropertyProtectionException e) {
throw new SensitivePropertyProtectionException("Error unprotecting value for " + key, e.getCause());
}
}
return retrievedValue;
}
}

91
nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-properties-loader/src/main/java/org/apache/nifi/properties/SensitivePropertyProtectionException.java Normal file
View File

@ -0,0 +1,91 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.nifi.properties;
public class SensitivePropertyProtectionException extends RuntimeException {
/**
* Constructs a new throwable with {@code null} as its detail message.
* The cause is not initialized, and may subsequently be initialized by a
* call to {@link #initCause}.
* <p>
* <p>The {@link #fillInStackTrace()} method is called to initialize
* the stack trace data in the newly created throwable.
*/
public SensitivePropertyProtectionException() {
}
/**
* Constructs a new throwable with the specified detail message. The
* cause is not initialized, and may subsequently be initialized by
* a call to {@link #initCause}.
* <p>
* <p>The {@link #fillInStackTrace()} method is called to initialize
* the stack trace data in the newly created throwable.
*
* @param message the detail message. The detail message is saved for
* later retrieval by the {@link #getMessage()} method.
*/
public SensitivePropertyProtectionException(String message) {
super(message);
}
/**
* Constructs a new throwable with the specified detail message and
* cause. <p>Note that the detail message associated with
* {@code cause} is <i>not</i> automatically incorporated in
* this throwable's detail message.
* <p>
* <p>The {@link #fillInStackTrace()} method is called to initialize
* the stack trace data in the newly created throwable.
*
* @param message the detail message (which is saved for later retrieval
* by the {@link #getMessage()} method).
* @param cause the cause (which is saved for later retrieval by the
* {@link #getCause()} method). (A {@code null} value is
* permitted, and indicates that the cause is nonexistent or
* unknown.)
* @since 1.4
*/
public SensitivePropertyProtectionException(String message, Throwable cause) {
super(message, cause);
}
/**
* Constructs a new throwable with the specified cause and a detail
* message of {@code (cause==null ? null : cause.toString())} (which
* typically contains the class and detail message of {@code cause}).
* This constructor is useful for throwables that are little more than
* wrappers for other throwables (for example, PrivilegedActionException).
* <p>
* <p>The {@link #fillInStackTrace()} method is called to initialize
* the stack trace data in the newly created throwable.
*
* @param cause the cause (which is saved for later retrieval by the
* {@link #getCause()} method). (A {@code null} value is
* permitted, and indicates that the cause is nonexistent or
* unknown.)
* @since 1.4
*/
public SensitivePropertyProtectionException(Throwable cause) {
super(cause);
}
@Override
public String toString() {
return "SensitivePropertyProtectionException: " + getLocalizedMessage();
}
}

52
nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-properties-loader/src/main/java/org/apache/nifi/properties/SensitivePropertyProvider.java Normal file
View File

@ -0,0 +1,52 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.nifi.properties;
public interface SensitivePropertyProvider {
/**
* Returns the name of the underlying implementation.
*
* @return the name of this sensitive property provider
*/
String getName();
/**
* Returns the key used to identify the provider implementation in {@code nifi.properties}.
*
* @return the key to persist in the sibling property
*/
String getIdentifierKey();
/**
* Returns the "protected" form of this value. This is a form which can safely be persisted in the {@code nifi.properties} file without compromising the value.
* An encryption-based provider would return a cipher text, while a remote-lookup provider could return a unique ID to retrieve the secured value.
*
* @param unprotectedValue the sensitive value
* @return the value to persist in the {@code nifi.properties} file
*/
String protect(String unprotectedValue) throws SensitivePropertyProtectionException;
/**
* Returns the "unprotected" form of this value. This is the raw sensitive value which is used by the application logic.
* An encryption-based provider would decrypt a cipher text and return the plaintext, while a remote-lookup provider could retrieve the secured value.
*
* @param protectedValue the protected value read from the {@code nifi.properties} file
* @return the raw value to be used by the application
*/
String unprotect(String protectedValue) throws SensitivePropertyProtectionException;
}

23
nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-properties-loader/src/main/java/org/apache/nifi/properties/SensitivePropertyProviderFactory.java Normal file
View File

@ -0,0 +1,23 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.nifi.properties;
public interface SensitivePropertyProviderFactory {
SensitivePropertyProvider getProvider();
}

81
nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-properties-loader/src/main/java/org/apache/nifi/properties/StandardNiFiProperties.java Normal file
View File

@ -0,0 +1,81 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.nifi.properties;
import java.util.Enumeration;
import java.util.HashSet;
import java.util.Properties;
import java.util.Set;
import org.apache.nifi.util.NiFiProperties;
public class StandardNiFiProperties extends NiFiProperties {
private Properties rawProperties = new Properties();
public StandardNiFiProperties() {
this(null);
}
public StandardNiFiProperties(Properties props) {
this.rawProperties = props == null ? new Properties() : props;
}
/**
* Retrieves the property value for the given property key.
*
* @param key the key of property value to lookup
* @return value of property at given key or null if not found
*/
@Override
public String getProperty(String key) {
return rawProperties.getProperty(key);
}
/**
* Retrieves all known property keys.
*
* @return all known property keys
*/
@Override
public Set<String> getPropertyKeys() {
Set<String> propertyNames = new HashSet<>();
Enumeration e = getRawProperties().propertyNames();
for (; e.hasMoreElements(); ){
propertyNames.add((String) e.nextElement());
}
return propertyNames;
}
Properties getRawProperties() {
if (this.rawProperties == null) {
this.rawProperties = new Properties();
}
return this.rawProperties;
}
@Override
public int size() {
return getRawProperties().size();
}
@Override
public String toString() {
return "StandardNiFiProperties instance with " + size() + " properties";
}
}

97
nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-properties-loader/src/test/groovy/org/apache/nifi/properties/AESSensitivePropertyProviderFactoryTest.groovy Normal file
View File

@ -0,0 +1,97 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.nifi.properties
import org.bouncycastle.jce.provider.BouncyCastleProvider
import org.junit.After
import org.junit.Before
import org.junit.BeforeClass
import org.junit.Test
import org.junit.runner.RunWith
import org.junit.runners.JUnit4
import org.slf4j.Logger
import org.slf4j.LoggerFactory
import java.security.Security
@RunWith(JUnit4.class)
class AESSensitivePropertyProviderFactoryTest extends GroovyTestCase {
private static final Logger logger = LoggerFactory.getLogger(AESSensitivePropertyProviderFactoryTest.class)
private static final String KEY_HEX = "0123456789ABCDEFFEDCBA9876543210" * 2
@BeforeClass
public static void setUpOnce() throws Exception {
Security.addProvider(new BouncyCastleProvider())
logger.metaClass.methodMissing = { String name, args ->
logger.info("[${name?.toUpperCase()}] ${(args as List).join(" ")}")
}
}
@Before
public void setUp() throws Exception {
}
@After
public void tearDown() throws Exception {
}
@Test
public void testShouldGetProviderWithoutKey() throws Exception {
// Arrange
SensitivePropertyProviderFactory factory = new AESSensitivePropertyProviderFactory()
// Act
SensitivePropertyProvider provider = factory.getProvider()
// Assert
assert provider instanceof AESSensitivePropertyProvider
assert !provider.@key
assert !provider.@cipher
}
@Test
public void testShouldGetProviderWithKey() throws Exception {
// Arrange
SensitivePropertyProviderFactory factory = new AESSensitivePropertyProviderFactory(KEY_HEX)
// Act
SensitivePropertyProvider provider = factory.getProvider()
// Assert
assert provider instanceof AESSensitivePropertyProvider
assert provider.@key
assert provider.@cipher
}
@Test
public void testGetProviderShouldHandleEmptyKey() throws Exception {
// Arrange
SensitivePropertyProviderFactory factory = new AESSensitivePropertyProviderFactory("")
// Act
SensitivePropertyProvider provider = factory.getProvider()
// Assert
assert provider instanceof AESSensitivePropertyProvider
assert !provider.@key
assert !provider.@cipher
}
}

463
nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-properties-loader/src/test/groovy/org/apache/nifi/properties/AESSensitivePropertyProviderTest.groovy Normal file
View File

@ -0,0 +1,463 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.nifi.properties
import org.bouncycastle.jce.provider.BouncyCastleProvider
import org.bouncycastle.util.encoders.DecoderException
import org.bouncycastle.util.encoders.Hex
import org.junit.After
import org.junit.Assume
import org.junit.Before
import org.junit.BeforeClass
import org.junit.Test
import org.junit.runner.RunWith
import org.junit.runners.JUnit4
import org.slf4j.Logger
import org.slf4j.LoggerFactory
import javax.crypto.Cipher
import javax.crypto.spec.IvParameterSpec
import javax.crypto.spec.SecretKeySpec
import java.nio.charset.StandardCharsets
import java.security.SecureRandom
import java.security.Security
@RunWith(JUnit4.class)
class AESSensitivePropertyProviderTest extends GroovyTestCase {
private static final Logger logger = LoggerFactory.getLogger(AESSensitivePropertyProviderTest.class)
private static final String KEY_128_HEX = "0123456789ABCDEFFEDCBA9876543210"
private static final String KEY_256_HEX = KEY_128_HEX * 2
private static final int IV_LENGTH = AESSensitivePropertyProvider.getIvLength()
private static final List<Integer> KEY_SIZES = getAvailableKeySizes()
private static final SecureRandom secureRandom = new SecureRandom()
private static final Base64.Encoder encoder = Base64.encoder
private static final Base64.Decoder decoder = Base64.decoder
@BeforeClass
public static void setUpOnce() throws Exception {
Security.addProvider(new BouncyCastleProvider())
logger.metaClass.methodMissing = { String name, args ->
logger.info("[${name?.toUpperCase()}] ${(args as List).join(" ")}")
}
}
@Before
public void setUp() throws Exception {
}
@After
public void tearDown() throws Exception {
}
private static Cipher getCipher(boolean encrypt = true, int keySize = 256, byte[] iv = [0x00] * IV_LENGTH) {
Cipher cipher = Cipher.getInstance("AES/GCM/NoPadding")
String key = getKeyOfSize(keySize)
cipher.init((encrypt ? Cipher.ENCRYPT_MODE : Cipher.DECRYPT_MODE) as int, new SecretKeySpec(Hex.decode(key), "AES"), new IvParameterSpec(iv))
logger.setup("Initialized a cipher in ${encrypt ? "encrypt" : "decrypt"} mode with a key of length ${keySize} bits")
cipher
}
private static String getKeyOfSize(int keySize = 256) {
switch (keySize) {
case 128:
return KEY_128_HEX
case 192:
case 256:
if (Cipher.getMaxAllowedKeyLength("AES") < keySize) {
throw new IllegalArgumentException("The JCE unlimited strength cryptographic jurisdiction policies are not installed, so the max key size is 128 bits")
}
return KEY_256_HEX[0..<(keySize / 4)]
default:
throw new IllegalArgumentException("Key size ${keySize} bits is not valid")
}
}
private static List<Integer> getAvailableKeySizes() {
if (Cipher.getMaxAllowedKeyLength("AES") > 128) {
[128, 192, 256]
} else {
[128]
}
}
private static String manipulateString(String input, int start = 0, int end = input?.length()) {
if ((input[start..end] as List).unique().size() == 1) {
throw new IllegalArgumentException("Can't manipulate a String where the entire range is identical [${input[start..end]}]")
}
List shuffled = input[start..end] as List
Collections.shuffle(shuffled)
String reconstituted = input[0..<start] + shuffled.join() + input[end + 1..-1]
return reconstituted != input ? reconstituted : manipulateString(input, start, end)
}
@Test
public void testShouldThrowExceptionOnInitializationWithoutBouncyCastle() throws Exception {
// Arrange
try {
Security.removeProvider(new BouncyCastleProvider().getName())
// Act
def msg = shouldFail(SensitivePropertyProtectionException) {
SensitivePropertyProvider spp = new AESSensitivePropertyProvider(Hex.decode(KEY_128_HEX))
logger.error("This should not be reached")
}
// Assert
assert msg =~ "Error initializing the protection cipher"
} finally {
Security.addProvider(new BouncyCastleProvider())
}
}
// TODO: testShouldGetName()
@Test
public void testShouldProtectValue() throws Exception {
final String PLAINTEXT = "This is a plaintext value"
// Act
Map<Integer, String> CIPHER_TEXTS = KEY_SIZES.collectEntries { int keySize ->
SensitivePropertyProvider spp = new AESSensitivePropertyProvider(Hex.decode(getKeyOfSize(keySize)))
logger.info("Initialized ${spp.name} with key size ${keySize}")
[(keySize): spp.protect(PLAINTEXT)]
}
CIPHER_TEXTS.each { ks, ct -> logger.info("Encrypted for ${ks} length key: ${ct}") }
// Assert
// The IV generation is part of #protect, so the expected cipher text values must be generated after #protect has run
Map<Integer, Cipher> decryptionCiphers = CIPHER_TEXTS.collectEntries { int keySize, String cipherText ->
// The 12 byte IV is the first 16 Base64-encoded characters of the "complete" cipher text
byte[] iv = decoder.decode(cipherText[0..<16])
[(keySize): getCipher(false, keySize, iv)]
}
Map<Integer, String> plaintexts = decryptionCiphers.collectEntries { Map.Entry<Integer, Cipher> e ->
String cipherTextWithoutIVAndDelimiter = CIPHER_TEXTS[e.key][18..-1]
String plaintext = new String(e.value.doFinal(decoder.decode(cipherTextWithoutIVAndDelimiter)), StandardCharsets.UTF_8)
[(e.key): plaintext]
}
CIPHER_TEXTS.each { key, ct -> logger.expected("Cipher text for ${key} length key: ${ct}") }
assert plaintexts.every { int ks, String pt -> pt == PLAINTEXT }
}
@Test
public void testShouldHandleProtectEmptyValue() throws Exception {
final List<String> EMPTY_PLAINTEXTS = ["", " ", null]
// Act
KEY_SIZES.collectEntries { int keySize ->
SensitivePropertyProvider spp = new AESSensitivePropertyProvider(Hex.decode(getKeyOfSize(keySize)))
logger.info("Initialized ${spp.name} with key size ${keySize}")
EMPTY_PLAINTEXTS.each { String emptyPlaintext ->
def msg = shouldFail(IllegalArgumentException) {
spp.protect(emptyPlaintext)
}
logger.expected("${msg} for keySize ${keySize} and plaintext [${emptyPlaintext}]")
// Assert
assert msg == "Cannot encrypt an empty value"
}
}
}
@Test
public void testShouldUnprotectValue() throws Exception {
// Arrange
final String PLAINTEXT = "This is a plaintext value"
Map<Integer, Cipher> encryptionCiphers = KEY_SIZES.collectEntries { int keySize ->
byte[] iv = new byte[IV_LENGTH]
secureRandom.nextBytes(iv)
[(keySize): getCipher(true, keySize, iv)]
}
Map<Integer, String> CIPHER_TEXTS = encryptionCiphers.collectEntries { Map.Entry<Integer, Cipher> e ->
String iv = encoder.encodeToString(e.value.getIV())
String cipherText = encoder.encodeToString(e.value.doFinal(PLAINTEXT.getBytes(StandardCharsets.UTF_8)))
[(e.key): "${iv}||${cipherText}"]
}
CIPHER_TEXTS.each { key, ct -> logger.expected("Cipher text for ${key} length key: ${ct}") }
// Act
Map<Integer, String> plaintexts = CIPHER_TEXTS.collectEntries { int keySize, String cipherText ->
SensitivePropertyProvider spp = new AESSensitivePropertyProvider(Hex.decode(getKeyOfSize(keySize)))
logger.info("Initialized ${spp.name} with key size ${keySize}")
[(keySize): spp.unprotect(cipherText)]
}
plaintexts.each { ks, pt -> logger.info("Decrypted for ${ks} length key: ${pt}") }
// Assert
assert plaintexts.every { int ks, String pt -> pt == PLAINTEXT }
}
/**
* Tests inputs where the entire String is empty/blank space/{@code null}.
*
* @throws Exception
*/
@Test
public void testShouldHandleUnprotectEmptyValue() throws Exception {
// Arrange
final List<String> EMPTY_CIPHER_TEXTS = ["", " ", null]
// Act
KEY_SIZES.each { int keySize ->
SensitivePropertyProvider spp = new AESSensitivePropertyProvider(Hex.decode(getKeyOfSize(keySize)))
logger.info("Initialized ${spp.name} with key size ${keySize}")
EMPTY_CIPHER_TEXTS.each { String emptyCipherText ->
def msg = shouldFail(IllegalArgumentException) {
spp.unprotect(emptyCipherText)
}
logger.expected("${msg} for keySize ${keySize} and cipher text [${emptyCipherText}]")
// Assert
assert msg == "Cannot decrypt a cipher text shorter than ${AESSensitivePropertyProvider.minCipherTextLength} chars".toString()
}
}
}
@Test
public void testShouldHandleUnprotectMalformedValue() throws Exception {
// Arrange
final String PLAINTEXT = "This is a plaintext value"
// Act
KEY_SIZES.each { int keySize ->
SensitivePropertyProvider spp = new AESSensitivePropertyProvider(Hex.decode(getKeyOfSize(keySize)))
logger.info("Initialized ${spp.name} with key size ${keySize}")
String cipherText = spp.protect(PLAINTEXT)
// Swap two characters in the cipher text
final String MALFORMED_CIPHER_TEXT = manipulateString(cipherText, 25, 28)
logger.info("Manipulated ${cipherText} to\n${MALFORMED_CIPHER_TEXT.padLeft(163)}")
def msg = shouldFail(SensitivePropertyProtectionException) {
spp.unprotect(MALFORMED_CIPHER_TEXT)
}
logger.expected("${msg} for keySize ${keySize} and cipher text [${MALFORMED_CIPHER_TEXT}]")
// Assert
assert msg == "Error decrypting a protected value"
}
}
@Test
public void testShouldHandleUnprotectMissingIV() throws Exception {
// Arrange
final String PLAINTEXT = "This is a plaintext value"
// Act
KEY_SIZES.each { int keySize ->
SensitivePropertyProvider spp = new AESSensitivePropertyProvider(Hex.decode(getKeyOfSize(keySize)))
logger.info("Initialized ${spp.name} with key size ${keySize}")
String cipherText = spp.protect(PLAINTEXT)
// Remove the IV from the "complete" cipher text
final String MISSING_IV_CIPHER_TEXT = cipherText[18..-1]
logger.info("Manipulated ${cipherText} to\n${MISSING_IV_CIPHER_TEXT.padLeft(163)}")
def msg = shouldFail(IllegalArgumentException) {
spp.unprotect(MISSING_IV_CIPHER_TEXT)
}
logger.expected("${msg} for keySize ${keySize} and cipher text [${MISSING_IV_CIPHER_TEXT}]")
// Remove the IV from the "complete" cipher text but keep the delimiter
final String MISSING_IV_CIPHER_TEXT_WITH_DELIMITER = cipherText[16..-1]
logger.info("Manipulated ${cipherText} to\n${MISSING_IV_CIPHER_TEXT_WITH_DELIMITER.padLeft(163)}")
def msgWithDelimiter = shouldFail(DecoderException) {
spp.unprotect(MISSING_IV_CIPHER_TEXT_WITH_DELIMITER)
}
logger.expected("${msgWithDelimiter} for keySize ${keySize} and cipher text [${MISSING_IV_CIPHER_TEXT_WITH_DELIMITER}]")
// Assert
assert msg == "The cipher text does not contain the delimiter || -- it should be of the form Base64(IV) || Base64(cipherText)"
// Assert
assert msgWithDelimiter =~ "unable to decode base64 string"
}
}
/**
* Tests inputs which have a valid IV and delimiter but no "cipher text".
*
* @throws Exception
*/
@Test
public void testShouldHandleUnprotectEmptyCipherText() throws Exception {
// Arrange
final String IV_AND_DELIMITER = "${encoder.encodeToString("Bad IV value".getBytes(StandardCharsets.UTF_8))}||"
logger.info("IV and delimiter: ${IV_AND_DELIMITER}")
final List<String> EMPTY_CIPHER_TEXTS = ["", " ", "\n"].collect { "${IV_AND_DELIMITER}${it}" }
// Act
KEY_SIZES.each { int keySize ->
SensitivePropertyProvider spp = new AESSensitivePropertyProvider(Hex.decode(getKeyOfSize(keySize)))
logger.info("Initialized ${spp.name} with key size ${keySize}")
EMPTY_CIPHER_TEXTS.each { String emptyCipherText ->
def msg = shouldFail(IllegalArgumentException) {
spp.unprotect(emptyCipherText)
}
logger.expected("${msg} for keySize ${keySize} and cipher text [${emptyCipherText}]")
// Assert
assert msg == "Cannot decrypt a cipher text shorter than ${AESSensitivePropertyProvider.minCipherTextLength} chars".toString()
}
}
}
@Test
public void testShouldHandleUnprotectMalformedIV() throws Exception {
// Arrange
final String PLAINTEXT = "This is a plaintext value"
// Act
KEY_SIZES.each { int keySize ->
SensitivePropertyProvider spp = new AESSensitivePropertyProvider(Hex.decode(getKeyOfSize(keySize)))
logger.info("Initialized ${spp.name} with key size ${keySize}")
String cipherText = spp.protect(PLAINTEXT)
// Swap two characters in the IV
final String MALFORMED_IV_CIPHER_TEXT = manipulateString(cipherText, 8, 11)
logger.info("Manipulated ${cipherText} to\n${MALFORMED_IV_CIPHER_TEXT.padLeft(163)}")
def msg = shouldFail(SensitivePropertyProtectionException) {
spp.unprotect(MALFORMED_IV_CIPHER_TEXT)
}
logger.expected("${msg} for keySize ${keySize} and cipher text [${MALFORMED_IV_CIPHER_TEXT}]")
// Assert
assert msg == "Error decrypting a protected value"
}
}
@Test
public void testShouldGetImplementationKeyWithDifferentMaxKeyLengths() throws Exception {
// Arrange
final int MAX_KEY_SIZE = getAvailableKeySizes().max()
final String EXPECTED_IMPL_KEY = "aes/gcm/${MAX_KEY_SIZE}"
logger.expected("Implementation key: ${EXPECTED_IMPL_KEY}")
// Act
String key = new AESSensitivePropertyProvider(getKeyOfSize(MAX_KEY_SIZE)).getIdentifierKey()
logger.info("Implementation key: ${key}")
// Assert
assert key == EXPECTED_IMPL_KEY
}
@Test
public void testShouldNotAllowEmptyKey() throws Exception {
// Arrange
final String INVALID_KEY = ""
// Act
def msg = shouldFail(SensitivePropertyProtectionException) {
AESSensitivePropertyProvider spp = new AESSensitivePropertyProvider(INVALID_KEY)
}
// Assert
assert msg == "The key cannot be empty"
}
@Test
public void testShouldNotAllowIncorrectlySizedKey() throws Exception {
// Arrange
final String INVALID_KEY = "Z" * 31
// Act
def msg = shouldFail(SensitivePropertyProtectionException) {
AESSensitivePropertyProvider spp = new AESSensitivePropertyProvider(INVALID_KEY)
}
// Assert
assert msg == "The key must be a valid hexadecimal key"
}
@Test
public void testShouldNotAllowInvalidKey() throws Exception {
// Arrange
final String INVALID_KEY = "Z" * 32
// Act
def msg = shouldFail(SensitivePropertyProtectionException) {
AESSensitivePropertyProvider spp = new AESSensitivePropertyProvider(INVALID_KEY)
}
// Assert
assert msg == "The key must be a valid hexadecimal key"
}
/**
* This test is to ensure internal consistency and allow for encrypting value for various property files
*/
@Test
public void testShouldEncryptArbitraryValues() {
// Arrange
def values = ["thisIsABadSensitiveKeyPassword", "thisIsABadKeystorePassword", "thisIsABadKeyPassword", "thisIsABadTruststorePassword", "This is an encrypted banner message"]
String key = getKeyOfSize(128)
// key = "0" * 64
SensitivePropertyProvider spp = new AESSensitivePropertyProvider(key)
// Act
def encryptedValues = values.collect { String v ->
def encryptedValue = spp.protect(v)
logger.info("${v} -> ${encryptedValue}")
def (String iv, String cipherText) = encryptedValue.tokenize("||")
logger.info("Normal Base64 encoding would be ${encoder.encodeToString(decoder.decode(iv))}||${encoder.encodeToString(decoder.decode(cipherText))}")
encryptedValue
}
// Assert
assert values == encryptedValues.collect { spp.unprotect(it) }
}
/**
* This test is to ensure external compatibility in case someone encodes the encrypted value with Base64 and does not remove the padding
*/
@Test
public void testShouldDecryptPaddedValue() {
// Arrange
Assume.assumeTrue("JCE unlimited strength crypto policy must be installed for this test", Cipher.getMaxAllowedKeyLength("AES") > 128)
final String EXPECTED_VALUE = "thisIsABadKeyPassword"
String cipherText = "ac/BaE35SL/esLiJ||+ULRvRLYdIDA2VqpE0eQXDEMjaLBMG2kbKOdOwBk/hGebDKlVg=="
String unpaddedCipherText = cipherText.replaceAll("=", "")
String key = getKeyOfSize(256)
SensitivePropertyProvider spp = new AESSensitivePropertyProvider(key)
// Act
String rawValue = spp.unprotect(cipherText)
logger.info("Decrypted ${cipherText} to ${rawValue}")
String rawUnpaddedValue = spp.unprotect(unpaddedCipherText)
logger.info("Decrypted ${unpaddedCipherText} to ${rawUnpaddedValue}")
// Assert
assert rawValue == EXPECTED_VALUE
assert rawUnpaddedValue == EXPECTED_VALUE
}
}

393
nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-properties-loader/src/test/groovy/org/apache/nifi/properties/NiFiPropertiesLoaderGroovyTest.groovy Normal file
View File

@ -0,0 +1,393 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.nifi.properties
import org.apache.nifi.util.NiFiProperties
import org.bouncycastle.jce.provider.BouncyCastleProvider
import org.junit.After
import org.junit.AfterClass
import org.junit.Before
import org.junit.BeforeClass
import org.junit.Test
import org.junit.runner.RunWith
import org.junit.runners.JUnit4
import org.slf4j.Logger
import org.slf4j.LoggerFactory
import javax.crypto.Cipher
import java.nio.file.Files
import java.nio.file.attribute.PosixFilePermission
import java.security.Security
@RunWith(JUnit4.class)
class NiFiPropertiesLoaderGroovyTest extends GroovyTestCase {
private static final Logger logger = LoggerFactory.getLogger(NiFiPropertiesLoaderGroovyTest.class)
final def DEFAULT_SENSITIVE_PROPERTIES = [
"nifi.sensitive.props.key",
"nifi.security.keystorePasswd",
"nifi.security.keyPasswd",
"nifi.security.truststorePasswd"
]
final def COMMON_ADDITIONAL_SENSITIVE_PROPERTIES = [
"nifi.sensitive.props.algorithm",
"nifi.kerberos.service.principal",
"nifi.kerberos.krb5.file",
"nifi.kerberos.keytab.location"
]
private static final String KEY_HEX = "0123456789ABCDEFFEDCBA9876543210" * 2
private static String originalPropertiesPath = System.getProperty(NiFiProperties.PROPERTIES_FILE_PATH)
private
final Set<PosixFilePermission> ownerReadWrite = [PosixFilePermission.OWNER_WRITE, PosixFilePermission.OWNER_READ]
@BeforeClass
public static void setUpOnce() throws Exception {
Security.addProvider(new BouncyCastleProvider())
logger.metaClass.methodMissing = { String name, args ->
logger.info("[${name?.toUpperCase()}] ${(args as List).join(" ")}")
}
}
@Before
public void setUp() throws Exception {
}
@After
public void tearDown() throws Exception {
// Clear the sensitive property providers between runs
// if (ProtectedNiFiProperties.@localProviderCache) {
// ProtectedNiFiProperties.@localProviderCache = [:]
// }
NiFiPropertiesLoader.@sensitivePropertyProviderFactory = null
}
@AfterClass
public static void tearDownOnce() {
if (originalPropertiesPath) {
System.setProperty(NiFiProperties.PROPERTIES_FILE_PATH, originalPropertiesPath)
}
}
@Test
public void testConstructorShouldCreateNewInstance() throws Exception {
// Arrange
// Act
NiFiPropertiesLoader niFiPropertiesLoader = new NiFiPropertiesLoader()
// Assert
assert !niFiPropertiesLoader.@instance
assert !niFiPropertiesLoader.@keyHex
}
@Test
public void testShouldCreateInstanceWithKey() throws Exception {
// Arrange
// Act
NiFiPropertiesLoader niFiPropertiesLoader = NiFiPropertiesLoader.withKey(KEY_HEX)
// Assert
assert !niFiPropertiesLoader.@instance
assert niFiPropertiesLoader.@keyHex == KEY_HEX
}
@Test
public void testShouldGetDefaultProviderKey() throws Exception {
// Arrange
final String EXPECTED_PROVIDER_KEY = "aes/gcm/${Cipher.getMaxAllowedKeyLength("AES") > 128 ? 256 : 128}"
logger.info("Expected provider key: ${EXPECTED_PROVIDER_KEY}")
// Act
String defaultKey = NiFiPropertiesLoader.getDefaultProviderKey()
logger.info("Default key: ${defaultKey}")
// Assert
assert defaultKey == EXPECTED_PROVIDER_KEY
}
@Test
public void testShouldInitializeSensitivePropertyProviderFactory() throws Exception {
// Arrange
NiFiPropertiesLoader niFiPropertiesLoader = new NiFiPropertiesLoader()
// Act
niFiPropertiesLoader.initializeSensitivePropertyProviderFactory()
// Assert
assert niFiPropertiesLoader.@sensitivePropertyProviderFactory
}
@Test
public void testShouldLoadUnprotectedPropertiesFromFile() throws Exception {
// Arrange
File unprotectedFile = new File("src/test/resources/conf/nifi.properties")
NiFiPropertiesLoader niFiPropertiesLoader = new NiFiPropertiesLoader()
// Act
NiFiProperties niFiProperties = niFiPropertiesLoader.load(unprotectedFile)
// Assert
assert niFiProperties.size() > 0
// Ensure it is not a ProtectedNiFiProperties
assert niFiProperties instanceof StandardNiFiProperties
}
@Test
public void testShouldNotLoadUnprotectedPropertiesFromNullFile() throws Exception {
// Arrange
NiFiPropertiesLoader niFiPropertiesLoader = new NiFiPropertiesLoader()
// Act
def msg = shouldFail(IllegalArgumentException) {
NiFiProperties niFiProperties = niFiPropertiesLoader.load(null as File)
}
logger.expected(msg)
// Assert
assert msg == "NiFi properties file missing or unreadable"
}
@Test
public void testShouldNotLoadUnprotectedPropertiesFromMissingFile() throws Exception {
// Arrange
File missingFile = new File("src/test/resources/conf/nifi_missing.properties")
assert !missingFile.exists()
NiFiPropertiesLoader niFiPropertiesLoader = new NiFiPropertiesLoader()
// Act
def msg = shouldFail(IllegalArgumentException) {
NiFiProperties niFiProperties = niFiPropertiesLoader.load(missingFile)
}
logger.expected(msg)
// Assert
assert msg == "NiFi properties file missing or unreadable"
}
@Test
public void testShouldNotLoadUnprotectedPropertiesFromUnreadableFile() throws Exception {
// Arrange
File unreadableFile = new File("src/test/resources/conf/nifi_no_permissions.properties")
Files.setPosixFilePermissions(unreadableFile.toPath(), [] as Set)
assert !unreadableFile.canRead()
NiFiPropertiesLoader niFiPropertiesLoader = new NiFiPropertiesLoader()
// Act
def msg = shouldFail(IllegalArgumentException) {
NiFiProperties niFiProperties = niFiPropertiesLoader.load(unreadableFile)
}
logger.expected(msg)
// Assert
assert msg == "NiFi properties file missing or unreadable"
// Clean up to allow for indexing, etc.
Files.setPosixFilePermissions(unreadableFile.toPath(), ownerReadWrite)
}
@Test
public void testShouldLoadUnprotectedPropertiesFromPath() throws Exception {
// Arrange
File unprotectedFile = new File("src/test/resources/conf/nifi.properties")
NiFiPropertiesLoader niFiPropertiesLoader = new NiFiPropertiesLoader()
// Act
NiFiProperties niFiProperties = niFiPropertiesLoader.load(unprotectedFile.path)
// Assert
assert niFiProperties.size() > 0
// Ensure it is not a ProtectedNiFiProperties
assert niFiProperties instanceof StandardNiFiProperties
}
@Test
public void testShouldLoadUnprotectedPropertiesFromProtectedFile() throws Exception {
// Arrange
File protectedFile = new File("src/test/resources/conf/nifi_with_sensitive_properties_protected_aes.properties")
NiFiPropertiesLoader niFiPropertiesLoader = NiFiPropertiesLoader.withKey(KEY_HEX)
final def EXPECTED_PLAIN_VALUES = [
(NiFiProperties.SENSITIVE_PROPS_KEY): "thisIsABadSensitiveKeyPassword",
(NiFiProperties.SECURITY_KEYSTORE_PASSWD): "thisIsABadKeystorePassword",
(NiFiProperties.SECURITY_KEY_PASSWD): "thisIsABadKeyPassword",
]
// This method is covered in tests above, so safe to use here to retrieve protected properties
ProtectedNiFiProperties protectedNiFiProperties = niFiPropertiesLoader.readProtectedPropertiesFromDisk(protectedFile)
int totalKeysCount = protectedNiFiProperties.getPropertyKeysIncludingProtectionSchemes().size()
int protectedKeysCount = protectedNiFiProperties.getProtectedPropertyKeys().size()
logger.info("Read ${totalKeysCount} total properties (${protectedKeysCount} protected) from ${protectedFile.canonicalPath}")
// Act
NiFiProperties niFiProperties = niFiPropertiesLoader.load(protectedFile)
// Assert
assert niFiProperties.size() == totalKeysCount - protectedKeysCount
// Ensure that any key marked as protected above is different in this instance
protectedNiFiProperties.getProtectedPropertyKeys().keySet().each { String key ->
String plainValue = niFiProperties.getProperty(key)
String protectedValue = protectedNiFiProperties.getProperty(key)
logger.info("Checking that [${protectedValue}] -> [${plainValue}] == [${EXPECTED_PLAIN_VALUES[key]}]")
assert plainValue == EXPECTED_PLAIN_VALUES[key]
assert plainValue != protectedValue
assert plainValue.length() <= protectedValue.length()
}
// Ensure it is not a ProtectedNiFiProperties
assert niFiProperties instanceof StandardNiFiProperties
}
@Test
public void testShouldExtractKeyFromBootstrapFile() throws Exception {
// Arrange
def defaultNiFiPropertiesFilePath = "src/test/resources/bootstrap_tests/conf/nifi.properties"
System.setProperty(NiFiProperties.PROPERTIES_FILE_PATH, defaultNiFiPropertiesFilePath)
// Act
String key = NiFiPropertiesLoader.extractKeyFromBootstrapFile()
// Assert
assert key == KEY_HEX
}
@Test
public void testShouldNotExtractKeyFromBootstrapFileWithoutKeyLine() throws Exception {
// Arrange
def defaultNiFiPropertiesFilePath = "src/test/resources/bootstrap_tests/missing_key_line/nifi.properties"
System.setProperty(NiFiProperties.PROPERTIES_FILE_PATH, defaultNiFiPropertiesFilePath)
// Act
String key = NiFiPropertiesLoader.extractKeyFromBootstrapFile()
// Assert
assert key == ""
}
@Test
public void testShouldNotExtractKeyFromBootstrapFileWithoutKey() throws Exception {
// Arrange
def defaultNiFiPropertiesFilePath = "src/test/resources/bootstrap_tests/missing_key_line/nifi.properties"
System.setProperty(NiFiProperties.PROPERTIES_FILE_PATH, defaultNiFiPropertiesFilePath)
// Act
String key = NiFiPropertiesLoader.extractKeyFromBootstrapFile()
// Assert
assert key == ""
}
@Test
public void testShouldNotExtractKeyFromMissingBootstrapFile() throws Exception {
// Arrange
def defaultNiFiPropertiesFilePath = "src/test/resources/bootstrap_tests/missing_bootstrap/nifi.properties"
System.setProperty(NiFiProperties.PROPERTIES_FILE_PATH, defaultNiFiPropertiesFilePath)
// Act
def msg = shouldFail(IOException) {
String key = NiFiPropertiesLoader.extractKeyFromBootstrapFile()
}
logger.expected(msg)
// Assert
assert msg == "Cannot read from bootstrap.conf"
}
@Test
public void testShouldNotExtractKeyFromUnreadableBootstrapFile() throws Exception {
// Arrange
File unreadableFile = new File("src/test/resources/bootstrap_tests/unreadable_bootstrap/bootstrap.conf")
Set<PosixFilePermission> originalPermissions = Files.getPosixFilePermissions(unreadableFile.toPath())
Files.setPosixFilePermissions(unreadableFile.toPath(), [] as Set)
assert !unreadableFile.canRead()
def defaultNiFiPropertiesFilePath = "src/test/resources/bootstrap_tests/unreadable_bootstrap/nifi.properties"
System.setProperty(NiFiProperties.PROPERTIES_FILE_PATH, defaultNiFiPropertiesFilePath)
// Act
def msg = shouldFail(IOException) {
String key = NiFiPropertiesLoader.extractKeyFromBootstrapFile()
}
logger.expected(msg)
// Assert
assert msg == "Cannot read from bootstrap.conf"
// Clean up to allow for indexing, etc.
Files.setPosixFilePermissions(unreadableFile.toPath(), originalPermissions)
}
@Test
public void testShouldNotExtractKeyFromUnreadableConfDir() throws Exception {
// Arrange
File unreadableDir = new File("src/test/resources/bootstrap_tests/unreadable_conf")
Set<PosixFilePermission> originalPermissions = Files.getPosixFilePermissions(unreadableDir.toPath())
Files.setPosixFilePermissions(unreadableDir.toPath(), [] as Set)
assert !unreadableDir.canRead()
def defaultNiFiPropertiesFilePath = "src/test/resources/bootstrap_tests/unreadable_conf/nifi.properties"
System.setProperty(NiFiProperties.PROPERTIES_FILE_PATH, defaultNiFiPropertiesFilePath)
// Act
def msg = shouldFail(IOException) {
String key = NiFiPropertiesLoader.extractKeyFromBootstrapFile()
}
logger.expected(msg)
// Assert
assert msg == "Cannot read from bootstrap.conf"
// Clean up to allow for indexing, etc.
Files.setPosixFilePermissions(unreadableDir.toPath(), originalPermissions)
}
@Test
public void testShouldLoadUnprotectedPropertiesFromProtectedDefaultFileAndUseBootstrapKey() throws Exception {
// Arrange
File protectedFile = new File("src/test/resources/bootstrap_tests/conf/nifi_with_sensitive_properties_protected_aes.properties")
System.setProperty(NiFiProperties.PROPERTIES_FILE_PATH, protectedFile.path)
NiFiPropertiesLoader niFiPropertiesLoader = NiFiPropertiesLoader.withKey(KEY_HEX)
NiFiProperties normalReadProperties = niFiPropertiesLoader.load(protectedFile)
logger.info("Read ${normalReadProperties.size()} total properties from ${protectedFile.canonicalPath}")
// Act
NiFiProperties niFiProperties = NiFiPropertiesLoader.loadDefaultWithKeyFromBootstrap()
// Assert
assert niFiProperties.size() == normalReadProperties.size()
def readPropertiesAndValues = niFiProperties.getPropertyKeys().collectEntries {
[(it): niFiProperties.getProperty(it)]
}
def expectedPropertiesAndValues = normalReadProperties.getPropertyKeys().collectEntries {
[(it): normalReadProperties.getProperty(it)]
}
assert readPropertiesAndValues == expectedPropertiesAndValues
}
}

860
nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-properties-loader/src/test/groovy/org/apache/nifi/properties/ProtectedNiFiPropertiesGroovyTest.groovy Normal file
View File

@ -0,0 +1,860 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.nifi.properties
import org.apache.nifi.util.NiFiProperties
import org.bouncycastle.jce.provider.BouncyCastleProvider
import org.junit.After
import org.junit.AfterClass
import org.junit.Before
import org.junit.BeforeClass
import org.junit.Test
import org.junit.runner.RunWith
import org.junit.runners.JUnit4
import org.slf4j.Logger
import org.slf4j.LoggerFactory
import java.security.Security
@RunWith(JUnit4.class)
class ProtectedNiFiPropertiesGroovyTest extends GroovyTestCase {
private static final Logger logger = LoggerFactory.getLogger(ProtectedNiFiPropertiesGroovyTest.class)
final def DEFAULT_SENSITIVE_PROPERTIES = [
"nifi.sensitive.props.key",
"nifi.security.keystorePasswd",
"nifi.security.keyPasswd",
"nifi.security.truststorePasswd"
]
final def COMMON_ADDITIONAL_SENSITIVE_PROPERTIES = [
"nifi.sensitive.props.algorithm",
"nifi.kerberos.service.principal",
"nifi.kerberos.krb5.file",
"nifi.kerberos.keytab.location"
]
private static final String KEY_HEX = "0123456789ABCDEFFEDCBA9876543210" * 2
private static String originalPropertiesPath = System.getProperty(NiFiProperties.PROPERTIES_FILE_PATH)
@BeforeClass
public static void setUpOnce() throws Exception {
Security.addProvider(new BouncyCastleProvider())
logger.metaClass.methodMissing = { String name, args ->
logger.info("[${name?.toUpperCase()}] ${(args as List).join(" ")}")
}
}
@Before
public void setUp() throws Exception {
}
@After
public void tearDown() throws Exception {
}
@AfterClass
public static void tearDownOnce() {
if (originalPropertiesPath) {
System.setProperty(NiFiProperties.PROPERTIES_FILE_PATH, originalPropertiesPath)
}
}
private static ProtectedNiFiProperties loadFromFile(String propertiesFilePath) {
String filePath
try {
filePath = ProtectedNiFiPropertiesGroovyTest.class.getResource(propertiesFilePath).toURI().getPath()
} catch (URISyntaxException ex) {
throw new RuntimeException("Cannot load properties file due to "
+ ex.getLocalizedMessage(), ex)
}
File file = new File(filePath)
if (file == null || !file.exists() || !file.canRead()) {
String path = (file == null ? "missing file" : file.getAbsolutePath())
logger.error("Cannot read from '{}' -- file is missing or not readable", path)
throw new IllegalArgumentException("NiFi properties file missing or unreadable")
}
Properties rawProperties = new Properties()
InputStream inStream = null
try {
inStream = new BufferedInputStream(new FileInputStream(file))
rawProperties.load(inStream)
logger.info("Loaded {} properties from {}", rawProperties.size(), file.getAbsolutePath())
ProtectedNiFiProperties protectedNiFiProperties = new ProtectedNiFiProperties(rawProperties)
// If it has protected keys, inject the SPP
if (protectedNiFiProperties.hasProtectedKeys()) {
protectedNiFiProperties.addSensitivePropertyProvider(new AESSensitivePropertyProvider(KEY_HEX))
}
return protectedNiFiProperties
} catch (final Exception ex) {
logger.error("Cannot load properties file due to " + ex.getLocalizedMessage())
throw new RuntimeException("Cannot load properties file due to "
+ ex.getLocalizedMessage(), ex)
} finally {
if (null != inStream) {
try {
inStream.close()
} catch (final Exception ex) {
/**
* do nothing *
*/
}
}
}
}
@Test
public void testConstructorShouldCreateNewInstance() throws Exception {
// Arrange
// Act
NiFiProperties niFiProperties = new StandardNiFiProperties()
logger.info("niFiProperties has ${niFiProperties.size()} properties: ${niFiProperties.getPropertyKeys()}")
// Assert
assert niFiProperties.size() == 0
assert niFiProperties.getPropertyKeys() == [] as Set
}
@Test
public void testConstructorShouldAcceptRawProperties() throws Exception {
// Arrange
Properties rawProperties = new Properties()
rawProperties.setProperty("key", "value")
logger.info("rawProperties has ${rawProperties.size()} properties: ${rawProperties.stringPropertyNames()}")
assert rawProperties.size() == 1
// Act
NiFiProperties niFiProperties = new StandardNiFiProperties(rawProperties)
logger.info("niFiProperties has ${niFiProperties.size()} properties: ${niFiProperties.getPropertyKeys()}")
// Assert
assert niFiProperties.size() == 1
assert niFiProperties.getPropertyKeys() == ["key"] as Set
}
@Test
public void testConstructorShouldAcceptNiFiProperties() throws Exception {
// Arrange
Properties rawProperties = new Properties()
rawProperties.setProperty("key", "value")
rawProperties.setProperty("key.protected", "value2")
NiFiProperties niFiProperties = new StandardNiFiProperties(rawProperties)
logger.info("niFiProperties has ${niFiProperties.size()} properties: ${niFiProperties.getPropertyKeys()}")
assert niFiProperties.size() == 2
// Act
ProtectedNiFiProperties protectedNiFiProperties = new ProtectedNiFiProperties(niFiProperties)
logger.info("protectedNiFiProperties has ${protectedNiFiProperties.size()} properties: ${protectedNiFiProperties.getPropertyKeys()}")
// Assert
def allKeys = protectedNiFiProperties.getPropertyKeysIncludingProtectionSchemes()
assert allKeys == ["key", "key.protected"] as Set
assert allKeys.size() == niFiProperties.size()
}
@Test
public void testShouldAllowMultipleInstances() throws Exception {
// Arrange
Properties rawProperties = new Properties()
rawProperties.setProperty("key", "value")
logger.info("rawProperties has ${rawProperties.size()} properties: ${rawProperties.stringPropertyNames()}")
assert rawProperties.size() == 1
// Act
NiFiProperties niFiProperties = new StandardNiFiProperties(rawProperties)
logger.info("niFiProperties has ${niFiProperties.size()} properties: ${niFiProperties.getPropertyKeys()}")
NiFiProperties emptyProperties = new StandardNiFiProperties()
logger.info("emptyProperties has ${emptyProperties.size()} properties: ${emptyProperties.getPropertyKeys()}")
// Assert
assert niFiProperties.size() == 1
assert niFiProperties.getPropertyKeys() == ["key"] as Set
assert emptyProperties.size() == 0
assert emptyProperties.getPropertyKeys() == [] as Set
}
@Test
public void testShouldDetectIfPropertyIsSensitive() throws Exception {
// Arrange
final String INSENSITIVE_PROPERTY_KEY = "nifi.ui.banner.text"
final String SENSITIVE_PROPERTY_KEY = "nifi.security.keystorePasswd"
ProtectedNiFiProperties properties = loadFromFile("/conf/nifi.properties")
// Act
boolean bannerIsSensitive = properties.isPropertySensitive(INSENSITIVE_PROPERTY_KEY)
logger.info("${INSENSITIVE_PROPERTY_KEY} is ${bannerIsSensitive ? "SENSITIVE" : "NOT SENSITIVE"}")
boolean passwordIsSensitive = properties.isPropertySensitive(SENSITIVE_PROPERTY_KEY)
logger.info("${SENSITIVE_PROPERTY_KEY} is ${passwordIsSensitive ? "SENSITIVE" : "NOT SENSITIVE"}")
// Assert
assert !bannerIsSensitive
assert passwordIsSensitive
}
@Test
public void testShouldGetDefaultSensitiveProperties() throws Exception {
// Arrange
logger.expected("${DEFAULT_SENSITIVE_PROPERTIES.size()} default sensitive properties: ${DEFAULT_SENSITIVE_PROPERTIES.join(", ")}")
ProtectedNiFiProperties properties = loadFromFile("/conf/nifi.properties")
// Act
List defaultSensitiveProperties = properties.getSensitivePropertyKeys()
logger.info("${defaultSensitiveProperties.size()} default sensitive properties: ${defaultSensitiveProperties.join(", ")}")
// Assert
assert defaultSensitiveProperties.size() == DEFAULT_SENSITIVE_PROPERTIES.size()
assert defaultSensitiveProperties.containsAll(DEFAULT_SENSITIVE_PROPERTIES)
}
@Test
public void testShouldGetAdditionalSensitiveProperties() throws Exception {
// Arrange
def completeSensitiveProperties = DEFAULT_SENSITIVE_PROPERTIES + ["nifi.ui.banner.text", "nifi.version"]
logger.expected("${completeSensitiveProperties.size()} total sensitive properties: ${completeSensitiveProperties.join(", ")}")
ProtectedNiFiProperties properties = loadFromFile("/conf/nifi_with_additional_sensitive_keys.properties")
// Act
List retrievedSensitiveProperties = properties.getSensitivePropertyKeys()
logger.info("${retrievedSensitiveProperties.size()} retrieved sensitive properties: ${retrievedSensitiveProperties.join(", ")}")
// Assert
assert retrievedSensitiveProperties.size() == completeSensitiveProperties.size()
assert retrievedSensitiveProperties.containsAll(completeSensitiveProperties)
}
// TODO: Add negative tests (fuzz additional.keys property, etc.)
@Test
public void testGetAdditionalSensitivePropertiesShouldNotIncludeSelf() throws Exception {
// Arrange
def completeSensitiveProperties = DEFAULT_SENSITIVE_PROPERTIES + ["nifi.ui.banner.text", "nifi.version"]
logger.expected("${completeSensitiveProperties.size()} total sensitive properties: ${completeSensitiveProperties.join(", ")}")
ProtectedNiFiProperties properties = loadFromFile("/conf/nifi_with_additional_sensitive_keys.properties")
// Act
List retrievedSensitiveProperties = properties.getSensitivePropertyKeys()
logger.info("${retrievedSensitiveProperties.size()} retrieved sensitive properties: ${retrievedSensitiveProperties.join(", ")}")
// Assert
assert retrievedSensitiveProperties.size() == completeSensitiveProperties.size()
assert retrievedSensitiveProperties.containsAll(completeSensitiveProperties)
}
/**
* In the default (no protection enabled) scenario, a call to retrieve a sensitive property should return the raw value transparently.
* @throws Exception
*/
@Test
public void testShouldGetUnprotectedValueOfSensitiveProperty() throws Exception {
// Arrange
final String KEYSTORE_PASSWORD_KEY = "nifi.security.keystorePasswd"
final String EXPECTED_KEYSTORE_PASSWORD = "thisIsABadKeystorePassword"
ProtectedNiFiProperties properties = loadFromFile("/conf/nifi_with_sensitive_properties_unprotected.properties")
boolean isSensitive = properties.isPropertySensitive(KEYSTORE_PASSWORD_KEY)
boolean isProtected = properties.isPropertyProtected(KEYSTORE_PASSWORD_KEY)
logger.info("The property is ${isSensitive ? "sensitive" : "not sensitive"} and ${isProtected ? "protected" : "not protected"}")
// Act
String retrievedKeystorePassword = properties.getProperty(KEYSTORE_PASSWORD_KEY)
logger.info("${KEYSTORE_PASSWORD_KEY}: ${retrievedKeystorePassword}")
// Assert
assert retrievedKeystorePassword == EXPECTED_KEYSTORE_PASSWORD
assert isSensitive
assert !isProtected
}
/**
* In the default (no protection enabled) scenario, a call to retrieve a sensitive property (which is empty) should return the raw value transparently.
* @throws Exception
*/
@Test
public void testShouldGetEmptyUnprotectedValueOfSensitiveProperty() throws Exception {
// Arrange
final String TRUSTSTORE_PASSWORD_KEY = "nifi.security.truststorePasswd"
final String EXPECTED_TRUSTSTORE_PASSWORD = ""
ProtectedNiFiProperties properties = loadFromFile("/conf/nifi_with_sensitive_properties_unprotected.properties")
boolean isSensitive = properties.isPropertySensitive(TRUSTSTORE_PASSWORD_KEY)
boolean isProtected = properties.isPropertyProtected(TRUSTSTORE_PASSWORD_KEY)
logger.info("The property is ${isSensitive ? "sensitive" : "not sensitive"} and ${isProtected ? "protected" : "not protected"}")
// Act
NiFiProperties unprotectedProperties = properties.getUnprotectedProperties()
String retrievedTruststorePassword = unprotectedProperties.getProperty(TRUSTSTORE_PASSWORD_KEY)
logger.info("${TRUSTSTORE_PASSWORD_KEY}: ${retrievedTruststorePassword}")
// Assert
assert retrievedTruststorePassword == EXPECTED_TRUSTSTORE_PASSWORD
assert isSensitive
assert !isProtected
}
/**
* The new model no longer needs to maintain the protected state -- it is used as a wrapper/decorator during load to unprotect the sensitive properties and then return an instance of raw properties.
*
* @throws Exception
*/
@Test
public void testShouldGetUnprotectedValueOfSensitivePropertyWhenProtected() throws Exception {
// Arrange
final String KEYSTORE_PASSWORD_KEY = "nifi.security.keystorePasswd"
final String EXPECTED_KEYSTORE_PASSWORD = "thisIsABadKeystorePassword"
ProtectedNiFiProperties properties = loadFromFile("/conf/nifi_with_sensitive_properties_protected_aes.properties")
boolean isSensitive = properties.isPropertySensitive(KEYSTORE_PASSWORD_KEY)
boolean isProtected = properties.isPropertyProtected(KEYSTORE_PASSWORD_KEY)
logger.info("The property is ${isSensitive ? "sensitive" : "not sensitive"} and ${isProtected ? "protected" : "not protected"}")
// Act
NiFiProperties unprotectedProperties = properties.getUnprotectedProperties()
String retrievedKeystorePassword = unprotectedProperties.getProperty(KEYSTORE_PASSWORD_KEY)
logger.info("${KEYSTORE_PASSWORD_KEY}: ${retrievedKeystorePassword}")
// Assert
assert retrievedKeystorePassword == EXPECTED_KEYSTORE_PASSWORD
assert isSensitive
assert isProtected
}
/**
* In the protection enabled scenario, a call to retrieve a sensitive property should handle if the property is protected with an unknown protection scheme.
* @throws Exception
*/
@Test
public void testGetValueOfSensitivePropertyShouldHandleUnknownProtectionScheme() throws Exception {
// Arrange
final String KEYSTORE_PASSWORD_KEY = "nifi.security.keystorePasswd"
// Raw properties
Properties rawProperties = new Properties()
rawProperties.load(new File("src/test/resources/conf/nifi_with_sensitive_properties_protected_unknown.properties").newInputStream())
final String RAW_KEYSTORE_PASSWORD = rawProperties.getProperty(KEYSTORE_PASSWORD_KEY)
logger.info("Raw value for ${KEYSTORE_PASSWORD_KEY}: ${RAW_KEYSTORE_PASSWORD}")
ProtectedNiFiProperties properties = loadFromFile("/conf/nifi_with_sensitive_properties_protected_unknown.properties")
boolean isSensitive = properties.isPropertySensitive(KEYSTORE_PASSWORD_KEY)
boolean isProtected = properties.isPropertyProtected(KEYSTORE_PASSWORD_KEY)
// While the value is "protected", the scheme is not registered, so treat it as raw
logger.info("The property is ${isSensitive ? "sensitive" : "not sensitive"} and ${isProtected ? "protected" : "not protected"}")
// Act
NiFiProperties unprotectedProperties = properties.getUnprotectedProperties()
String retrievedKeystorePassword = unprotectedProperties.getProperty(KEYSTORE_PASSWORD_KEY)
logger.info("${KEYSTORE_PASSWORD_KEY}: ${retrievedKeystorePassword}")
// Assert
assert retrievedKeystorePassword == RAW_KEYSTORE_PASSWORD
assert isSensitive
assert isProtected
}
/**
* In the protection enabled scenario, a call to retrieve a sensitive property should handle if the property is unable to be unprotected due to a malformed value.
* @throws Exception
*/
@Test
public void testGetValueOfSensitivePropertyShouldHandleSingleMalformedValue() throws Exception {
// Arrange
final String KEYSTORE_PASSWORD_KEY = "nifi.security.keystorePasswd"
// Raw properties
Properties rawProperties = new Properties()
rawProperties.load(new File("src/test/resources/conf/nifi_with_sensitive_properties_protected_aes_single_malformed.properties").newInputStream())
final String RAW_KEYSTORE_PASSWORD = rawProperties.getProperty(KEYSTORE_PASSWORD_KEY)
logger.info("Raw value for ${KEYSTORE_PASSWORD_KEY}: ${RAW_KEYSTORE_PASSWORD}")
ProtectedNiFiProperties properties = loadFromFile("/conf/nifi_with_sensitive_properties_protected_aes_single_malformed.properties")
boolean isSensitive = properties.isPropertySensitive(KEYSTORE_PASSWORD_KEY)
boolean isProtected = properties.isPropertyProtected(KEYSTORE_PASSWORD_KEY)
logger.info("The property is ${isSensitive ? "sensitive" : "not sensitive"} and ${isProtected ? "protected" : "not protected"}")
// Act
def msg = shouldFail(SensitivePropertyProtectionException) {
NiFiProperties unprotectedProperties = properties.getUnprotectedProperties()
String retrievedKeystorePassword = unprotectedProperties.getProperty(KEYSTORE_PASSWORD_KEY)
logger.info("${KEYSTORE_PASSWORD_KEY}: ${retrievedKeystorePassword}")
}
logger.expected(msg)
// Assert
assert msg =~ "Failed to unprotect key ${KEYSTORE_PASSWORD_KEY}"
assert isSensitive
assert isProtected
}
/**
* In the protection enabled scenario, a call to retrieve a sensitive property should handle if the property is unable to be unprotected due to a malformed value.
* @throws Exception
*/
@Test
public void testGetValueOfSensitivePropertyShouldHandleMultipleMalformedValues() throws Exception {
// Arrange
// Raw properties
Properties rawProperties = new Properties()
rawProperties.load(new File("src/test/resources/conf/nifi_with_sensitive_properties_protected_aes_multiple_malformed.properties").newInputStream())
ProtectedNiFiProperties properties = loadFromFile("/conf/nifi_with_sensitive_properties_protected_aes_multiple_malformed.properties")
// Iterate over the protected keys and track the ones that fail to decrypt
SensitivePropertyProvider spp = new AESSensitivePropertyProvider(KEY_HEX)
Set<String> malformedKeys = properties.getProtectedPropertyKeys()
.findAll { String key, String scheme -> scheme == spp.identifierKey }
.keySet().collect { String key ->
try {
String rawValue = spp.unprotect(properties.getProperty(key))
return
} catch (SensitivePropertyProtectionException e) {
logger.expected("Caught a malformed value for ${key}")
return key
}
}
logger.expected("Malformed keys: ${malformedKeys.join(", ")}")
// Act
def e = groovy.test.GroovyAssert.shouldFail(SensitivePropertyProtectionException) {
NiFiProperties unprotectedProperties = properties.getUnprotectedProperties()
}
logger.expected(e.getMessage())
// Assert
assert e instanceof MultipleSensitivePropertyProtectionException
assert e.getMessage() =~ "Failed to unprotect keys"
assert e.getFailedKeys() == malformedKeys
}
/**
* In the default (no protection enabled) scenario, a call to retrieve a sensitive property (which is empty) should return the raw value transparently.
* @throws Exception
*/
@Test
public void testShouldGetEmptyUnprotectedValueOfSensitivePropertyWithDefault() throws Exception {
// Arrange
final String TRUSTSTORE_PASSWORD_KEY = "nifi.security.truststorePasswd"
final String EXPECTED_TRUSTSTORE_PASSWORD = ""
final String DEFAULT_VALUE = "defaultValue"
// Raw properties
Properties rawProperties = new Properties()
rawProperties.load(new File("src/test/resources/conf/nifi_with_sensitive_properties_unprotected.properties").newInputStream())
final String RAW_TRUSTSTORE_PASSWORD = rawProperties.getProperty(TRUSTSTORE_PASSWORD_KEY)
logger.info("Raw value for ${TRUSTSTORE_PASSWORD_KEY}: ${RAW_TRUSTSTORE_PASSWORD}")
assert RAW_TRUSTSTORE_PASSWORD == EXPECTED_TRUSTSTORE_PASSWORD
ProtectedNiFiProperties properties = loadFromFile("/conf/nifi_with_sensitive_properties_unprotected.properties")
boolean isSensitive = properties.isPropertySensitive(TRUSTSTORE_PASSWORD_KEY)
boolean isProtected = properties.isPropertyProtected(TRUSTSTORE_PASSWORD_KEY)
logger.info("The property is ${isSensitive ? "sensitive" : "not sensitive"} and ${isProtected ? "protected" : "not protected"}")
// Act
String retrievedTruststorePassword = properties.getProperty(TRUSTSTORE_PASSWORD_KEY, DEFAULT_VALUE)
logger.info("${TRUSTSTORE_PASSWORD_KEY}: ${retrievedTruststorePassword}")
// Assert
assert retrievedTruststorePassword == DEFAULT_VALUE
assert isSensitive
assert !isProtected
}
/**
* In the protection enabled scenario, a call to retrieve a sensitive property should return the raw value transparently.
* @throws Exception
*/
@Test
public void testShouldGetUnprotectedValueOfSensitivePropertyWhenProtectedWithDefault() throws Exception {
// Arrange
final String KEYSTORE_PASSWORD_KEY = "nifi.security.keystorePasswd"
final String EXPECTED_KEYSTORE_PASSWORD = "thisIsABadKeystorePassword"
final String DEFAULT_VALUE = "defaultValue"
// Raw properties
Properties rawProperties = new Properties()
rawProperties.load(new File("src/test/resources/conf/nifi_with_sensitive_properties_protected_aes.properties").newInputStream())
final String RAW_KEYSTORE_PASSWORD = rawProperties.getProperty(KEYSTORE_PASSWORD_KEY)
logger.info("Raw value for ${KEYSTORE_PASSWORD_KEY}: ${RAW_KEYSTORE_PASSWORD}")
ProtectedNiFiProperties properties = loadFromFile("/conf/nifi_with_sensitive_properties_protected_aes.properties")
boolean isSensitive = properties.isPropertySensitive(KEYSTORE_PASSWORD_KEY)
boolean isProtected = properties.isPropertyProtected(KEYSTORE_PASSWORD_KEY)
logger.info("The property is ${isSensitive ? "sensitive" : "not sensitive"} and ${isProtected ? "protected" : "not protected"}")
// Act
NiFiProperties unprotectedProperties = properties.getUnprotectedProperties()
String retrievedKeystorePassword = unprotectedProperties.getProperty(KEYSTORE_PASSWORD_KEY, DEFAULT_VALUE)
logger.info("${KEYSTORE_PASSWORD_KEY}: ${retrievedKeystorePassword}")
// Assert
assert retrievedKeystorePassword == EXPECTED_KEYSTORE_PASSWORD
assert isSensitive
assert isProtected
}
// TODO: Test getProtected with multiple providers
/**
* In the protection enabled scenario, a call to retrieve a sensitive property should handle if the internal cache of providers is empty.
* @throws Exception
*/
@Test
public void testGetValueOfSensitivePropertyShouldHandleInvalidatedInternalCache() throws Exception {
// Arrange
final String KEYSTORE_PASSWORD_KEY = "nifi.security.keystorePasswd"
final String EXPECTED_KEYSTORE_PASSWORD = "thisIsABadKeystorePassword"
ProtectedNiFiProperties properties = loadFromFile("/conf/nifi_with_sensitive_properties_protected_aes.properties")
final String RAW_PASSWORD = properties.getProperty(KEYSTORE_PASSWORD_KEY)
logger.info("Read raw value from properties: ${RAW_PASSWORD}")
// Overwrite the internal cache
properties.localProviderCache = [:]
boolean isSensitive = properties.isPropertySensitive(KEYSTORE_PASSWORD_KEY)
boolean isProtected = properties.isPropertyProtected(KEYSTORE_PASSWORD_KEY)
logger.info("The property is ${isSensitive ? "sensitive" : "not sensitive"} and ${isProtected ? "protected" : "not protected"}")
// Act
NiFiProperties unprotectedProperties = properties.getUnprotectedProperties()
String retrievedKeystorePassword = unprotectedProperties.getProperty(KEYSTORE_PASSWORD_KEY)
logger.info("${KEYSTORE_PASSWORD_KEY}: ${retrievedKeystorePassword}")
// Assert
assert retrievedKeystorePassword == RAW_PASSWORD
assert isSensitive
assert isProtected
}
@Test
public void testShouldDetectIfPropertyIsProtected() throws Exception {
// Arrange
final String UNPROTECTED_PROPERTY_KEY = "nifi.security.truststorePasswd"
final String PROTECTED_PROPERTY_KEY = "nifi.security.keystorePasswd"
ProtectedNiFiProperties properties = loadFromFile("/conf/nifi_with_sensitive_properties_protected_aes.properties")
// Act
boolean unprotectedPasswordIsSensitive = properties.isPropertySensitive(UNPROTECTED_PROPERTY_KEY)
boolean unprotectedPasswordIsProtected = properties.isPropertyProtected(UNPROTECTED_PROPERTY_KEY)
logger.info("${UNPROTECTED_PROPERTY_KEY} is ${unprotectedPasswordIsSensitive ? "SENSITIVE" : "NOT SENSITIVE"}")
logger.info("${UNPROTECTED_PROPERTY_KEY} is ${unprotectedPasswordIsProtected ? "PROTECTED" : "NOT PROTECTED"}")
boolean protectedPasswordIsSensitive = properties.isPropertySensitive(PROTECTED_PROPERTY_KEY)
boolean protectedPasswordIsProtected = properties.isPropertyProtected(PROTECTED_PROPERTY_KEY)
logger.info("${PROTECTED_PROPERTY_KEY} is ${protectedPasswordIsSensitive ? "SENSITIVE" : "NOT SENSITIVE"}")
logger.info("${PROTECTED_PROPERTY_KEY} is ${protectedPasswordIsProtected ? "PROTECTED" : "NOT PROTECTED"}")
// Assert
assert unprotectedPasswordIsSensitive
assert !unprotectedPasswordIsProtected
assert protectedPasswordIsSensitive
assert protectedPasswordIsProtected
}
@Test
public void testShouldDetectIfPropertyWithEmptyProtectionSchemeIsProtected() throws Exception {
// Arrange
final String UNPROTECTED_PROPERTY_KEY = "nifi.sensitive.props.key"
ProtectedNiFiProperties properties = loadFromFile("/conf/nifi_with_sensitive_properties_unprotected_extra_line.properties")
// Act
boolean unprotectedPasswordIsSensitive = properties.isPropertySensitive(UNPROTECTED_PROPERTY_KEY)
boolean unprotectedPasswordIsProtected = properties.isPropertyProtected(UNPROTECTED_PROPERTY_KEY)
logger.info("${UNPROTECTED_PROPERTY_KEY} is ${unprotectedPasswordIsSensitive ? "SENSITIVE" : "NOT SENSITIVE"}")
logger.info("${UNPROTECTED_PROPERTY_KEY} is ${unprotectedPasswordIsProtected ? "PROTECTED" : "NOT PROTECTED"}")
// Assert
assert unprotectedPasswordIsSensitive
assert !unprotectedPasswordIsProtected
}
@Test
public void testShouldGetPercentageOfSensitivePropertiesProtected_0() throws Exception {
// Arrange
ProtectedNiFiProperties properties = loadFromFile("/conf/nifi.properties")
logger.info("Sensitive property keys: ${properties.getSensitivePropertyKeys()}")
logger.info("Protected property keys: ${properties.getProtectedPropertyKeys().keySet()}")
// Act
double percentProtected = properties.getPercentOfSensitivePropertiesProtected()
logger.info("${percentProtected}% (${properties.getProtectedPropertyKeys().size()} of ${properties.getSensitivePropertyKeys().size()}) protected")
// Assert
assert percentProtected == 0.0
}
@Test
public void testShouldGetPercentageOfSensitivePropertiesProtected_50() throws Exception {
// Arrange
ProtectedNiFiProperties properties = loadFromFile("/conf/nifi_with_sensitive_properties_protected_aes.properties")
logger.info("Sensitive property keys: ${properties.getSensitivePropertyKeys()}")
logger.info("Protected property keys: ${properties.getProtectedPropertyKeys().keySet()}")
// Act
double percentProtected = properties.getPercentOfSensitivePropertiesProtected()
logger.info("${percentProtected}% (${properties.getProtectedPropertyKeys().size()} of ${properties.getSensitivePropertyKeys().size()}) protected")
// Assert
assert percentProtected == 50.0
}
@Test
public void testShouldGetPercentageOfSensitivePropertiesProtected_100() throws Exception {
// Arrange
ProtectedNiFiProperties properties = loadFromFile("/conf/nifi_with_all_sensitive_properties_protected_aes.properties")
logger.info("Sensitive property keys: ${properties.getSensitivePropertyKeys()}")
logger.info("Protected property keys: ${properties.getProtectedPropertyKeys().keySet()}")
// Act
double percentProtected = properties.getPercentOfSensitivePropertiesProtected()
logger.info("${percentProtected}% (${properties.getProtectedPropertyKeys().size()} of ${properties.getSensitivePropertyKeys().size()}) protected")
// Assert
assert percentProtected == 100.0
}
@Test
public void testInstanceWithNoProtectedPropertiesShouldNotLoadSPP() throws Exception {
// Arrange
ProtectedNiFiProperties properties = loadFromFile("/conf/nifi.properties")
assert properties.@localProviderCache?.isEmpty()
logger.info("Has protected properties: ${properties.hasProtectedKeys()}")
assert !properties.hasProtectedKeys()
// Act
Map localCache = properties.@localProviderCache
logger.info("Internal cache ${localCache} has ${localCache.size()} providers loaded")
// Assert
assert localCache.isEmpty()
}
@Test
public void testShouldAddSensitivePropertyProvider() throws Exception {
// Arrange
ProtectedNiFiProperties properties = new ProtectedNiFiProperties()
assert properties.getSensitivePropertyProviders().isEmpty()
SensitivePropertyProvider mockProvider =
[unprotect : { String input ->
logger.mock("Mock call to #unprotect(${input})")
input.reverse()
},
getIdentifierKey: { -> "mockProvider" }] as SensitivePropertyProvider
// Act
properties.addSensitivePropertyProvider(mockProvider)
// Assert
assert properties.getSensitivePropertyProviders().size() == 1
}
@Test
public void testShouldNotAddNullSensitivePropertyProvider() throws Exception {
// Arrange
ProtectedNiFiProperties properties = new ProtectedNiFiProperties()
assert properties.getSensitivePropertyProviders().isEmpty()
// Act
def msg = shouldFail(IllegalArgumentException) {
properties.addSensitivePropertyProvider(null)
}
logger.expected(msg)
// Assert
assert properties.getSensitivePropertyProviders().size() == 0
assert msg == "Cannot add null SensitivePropertyProvider"
}
@Test
public void testShouldNotAllowOverwriteOfProvider() throws Exception {
// Arrange
ProtectedNiFiProperties properties = new ProtectedNiFiProperties()
assert properties.getSensitivePropertyProviders().isEmpty()
SensitivePropertyProvider mockProvider =
[unprotect : { String input ->
logger.mock("Mock call to 1#unprotect(${input})")
input.reverse()
},
getIdentifierKey: { -> "mockProvider" }] as SensitivePropertyProvider
properties.addSensitivePropertyProvider(mockProvider)
assert properties.getSensitivePropertyProviders().size() == 1
SensitivePropertyProvider mockProvider2 =
[unprotect : { String input ->
logger.mock("Mock call to 2#unprotect(${input})")
input.reverse()
},
getIdentifierKey: { -> "mockProvider" }] as SensitivePropertyProvider
// Act
def msg = shouldFail(UnsupportedOperationException) {
properties.addSensitivePropertyProvider(mockProvider2)
}
logger.expected(msg)
// Assert
assert msg == "Cannot overwrite existing sensitive property provider registered for mockProvider"
assert properties.getSensitivePropertyProviders().size() == 1
}
@Test
void testGetUnprotectedPropertiesShouldReturnInternalInstanceWhenNoneProtected() {
// Arrange
String noProtectedPropertiesPath = "/conf/nifi.properties"
ProtectedNiFiProperties protectedNiFiProperties = loadFromFile(noProtectedPropertiesPath)
logger.info("Loaded ${protectedNiFiProperties.size()} properties from ${noProtectedPropertiesPath}")
int hashCode = protectedNiFiProperties.internalNiFiProperties.hashCode()
logger.info("Hash code of internal instance: ${hashCode}")
// Act
NiFiProperties unprotectedNiFiProperties = protectedNiFiProperties.getUnprotectedProperties()
logger.info("Unprotected ${unprotectedNiFiProperties.size()} properties")
// Assert
assert unprotectedNiFiProperties.size() == protectedNiFiProperties.size()
assert unprotectedNiFiProperties.getPropertyKeys().every {
!unprotectedNiFiProperties.getProperty(it).endsWith(".protected")
}
logger.info("Hash code from returned unprotected instance: ${unprotectedNiFiProperties.hashCode()}")
assert unprotectedNiFiProperties.hashCode() == hashCode
}
@Test
void testGetUnprotectedPropertiesShouldDecryptProtectedProperties() {
// Arrange
String noProtectedPropertiesPath = "/conf/nifi_with_sensitive_properties_protected_aes.properties"
ProtectedNiFiProperties protectedNiFiProperties = loadFromFile(noProtectedPropertiesPath)
logger.info("Loaded ${protectedNiFiProperties.size()} properties from ${noProtectedPropertiesPath}")
int protectedPropertyCount = protectedNiFiProperties.getProtectedPropertyKeys().size()
int protectionSchemeCount = protectedNiFiProperties
.getPropertyKeys().findAll { it.endsWith(".protected") }
.size()
int expectedUnprotectedPropertyCount = protectedNiFiProperties.size() - protectionSchemeCount
String protectedProps = protectedNiFiProperties
.getProtectedPropertyKeys()
.collectEntries {
[(it.key): protectedNiFiProperties.getProperty(it.key)]
}.entrySet()
.join("\n")
logger.info("Detected ${protectedPropertyCount} protected properties and ${protectionSchemeCount} protection scheme properties")
logger.info("Protected properties: \n${protectedProps}")
logger.info("Expected unprotected property count: ${expectedUnprotectedPropertyCount}")
int hashCode = protectedNiFiProperties.internalNiFiProperties.hashCode()
logger.info("Hash code of internal instance: ${hashCode}")
// Act
NiFiProperties unprotectedNiFiProperties = protectedNiFiProperties.getUnprotectedProperties()
logger.info("Unprotected ${unprotectedNiFiProperties.size()} properties")
// Assert
assert unprotectedNiFiProperties.size() == expectedUnprotectedPropertyCount
assert unprotectedNiFiProperties.getPropertyKeys().every {
!unprotectedNiFiProperties.getProperty(it).endsWith(".protected")
}
logger.info("Hash code from returned unprotected instance: ${unprotectedNiFiProperties.hashCode()}")
assert unprotectedNiFiProperties.hashCode() != hashCode
}
@Test
void testShouldCalculateSize() {
// Arrange
Properties rawProperties = [key: "protectedValue", "key.protected": "scheme", "key2": "value2"] as Properties
ProtectedNiFiProperties protectedNiFiProperties = new ProtectedNiFiProperties(rawProperties)
logger.info("Raw properties (${rawProperties.size()}): ${rawProperties.keySet().join(", ")}")
// Act
int protectedSize = protectedNiFiProperties.size()
logger.info("Protected properties (${protectedNiFiProperties.size()}): ${protectedNiFiProperties.getPropertyKeys().join(", ")}")
// Assert
assert protectedSize == rawProperties.size() - 1
}
@Test
void testGetPropertyKeysShouldMatchSize() {
// Arrange
Properties rawProperties = [key: "protectedValue", "key.protected": "scheme", "key2": "value2"] as Properties
ProtectedNiFiProperties protectedNiFiProperties = new ProtectedNiFiProperties(rawProperties)
logger.info("Raw properties (${rawProperties.size()}): ${rawProperties.keySet().join(", ")}")
// Act
def filteredKeys = protectedNiFiProperties.getPropertyKeys()
logger.info("Protected properties (${protectedNiFiProperties.size()}): ${filteredKeys.join(", ")}")
// Assert
assert protectedNiFiProperties.size() == rawProperties.size() - 1
assert filteredKeys == rawProperties.keySet() - "key.protected"
}
@Test
void testShouldGetPropertyKeysIncludingProtectionSchemes() {
// Arrange
Properties rawProperties = [key: "protectedValue", "key.protected": "scheme", "key2": "value2"] as Properties
ProtectedNiFiProperties protectedNiFiProperties = new ProtectedNiFiProperties(rawProperties)
logger.info("Raw properties (${rawProperties.size()}): ${rawProperties.keySet().join(", ")}")
// Act
def allKeys = protectedNiFiProperties.getPropertyKeysIncludingProtectionSchemes()
logger.info("Protected properties with schemes (${allKeys.size()}): ${allKeys.join(", ")}")
// Assert
assert allKeys.size() == rawProperties.size()
assert allKeys == rawProperties.keySet()
}
// TODO: Add tests for protectPlainProperties
}

150
nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-properties-loader/src/test/groovy/org/apache/nifi/properties/StandardNiFiPropertiesGroovyTest.groovy Normal file
View File

@ -0,0 +1,150 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.nifi.properties
import org.apache.nifi.util.NiFiProperties
import org.junit.After
import org.junit.AfterClass
import org.junit.Before
import org.junit.BeforeClass
import org.junit.Test
import org.junit.runner.RunWith
import org.junit.runners.JUnit4
import org.slf4j.Logger
import org.slf4j.LoggerFactory
@RunWith(JUnit4.class)
class StandardNiFiPropertiesGroovyTest extends GroovyTestCase {
private static final Logger logger = LoggerFactory.getLogger(StandardNiFiPropertiesGroovyTest.class)
private static String originalPropertiesPath = System.getProperty(NiFiProperties.PROPERTIES_FILE_PATH)
@BeforeClass
public static void setUpOnce() throws Exception {
logger.metaClass.methodMissing = { String name, args ->
logger.info("[${name?.toUpperCase()}] ${(args as List).join(" ")}")
}
}
@Before
public void setUp() throws Exception {
}
@After
public void tearDown() throws Exception {
}
@AfterClass
public static void tearDownOnce() {
if (originalPropertiesPath) {
System.setProperty(NiFiProperties.PROPERTIES_FILE_PATH, originalPropertiesPath)
}
}
private static StandardNiFiProperties loadFromFile(String propertiesFilePath) {
String filePath;
try {
filePath = StandardNiFiPropertiesGroovyTest.class.getResource(propertiesFilePath).toURI().getPath();
} catch (URISyntaxException ex) {
throw new RuntimeException("Cannot load properties file due to "
+ ex.getLocalizedMessage(), ex);
}
System.setProperty(NiFiProperties.PROPERTIES_FILE_PATH, filePath);
StandardNiFiProperties properties = new StandardNiFiProperties();
// clear out existing properties
for (String prop : properties.stringPropertyNames()) {
properties.remove(prop);
}
InputStream inStream = null;
try {
inStream = new BufferedInputStream(new FileInputStream(filePath));
properties.load(inStream);
} catch (final Exception ex) {
throw new RuntimeException("Cannot load properties file due to "
+ ex.getLocalizedMessage(), ex);
} finally {
if (null != inStream) {
try {
inStream.close();
} catch (Exception ex) {
/**
* do nothing *
*/
}
}
}
return properties;
}
@Test
public void testConstructorShouldCreateNewInstance() throws Exception {
// Arrange
// Act
NiFiProperties niFiProperties = new StandardNiFiProperties()
logger.info("niFiProperties has ${niFiProperties.size()} properties: ${niFiProperties.getPropertyKeys()}")
// Assert
assert niFiProperties.size() == 0
assert niFiProperties.getPropertyKeys() == [] as Set
}
@Test
public void testConstructorShouldAcceptRawProperties() throws Exception {
// Arrange
Properties rawProperties = new Properties()
rawProperties.setProperty("key", "value")
logger.info("rawProperties has ${rawProperties.size()} properties: ${rawProperties.stringPropertyNames()}")
assert rawProperties.size() == 1
// Act
NiFiProperties niFiProperties = new StandardNiFiProperties(rawProperties)
logger.info("niFiProperties has ${niFiProperties.size()} properties: ${niFiProperties.getPropertyKeys()}")
// Assert
assert niFiProperties.size() == 1
assert niFiProperties.getPropertyKeys() == ["key"] as Set
}
@Test
public void testShouldAllowMultipleInstances() throws Exception {
// Arrange
Properties rawProperties = new Properties()
rawProperties.setProperty("key", "value")
logger.info("rawProperties has ${rawProperties.size()} properties: ${rawProperties.stringPropertyNames()}")
assert rawProperties.size() == 1
// Act
NiFiProperties niFiProperties = new StandardNiFiProperties(rawProperties)
logger.info("niFiProperties has ${niFiProperties.size()} properties: ${niFiProperties.getPropertyKeys()}")
NiFiProperties emptyProperties = new StandardNiFiProperties()
logger.info("emptyProperties has ${emptyProperties.size()} properties: ${emptyProperties.getPropertyKeys()}")
// Assert
assert niFiProperties.size() == 1
assert niFiProperties.getPropertyKeys() == ["key"] as Set
assert emptyProperties.size() == 0
assert emptyProperties.getPropertyKeys() == [] as Set
}
}

74
nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-properties-loader/src/test/resources/bootstrap_tests/conf/bootstrap.conf Normal file
View File

@ -0,0 +1,74 @@
#
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
# Java command to use when running NiFi
java=java
# Username to use when running NiFi. This value will be ignored on Windows.
run.as=
# Configure where NiFi's lib and conf directories live
lib.dir=./lib
conf.dir=./conf
# How long to wait after telling NiFi to shutdown before explicitly killing the Process
graceful.shutdown.seconds=20
# Disable JSR 199 so that we can use JSP's without running a JDK
java.arg.1=-Dorg.apache.jasper.compiler.disablejsr199=true
# JVM memory settings
java.arg.2=-Xms512m
java.arg.3=-Xmx512m
# Enable Remote Debugging
#java.arg.debug=-agentlib:jdwp=transport=dt_socket,server=y,suspend=n,address=8000
java.arg.4=-Djava.net.preferIPv4Stack=true
# allowRestrictedHeaders is required for Cluster/Node communications to work properly
java.arg.5=-Dsun.net.http.allowRestrictedHeaders=true
java.arg.6=-Djava.protocol.handler.pkgs=sun.net.www.protocol
# The G1GC is still considered experimental but has proven to be very advantageous in providing great
# performance without significant "stop-the-world" delays.
java.arg.13=-XX:+UseG1GC
#Set headless mode by default
java.arg.14=-Djava.awt.headless=true
# Master key in hexadecimal format for encrypted sensitive configuration values
nifi.bootstrap.sensitive.key=0123456789ABCDEFFEDCBA98765432100123456789ABCDEFFEDCBA9876543210
###
# Notification Services for notifying interested parties when NiFi is stopped, started, dies
###
# XML File that contains the definitions of the notification services
notification.services.file=./conf/bootstrap-notification-services.xml
# In the case that we are unable to send a notification for an event, how many times should we retry?
notification.max.attempts=5
# Comma-separated list of identifiers that are present in the notification.services.file; which services should be used to notify when NiFi is started?
#nifi.start.notification.services=email-notification
# Comma-separated list of identifiers that are present in the notification.services.file; which services should be used to notify when NiFi is stopped?
#nifi.stop.notification.services=email-notification
# Comma-separated list of identifiers that are present in the notification.services.file; which services should be used to notify when NiFi dies?
#nifi.dead.notification.services=email-notification

129
nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-properties-loader/src/test/resources/bootstrap_tests/conf/nifi_with_sensitive_properties_protected_aes.properties Normal file
View File

@ -0,0 +1,129 @@
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# Core Properties #
nifi.version=nifi-test 3.0.0
nifi.flow.configuration.file=./target/flow.xml.gz
nifi.flow.configuration.archive.dir=./target/archive/
nifi.flowcontroller.autoResumeState=true
nifi.flowcontroller.graceful.shutdown.period=10 sec
nifi.flowservice.writedelay.interval=2 sec
nifi.administrative.yield.duration=30 sec
nifi.reporting.task.configuration.file=./target/reporting-tasks.xml
nifi.controller.service.configuration.file=./target/controller-services.xml
nifi.templates.directory=./target/templates
nifi.ui.banner.text=UI Banner Text
nifi.ui.autorefresh.interval=30 sec
nifi.nar.library.directory=./target/resources/NiFiProperties/lib/
nifi.nar.library.directory.alt=./target/resources/NiFiProperties/lib2/
nifi.nar.working.directory=./target/work/nar/
# H2 Settings
nifi.database.directory=./target/database_repository
nifi.h2.url.append=;LOCK_TIMEOUT=25000;WRITE_DELAY=0;AUTO_SERVER=FALSE
# FlowFile Repository
nifi.flowfile.repository.directory=./target/test-repo
nifi.flowfile.repository.partitions=1
nifi.flowfile.repository.checkpoint.interval=2 mins
nifi.queue.swap.threshold=20000
nifi.swap.storage.directory=./target/test-repo/swap
nifi.swap.in.period=5 sec
nifi.swap.in.threads=1
nifi.swap.out.period=5 sec
nifi.swap.out.threads=4
# Content Repository
nifi.content.claim.max.appendable.size=10 MB
nifi.content.claim.max.flow.files=100
nifi.content.repository.directory.default=./target/content_repository
# Provenance Repository Properties
nifi.provenance.repository.storage.directory=./target/provenance_repository
nifi.provenance.repository.max.storage.time=24 hours
nifi.provenance.repository.max.storage.size=1 GB
nifi.provenance.repository.rollover.time=30 secs
nifi.provenance.repository.rollover.size=100 MB
# Site to Site properties
nifi.remote.input.socket.port=9990
nifi.remote.input.secure=true
# web properties #
nifi.web.war.directory=./target/lib
nifi.web.http.host=
nifi.web.http.port=
nifi.web.https.host=nifi.nifi.apache.org
nifi.web.https.port=8443
nifi.web.jetty.working.directory=./target/work/jetty
# security properties #
nifi.sensitive.props.key=n2z+tTTbHuZ4V4V2||uWhdasyDXD4ZG2lMAes/vqh6u4vaz4xgL4aEbF4Y/dXevqk3ulRcOwf1vc4RDQ==
nifi.sensitive.props.key.protected=aes/gcm/256
nifi.sensitive.props.algorithm=PBEWITHMD5AND256BITAES-CBC-OPENSSL
nifi.sensitive.props.provider=BC
nifi.sensitive.props.additional.keys=
nifi.security.keystore=/path/to/keystore.jks
nifi.security.keystoreType=JKS
nifi.security.keystorePasswd=oBjT92hIGRElIGOh||MZ6uYuWNBrOA6usq/Jt3DaD2e4otNirZDytac/w/KFe0HOkrJR03vcbo
nifi.security.keystorePasswd.protected=aes/gcm/256
nifi.security.keyPasswd=ac/BaE35SL/esLiJ||+ULRvRLYdIDA2VqpE0eQXDEMjaLBMG2kbKOdOwBk/hGebDKlVg==
nifi.security.keyPasswd.protected=aes/gcm/256
nifi.security.truststore=
nifi.security.truststoreType=
nifi.security.truststorePasswd=/X/RSlNr2QCJ1Kwe||dENJevX5P61ix+97airrtoBQoyasMFS6DG6fHbX+SZtw2VAMllSSnDeT97Q=
nifi.security.truststorePasswd.protected=aes/gcm/256
nifi.security.needClientAuth=
nifi.security.user.authorizer=
# cluster common properties (cluster manager and nodes must have same values) #
nifi.cluster.protocol.heartbeat.interval=5 sec
nifi.cluster.protocol.is.secure=false
nifi.cluster.protocol.socket.timeout=30 sec
nifi.cluster.protocol.connection.handshake.timeout=45 sec
# if multicast is used, then nifi.cluster.protocol.multicast.xxx properties must be configured #
nifi.cluster.protocol.use.multicast=false
nifi.cluster.protocol.multicast.address=
nifi.cluster.protocol.multicast.port=
nifi.cluster.protocol.multicast.service.broadcast.delay=500 ms
nifi.cluster.protocol.multicast.service.locator.attempts=3
nifi.cluster.protocol.multicast.service.locator.attempts.delay=1 sec
# cluster node properties (only configure for cluster nodes) #
nifi.cluster.is.node=false
nifi.cluster.node.address=
nifi.cluster.node.protocol.port=
nifi.cluster.node.protocol.threads=2
# if multicast is not used, nifi.cluster.node.unicast.xxx must have same values as nifi.cluster.manager.xxx #
nifi.cluster.node.unicast.manager.address=
nifi.cluster.node.unicast.manager.protocol.port=
nifi.cluster.node.unicast.manager.authority.provider.port=
# cluster manager properties (only configure for cluster manager) #
nifi.cluster.is.manager=false
nifi.cluster.manager.address=
nifi.cluster.manager.protocol.port=
nifi.cluster.manager.authority.provider.port=
nifi.cluster.manager.authority.provider.threads=10
nifi.cluster.manager.node.firewall.file=
nifi.cluster.manager.node.event.history.size=10
nifi.cluster.manager.node.api.connection.timeout=30 sec
nifi.cluster.manager.node.api.read.timeout=30 sec
nifi.cluster.manager.node.api.request.threads=10
nifi.cluster.manager.flow.retrieval.delay=5 sec
nifi.cluster.manager.protocol.threads=10
nifi.cluster.manager.safemode.duration=0 sec

14
nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-properties-loader/src/test/resources/bootstrap_tests/missing_bootstrap/nifi.properties Normal file
View File

@ -0,0 +1,14 @@
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

74
nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-properties-loader/src/test/resources/bootstrap_tests/missing_key/bootstrap.conf Normal file
View File

@ -0,0 +1,74 @@
#
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
# Java command to use when running NiFi
java=java
# Username to use when running NiFi. This value will be ignored on Windows.
run.as=
# Configure where NiFi's lib and conf directories live
lib.dir=./lib
conf.dir=./conf
# How long to wait after telling NiFi to shutdown before explicitly killing the Process
graceful.shutdown.seconds=20
# Disable JSR 199 so that we can use JSP's without running a JDK
java.arg.1=-Dorg.apache.jasper.compiler.disablejsr199=true
# JVM memory settings
java.arg.2=-Xms512m
java.arg.3=-Xmx512m
# Enable Remote Debugging
#java.arg.debug=-agentlib:jdwp=transport=dt_socket,server=y,suspend=n,address=8000
java.arg.4=-Djava.net.preferIPv4Stack=true
# allowRestrictedHeaders is required for Cluster/Node communications to work properly
java.arg.5=-Dsun.net.http.allowRestrictedHeaders=true
java.arg.6=-Djava.protocol.handler.pkgs=sun.net.www.protocol
# The G1GC is still considered experimental but has proven to be very advantageous in providing great
# performance without significant "stop-the-world" delays.
java.arg.13=-XX:+UseG1GC
#Set headless mode by default
java.arg.14=-Djava.awt.headless=true
# Master key in hexadecimal format for encrypted sensitive configuration values
nifi.bootstrap.sensitive.key=
###
# Notification Services for notifying interested parties when NiFi is stopped, started, dies
###
# XML File that contains the definitions of the notification services
notification.services.file=./conf/bootstrap-notification-services.xml
# In the case that we are unable to send a notification for an event, how many times should we retry?
notification.max.attempts=5
# Comma-separated list of identifiers that are present in the notification.services.file; which services should be used to notify when NiFi is started?
#nifi.start.notification.services=email-notification
# Comma-separated list of identifiers that are present in the notification.services.file; which services should be used to notify when NiFi is stopped?
#nifi.stop.notification.services=email-notification
# Comma-separated list of identifiers that are present in the notification.services.file; which services should be used to notify when NiFi dies?
#nifi.dead.notification.services=email-notification

14
nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-properties-loader/src/test/resources/bootstrap_tests/missing_key/nifi.properties Normal file
View File

@ -0,0 +1,14 @@
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

71
nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-properties-loader/src/test/resources/bootstrap_tests/missing_key_line/bootstrap.conf Normal file
View File

@ -0,0 +1,71 @@
#
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
# Java command to use when running NiFi
java=java
# Username to use when running NiFi. This value will be ignored on Windows.
run.as=
# Configure where NiFi's lib and conf directories live
lib.dir=./lib
conf.dir=./conf
# How long to wait after telling NiFi to shutdown before explicitly killing the Process
graceful.shutdown.seconds=20
# Disable JSR 199 so that we can use JSP's without running a JDK
java.arg.1=-Dorg.apache.jasper.compiler.disablejsr199=true
# JVM memory settings
java.arg.2=-Xms512m
java.arg.3=-Xmx512m
# Enable Remote Debugging
#java.arg.debug=-agentlib:jdwp=transport=dt_socket,server=y,suspend=n,address=8000
java.arg.4=-Djava.net.preferIPv4Stack=true
# allowRestrictedHeaders is required for Cluster/Node communications to work properly
java.arg.5=-Dsun.net.http.allowRestrictedHeaders=true
java.arg.6=-Djava.protocol.handler.pkgs=sun.net.www.protocol
# The G1GC is still considered experimental but has proven to be very advantageous in providing great
# performance without significant "stop-the-world" delays.
java.arg.13=-XX:+UseG1GC
#Set headless mode by default
java.arg.14=-Djava.awt.headless=true
###
# Notification Services for notifying interested parties when NiFi is stopped, started, dies
###
# XML File that contains the definitions of the notification services
notification.services.file=./conf/bootstrap-notification-services.xml
# In the case that we are unable to send a notification for an event, how many times should we retry?
notification.max.attempts=5
# Comma-separated list of identifiers that are present in the notification.services.file; which services should be used to notify when NiFi is started?
#nifi.start.notification.services=email-notification
# Comma-separated list of identifiers that are present in the notification.services.file; which services should be used to notify when NiFi is stopped?
#nifi.stop.notification.services=email-notification
# Comma-separated list of identifiers that are present in the notification.services.file; which services should be used to notify when NiFi dies?
#nifi.dead.notification.services=email-notification

14
nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-properties-loader/src/test/resources/bootstrap_tests/missing_key_line/nifi.properties Normal file
View File

@ -0,0 +1,14 @@
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

74
nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-properties-loader/src/test/resources/bootstrap_tests/unreadable_bootstrap/bootstrap.conf Executable file
View File

@ -0,0 +1,74 @@
#
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
# Java command to use when running NiFi
java=java
# Username to use when running NiFi. This value will be ignored on Windows.
run.as=
# Configure where NiFi's lib and conf directories live
lib.dir=./lib
conf.dir=./conf
# How long to wait after telling NiFi to shutdown before explicitly killing the Process
graceful.shutdown.seconds=20
# Disable JSR 199 so that we can use JSP's without running a JDK
java.arg.1=-Dorg.apache.jasper.compiler.disablejsr199=true
# JVM memory settings
java.arg.2=-Xms512m
java.arg.3=-Xmx512m
# Enable Remote Debugging
#java.arg.debug=-agentlib:jdwp=transport=dt_socket,server=y,suspend=n,address=8000
java.arg.4=-Djava.net.preferIPv4Stack=true
# allowRestrictedHeaders is required for Cluster/Node communications to work properly
java.arg.5=-Dsun.net.http.allowRestrictedHeaders=true
java.arg.6=-Djava.protocol.handler.pkgs=sun.net.www.protocol
# The G1GC is still considered experimental but has proven to be very advantageous in providing great
# performance without significant "stop-the-world" delays.
java.arg.13=-XX:+UseG1GC
#Set headless mode by default
java.arg.14=-Djava.awt.headless=true
# Master key in hexadecimal format for encrypted sensitive configuration values
nifi.bootstrap.sensitive.key=0123456789ABCDEFFEDCBA98765432100123456789ABCDEFFEDCBA9876543210
###
# Notification Services for notifying interested parties when NiFi is stopped, started, dies
###
# XML File that contains the definitions of the notification services
notification.services.file=./conf/bootstrap-notification-services.xml
# In the case that we are unable to send a notification for an event, how many times should we retry?
notification.max.attempts=5
# Comma-separated list of identifiers that are present in the notification.services.file; which services should be used to notify when NiFi is started?
#nifi.start.notification.services=email-notification
# Comma-separated list of identifiers that are present in the notification.services.file; which services should be used to notify when NiFi is stopped?
#nifi.stop.notification.services=email-notification
# Comma-separated list of identifiers that are present in the notification.services.file; which services should be used to notify when NiFi dies?
#nifi.dead.notification.services=email-notification

14
nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-properties-loader/src/test/resources/bootstrap_tests/unreadable_bootstrap/nifi.properties Normal file
View File

@ -0,0 +1,14 @@
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

74
nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-properties-loader/src/test/resources/bootstrap_tests/unreadable_conf/bootstrap.conf Executable file
View File

@ -0,0 +1,74 @@
#
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
# Java command to use when running NiFi
java=java
# Username to use when running NiFi. This value will be ignored on Windows.
run.as=
# Configure where NiFi's lib and conf directories live
lib.dir=./lib
conf.dir=./conf
# How long to wait after telling NiFi to shutdown before explicitly killing the Process
graceful.shutdown.seconds=20
# Disable JSR 199 so that we can use JSP's without running a JDK
java.arg.1=-Dorg.apache.jasper.compiler.disablejsr199=true
# JVM memory settings
java.arg.2=-Xms512m
java.arg.3=-Xmx512m
# Enable Remote Debugging
#java.arg.debug=-agentlib:jdwp=transport=dt_socket,server=y,suspend=n,address=8000
java.arg.4=-Djava.net.preferIPv4Stack=true
# allowRestrictedHeaders is required for Cluster/Node communications to work properly
java.arg.5=-Dsun.net.http.allowRestrictedHeaders=true
java.arg.6=-Djava.protocol.handler.pkgs=sun.net.www.protocol
# The G1GC is still considered experimental but has proven to be very advantageous in providing great
# performance without significant "stop-the-world" delays.
java.arg.13=-XX:+UseG1GC
#Set headless mode by default
java.arg.14=-Djava.awt.headless=true
# Master key in hexadecimal format for encrypted sensitive configuration values
nifi.bootstrap.sensitive.key=0123456789ABCDEFFEDCBA98765432100123456789ABCDEFFEDCBA9876543210
###
# Notification Services for notifying interested parties when NiFi is stopped, started, dies
###
# XML File that contains the definitions of the notification services
notification.services.file=./conf/bootstrap-notification-services.xml
# In the case that we are unable to send a notification for an event, how many times should we retry?
notification.max.attempts=5
# Comma-separated list of identifiers that are present in the notification.services.file; which services should be used to notify when NiFi is started?
#nifi.start.notification.services=email-notification
# Comma-separated list of identifiers that are present in the notification.services.file; which services should be used to notify when NiFi is stopped?
#nifi.stop.notification.services=email-notification
# Comma-separated list of identifiers that are present in the notification.services.file; which services should be used to notify when NiFi dies?
#nifi.dead.notification.services=email-notification

14
nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-properties-loader/src/test/resources/bootstrap_tests/unreadable_conf/nifi.properties Normal file
View File

@ -0,0 +1,14 @@
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

124
nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-properties-loader/src/test/resources/conf/nifi.blank.properties Normal file
View File

@ -0,0 +1,124 @@
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# Core Properties #
nifi.version=nifi-test 3.0.0
nifi.flow.configuration.file=./target/flow.xml.gz
nifi.flow.configuration.archive.dir=./target/archive/
nifi.flowcontroller.autoResumeState=true
nifi.flowcontroller.graceful.shutdown.period=10 sec
nifi.flowservice.writedelay.interval=2 sec
nifi.administrative.yield.duration=30 sec
nifi.reporting.task.configuration.file=./target/reporting-tasks.xml
nifi.controller.service.configuration.file=./target/controller-services.xml
nifi.templates.directory=./target/templates
nifi.ui.banner.text=UI Banner Text
nifi.ui.autorefresh.interval=30 sec
nifi.nar.library.directory=
nifi.custom.nar.library.directory.alt=
nifi.nar.working.directory=./target/work/nar/
# H2 Settings
nifi.database.directory=./target/database_repository
nifi.h2.url.append=;LOCK_TIMEOUT=25000;WRITE_DELAY=0;AUTO_SERVER=FALSE
# FlowFile Repository
nifi.flowfile.repository.directory=./target/test-repo
nifi.flowfile.repository.partitions=1
nifi.flowfile.repository.checkpoint.interval=2 mins
nifi.queue.swap.threshold=20000
nifi.swap.storage.directory=./target/test-repo/swap
nifi.swap.in.period=5 sec
nifi.swap.in.threads=1
nifi.swap.out.period=5 sec
nifi.swap.out.threads=4
# Content Repository
nifi.content.claim.max.appendable.size=10 MB
nifi.content.claim.max.flow.files=100
nifi.content.repository.directory.default=./target/content_repository
# Provenance Repository Properties
nifi.provenance.repository.storage.directory=./target/provenance_repository
nifi.provenance.repository.max.storage.time=24 hours
nifi.provenance.repository.max.storage.size=1 GB
nifi.provenance.repository.rollover.time=30 secs
nifi.provenance.repository.rollover.size=100 MB
# Site to Site properties
nifi.remote.input.socket.port=9990
nifi.remote.input.secure=true
# web properties #
nifi.web.war.directory=./target/lib
nifi.web.http.host=
nifi.web.http.port=8080
nifi.web.https.host=
nifi.web.https.port=
nifi.web.jetty.working.directory=./target/work/jetty
# security properties #
nifi.sensitive.props.key=key
nifi.sensitive.props.algorithm=PBEWITHMD5AND256BITAES-CBC-OPENSSL
nifi.sensitive.props.provider=BC
nifi.security.keystore=
nifi.security.keystoreType=
nifi.security.keystorePasswd=
nifi.security.keyPasswd=
nifi.security.truststore=
nifi.security.truststoreType=
nifi.security.truststorePasswd=
nifi.security.needClientAuth=
nifi.security.user.authorizer=
# cluster common properties (cluster manager and nodes must have same values) #
nifi.cluster.protocol.heartbeat.interval=5 sec
nifi.cluster.protocol.is.secure=false
nifi.cluster.protocol.socket.timeout=30 sec
nifi.cluster.protocol.connection.handshake.timeout=45 sec
# if multicast is used, then nifi.cluster.protocol.multicast.xxx properties must be configured #
nifi.cluster.protocol.use.multicast=false
nifi.cluster.protocol.multicast.address=
nifi.cluster.protocol.multicast.port=
nifi.cluster.protocol.multicast.service.broadcast.delay=500 ms
nifi.cluster.protocol.multicast.service.locator.attempts=3
nifi.cluster.protocol.multicast.service.locator.attempts.delay=1 sec
# cluster node properties (only configure for cluster nodes) #
nifi.cluster.is.node=false
nifi.cluster.node.address=
nifi.cluster.node.protocol.port=
nifi.cluster.node.protocol.threads=2
# if multicast is not used, nifi.cluster.node.unicast.xxx must have same values as nifi.cluster.manager.xxx #
nifi.cluster.node.unicast.manager.address=
nifi.cluster.node.unicast.manager.protocol.port=
nifi.cluster.node.unicast.manager.authority.provider.port=
# cluster manager properties (only configure for cluster manager) #
nifi.cluster.is.manager=false
nifi.cluster.manager.address=
nifi.cluster.manager.protocol.port=
nifi.cluster.manager.authority.provider.port=
nifi.cluster.manager.authority.provider.threads=10
nifi.cluster.manager.node.firewall.file=
nifi.cluster.manager.node.event.history.size=10
nifi.cluster.manager.node.api.connection.timeout=30 sec
nifi.cluster.manager.node.api.read.timeout=30 sec
nifi.cluster.manager.node.api.request.threads=10
nifi.cluster.manager.flow.retrieval.delay=5 sec
nifi.cluster.manager.protocol.threads=10
nifi.cluster.manager.safemode.duration=0 sec

122
nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-properties-loader/src/test/resources/conf/nifi.missing.properties Normal file
View File

@ -0,0 +1,122 @@
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# Core Properties #
nifi.version=nifi-test 3.0.0
nifi.flow.configuration.file=./target/flow.xml.gz
nifi.flow.configuration.archive.dir=./target/archive/
nifi.flowcontroller.autoResumeState=true
nifi.flowcontroller.graceful.shutdown.period=10 sec
nifi.flowservice.writedelay.interval=2 sec
nifi.administrative.yield.duration=30 sec
nifi.reporting.task.configuration.file=./target/reporting-tasks.xml
nifi.controller.service.configuration.file=./target/controller-services.xml
nifi.templates.directory=./target/templates
nifi.ui.banner.text=UI Banner Text
nifi.ui.autorefresh.interval=30 sec
nifi.nar.working.directory=./target/work/nar/
# H2 Settings
nifi.database.directory=./target/database_repository
nifi.h2.url.append=;LOCK_TIMEOUT=25000;WRITE_DELAY=0;AUTO_SERVER=FALSE
# FlowFile Repository
nifi.flowfile.repository.directory=./target/test-repo
nifi.flowfile.repository.partitions=1
nifi.flowfile.repository.checkpoint.interval=2 mins
nifi.queue.swap.threshold=20000
nifi.swap.storage.directory=./target/test-repo/swap
nifi.swap.in.period=5 sec
nifi.swap.in.threads=1
nifi.swap.out.period=5 sec
nifi.swap.out.threads=4
# Content Repository
nifi.content.claim.max.appendable.size=10 MB
nifi.content.claim.max.flow.files=100
nifi.content.repository.directory.default=./target/content_repository
# Provenance Repository Properties
nifi.provenance.repository.storage.directory=./target/provenance_repository
nifi.provenance.repository.max.storage.time=24 hours
nifi.provenance.repository.max.storage.size=1 GB
nifi.provenance.repository.rollover.time=30 secs
nifi.provenance.repository.rollover.size=100 MB
# Site to Site properties
nifi.remote.input.socket.port=9990
nifi.remote.input.secure=true
# web properties #
nifi.web.war.directory=./target/lib
nifi.web.http.host=
nifi.web.http.port=8080
nifi.web.https.host=
nifi.web.https.port=
nifi.web.jetty.working.directory=./target/work/jetty
# security properties #
nifi.sensitive.props.key=key
nifi.sensitive.props.algorithm=PBEWITHMD5AND256BITAES-CBC-OPENSSL
nifi.sensitive.props.provider=BC
nifi.security.keystore=
nifi.security.keystoreType=
nifi.security.keystorePasswd=
nifi.security.keyPasswd=
nifi.security.truststore=
nifi.security.truststoreType=
nifi.security.truststorePasswd=
nifi.security.needClientAuth=
nifi.security.user.authorizer=
# cluster common properties (cluster manager and nodes must have same values) #
nifi.cluster.protocol.heartbeat.interval=5 sec
nifi.cluster.protocol.is.secure=false
nifi.cluster.protocol.socket.timeout=30 sec
nifi.cluster.protocol.connection.handshake.timeout=45 sec
# if multicast is used, then nifi.cluster.protocol.multicast.xxx properties must be configured #
nifi.cluster.protocol.use.multicast=false
nifi.cluster.protocol.multicast.address=
nifi.cluster.protocol.multicast.port=
nifi.cluster.protocol.multicast.service.broadcast.delay=500 ms
nifi.cluster.protocol.multicast.service.locator.attempts=3
nifi.cluster.protocol.multicast.service.locator.attempts.delay=1 sec
# cluster node properties (only configure for cluster nodes) #
nifi.cluster.is.node=false
nifi.cluster.node.address=
nifi.cluster.node.protocol.port=
nifi.cluster.node.protocol.threads=2
# if multicast is not used, nifi.cluster.node.unicast.xxx must have same values as nifi.cluster.manager.xxx #
nifi.cluster.node.unicast.manager.address=
nifi.cluster.node.unicast.manager.protocol.port=
nifi.cluster.node.unicast.manager.authority.provider.port=
# cluster manager properties (only configure for cluster manager) #
nifi.cluster.is.manager=false
nifi.cluster.manager.address=
nifi.cluster.manager.protocol.port=
nifi.cluster.manager.authority.provider.port=
nifi.cluster.manager.authority.provider.threads=10
nifi.cluster.manager.node.firewall.file=
nifi.cluster.manager.node.event.history.size=10
nifi.cluster.manager.node.api.connection.timeout=30 sec
nifi.cluster.manager.node.api.read.timeout=30 sec
nifi.cluster.manager.node.api.request.threads=10
nifi.cluster.manager.flow.retrieval.delay=5 sec
nifi.cluster.manager.protocol.threads=10
nifi.cluster.manager.safemode.duration=0 sec

124
nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-properties-loader/src/test/resources/conf/nifi.properties Normal file
View File

@ -0,0 +1,124 @@
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# Core Properties #
nifi.version=nifi-test 3.0.0
nifi.flow.configuration.file=./target/flow.xml.gz
nifi.flow.configuration.archive.dir=./target/archive/
nifi.flowcontroller.autoResumeState=true
nifi.flowcontroller.graceful.shutdown.period=10 sec
nifi.flowservice.writedelay.interval=2 sec
nifi.administrative.yield.duration=30 sec
nifi.reporting.task.configuration.file=./target/reporting-tasks.xml
nifi.controller.service.configuration.file=./target/controller-services.xml
nifi.templates.directory=./target/templates
nifi.ui.banner.text=UI Banner Text
nifi.ui.autorefresh.interval=30 sec
nifi.nar.library.directory=./target/resources/NiFiProperties/lib/
nifi.nar.library.directory.alt=./target/resources/NiFiProperties/lib2/
nifi.nar.working.directory=./target/work/nar/
# H2 Settings
nifi.database.directory=./target/database_repository
nifi.h2.url.append=;LOCK_TIMEOUT=25000;WRITE_DELAY=0;AUTO_SERVER=FALSE
# FlowFile Repository
nifi.flowfile.repository.directory=./target/test-repo
nifi.flowfile.repository.partitions=1
nifi.flowfile.repository.checkpoint.interval=2 mins
nifi.queue.swap.threshold=20000
nifi.swap.storage.directory=./target/test-repo/swap
nifi.swap.in.period=5 sec
nifi.swap.in.threads=1
nifi.swap.out.period=5 sec
nifi.swap.out.threads=4
# Content Repository
nifi.content.claim.max.appendable.size=10 MB
nifi.content.claim.max.flow.files=100
nifi.content.repository.directory.default=./target/content_repository
# Provenance Repository Properties
nifi.provenance.repository.storage.directory=./target/provenance_repository
nifi.provenance.repository.max.storage.time=24 hours
nifi.provenance.repository.max.storage.size=1 GB
nifi.provenance.repository.rollover.time=30 secs
nifi.provenance.repository.rollover.size=100 MB
# Site to Site properties
nifi.remote.input.socket.port=9990
nifi.remote.input.secure=true
# web properties #
nifi.web.war.directory=./target/lib
nifi.web.http.host=
nifi.web.http.port=8080
nifi.web.https.host=
nifi.web.https.port=
nifi.web.jetty.working.directory=./target/work/jetty
# security properties #
nifi.sensitive.props.key=key
nifi.sensitive.props.algorithm=PBEWITHMD5AND256BITAES-CBC-OPENSSL
nifi.sensitive.props.provider=BC
nifi.security.keystore=
nifi.security.keystoreType=
nifi.security.keystorePasswd=
nifi.security.keyPasswd=
nifi.security.truststore=
nifi.security.truststoreType=
nifi.security.truststorePasswd=
nifi.security.needClientAuth=
nifi.security.user.authorizer=
# cluster common properties (cluster manager and nodes must have same values) #
nifi.cluster.protocol.heartbeat.interval=5 sec
nifi.cluster.protocol.is.secure=false
nifi.cluster.protocol.socket.timeout=30 sec
nifi.cluster.protocol.connection.handshake.timeout=45 sec
# if multicast is used, then nifi.cluster.protocol.multicast.xxx properties must be configured #
nifi.cluster.protocol.use.multicast=false
nifi.cluster.protocol.multicast.address=
nifi.cluster.protocol.multicast.port=
nifi.cluster.protocol.multicast.service.broadcast.delay=500 ms
nifi.cluster.protocol.multicast.service.locator.attempts=3
nifi.cluster.protocol.multicast.service.locator.attempts.delay=1 sec
# cluster node properties (only configure for cluster nodes) #
nifi.cluster.is.node=false
nifi.cluster.node.address=
nifi.cluster.node.protocol.port=
nifi.cluster.node.protocol.threads=2
# if multicast is not used, nifi.cluster.node.unicast.xxx must have same values as nifi.cluster.manager.xxx #
nifi.cluster.node.unicast.manager.address=
nifi.cluster.node.unicast.manager.protocol.port=
nifi.cluster.node.unicast.manager.authority.provider.port=
# cluster manager properties (only configure for cluster manager) #
nifi.cluster.is.manager=false
nifi.cluster.manager.address=
nifi.cluster.manager.protocol.port=
nifi.cluster.manager.authority.provider.port=
nifi.cluster.manager.authority.provider.threads=10
nifi.cluster.manager.node.firewall.file=
nifi.cluster.manager.node.event.history.size=10
nifi.cluster.manager.node.api.connection.timeout=30 sec
nifi.cluster.manager.node.api.read.timeout=30 sec
nifi.cluster.manager.node.api.request.threads=10
nifi.cluster.manager.flow.retrieval.delay=5 sec
nifi.cluster.manager.protocol.threads=10
nifi.cluster.manager.safemode.duration=0 sec

14
nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-properties-loader/src/test/resources/conf/nifi_no_permissions.properties Normal file
View File

@ -0,0 +1,14 @@
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

125
nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-properties-loader/src/test/resources/conf/nifi_with_additional_sensitive_keys.properties Normal file
View File

@ -0,0 +1,125 @@
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# Core Properties #
nifi.version=nifi-test 3.0.0
nifi.flow.configuration.file=./target/flow.xml.gz
nifi.flow.configuration.archive.dir=./target/archive/
nifi.flowcontroller.autoResumeState=true
nifi.flowcontroller.graceful.shutdown.period=10 sec
nifi.flowservice.writedelay.interval=2 sec
nifi.administrative.yield.duration=30 sec
nifi.reporting.task.configuration.file=./target/reporting-tasks.xml
nifi.controller.service.configuration.file=./target/controller-services.xml
nifi.templates.directory=./target/templates
nifi.ui.banner.text=UI Banner Text
nifi.ui.autorefresh.interval=30 sec
nifi.nar.library.directory=./target/resources/NiFiProperties/lib/
nifi.nar.library.directory.alt=./target/resources/NiFiProperties/lib2/
nifi.nar.working.directory=./target/work/nar/
# H2 Settings
nifi.database.directory=./target/database_repository
nifi.h2.url.append=;LOCK_TIMEOUT=25000;WRITE_DELAY=0;AUTO_SERVER=FALSE
# FlowFile Repository
nifi.flowfile.repository.directory=./target/test-repo
nifi.flowfile.repository.partitions=1
nifi.flowfile.repository.checkpoint.interval=2 mins
nifi.queue.swap.threshold=20000
nifi.swap.storage.directory=./target/test-repo/swap
nifi.swap.in.period=5 sec
nifi.swap.in.threads=1
nifi.swap.out.period=5 sec
nifi.swap.out.threads=4
# Content Repository
nifi.content.claim.max.appendable.size=10 MB
nifi.content.claim.max.flow.files=100
nifi.content.repository.directory.default=./target/content_repository
# Provenance Repository Properties
nifi.provenance.repository.storage.directory=./target/provenance_repository
nifi.provenance.repository.max.storage.time=24 hours
nifi.provenance.repository.max.storage.size=1 GB
nifi.provenance.repository.rollover.time=30 secs
nifi.provenance.repository.rollover.size=100 MB
# Site to Site properties
nifi.remote.input.socket.port=9990
nifi.remote.input.secure=true
# web properties #
nifi.web.war.directory=./target/lib
nifi.web.http.host=
nifi.web.http.port=8080
nifi.web.https.host=
nifi.web.https.port=
nifi.web.jetty.working.directory=./target/work/jetty
# security properties #
nifi.sensitive.props.key=key
nifi.sensitive.props.algorithm=PBEWITHMD5AND256BITAES-CBC-OPENSSL
nifi.sensitive.props.provider=BC
nifi.sensitive.props.additional.keys=nifi.ui.banner.text, nifi.version
nifi.security.keystore=
nifi.security.keystoreType=
nifi.security.keystorePasswd=
nifi.security.keyPasswd=
nifi.security.truststore=
nifi.security.truststoreType=
nifi.security.truststorePasswd=
nifi.security.needClientAuth=
nifi.security.user.authorizer=
# cluster common properties (cluster manager and nodes must have same values) #
nifi.cluster.protocol.heartbeat.interval=5 sec
nifi.cluster.protocol.is.secure=false
nifi.cluster.protocol.socket.timeout=30 sec
nifi.cluster.protocol.connection.handshake.timeout=45 sec
# if multicast is used, then nifi.cluster.protocol.multicast.xxx properties must be configured #
nifi.cluster.protocol.use.multicast=false
nifi.cluster.protocol.multicast.address=
nifi.cluster.protocol.multicast.port=
nifi.cluster.protocol.multicast.service.broadcast.delay=500 ms
nifi.cluster.protocol.multicast.service.locator.attempts=3
nifi.cluster.protocol.multicast.service.locator.attempts.delay=1 sec
# cluster node properties (only configure for cluster nodes) #
nifi.cluster.is.node=false
nifi.cluster.node.address=
nifi.cluster.node.protocol.port=
nifi.cluster.node.protocol.threads=2
# if multicast is not used, nifi.cluster.node.unicast.xxx must have same values as nifi.cluster.manager.xxx #
nifi.cluster.node.unicast.manager.address=
nifi.cluster.node.unicast.manager.protocol.port=
nifi.cluster.node.unicast.manager.authority.provider.port=
# cluster manager properties (only configure for cluster manager) #
nifi.cluster.is.manager=false
nifi.cluster.manager.address=
nifi.cluster.manager.protocol.port=
nifi.cluster.manager.authority.provider.port=
nifi.cluster.manager.authority.provider.threads=10
nifi.cluster.manager.node.firewall.file=
nifi.cluster.manager.node.event.history.size=10
nifi.cluster.manager.node.api.connection.timeout=30 sec
nifi.cluster.manager.node.api.read.timeout=30 sec
nifi.cluster.manager.node.api.request.threads=10
nifi.cluster.manager.flow.retrieval.delay=5 sec
nifi.cluster.manager.protocol.threads=10
nifi.cluster.manager.safemode.duration=0 sec

129
nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-properties-loader/src/test/resources/conf/nifi_with_all_sensitive_properties_protected_aes.properties Normal file
View File

@ -0,0 +1,129 @@
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# Core Properties #
nifi.version=nifi-test 3.0.0
nifi.flow.configuration.file=./target/flow.xml.gz
nifi.flow.configuration.archive.dir=./target/archive/
nifi.flowcontroller.autoResumeState=true
nifi.flowcontroller.graceful.shutdown.period=10 sec
nifi.flowservice.writedelay.interval=2 sec
nifi.administrative.yield.duration=30 sec
nifi.reporting.task.configuration.file=./target/reporting-tasks.xml
nifi.controller.service.configuration.file=./target/controller-services.xml
nifi.templates.directory=./target/templates
nifi.ui.banner.text=UI Banner Text
nifi.ui.autorefresh.interval=30 sec
nifi.nar.library.directory=./target/resources/NiFiProperties/lib/
nifi.nar.library.directory.alt=./target/resources/NiFiProperties/lib2/
nifi.nar.working.directory=./target/work/nar/
# H2 Settings
nifi.database.directory=./target/database_repository
nifi.h2.url.append=;LOCK_TIMEOUT=25000;WRITE_DELAY=0;AUTO_SERVER=FALSE
# FlowFile Repository
nifi.flowfile.repository.directory=./target/test-repo
nifi.flowfile.repository.partitions=1
nifi.flowfile.repository.checkpoint.interval=2 mins
nifi.queue.swap.threshold=20000
nifi.swap.storage.directory=./target/test-repo/swap
nifi.swap.in.period=5 sec
nifi.swap.in.threads=1
nifi.swap.out.period=5 sec
nifi.swap.out.threads=4
# Content Repository
nifi.content.claim.max.appendable.size=10 MB
nifi.content.claim.max.flow.files=100
nifi.content.repository.directory.default=./target/content_repository
# Provenance Repository Properties
nifi.provenance.repository.storage.directory=./target/provenance_repository
nifi.provenance.repository.max.storage.time=24 hours
nifi.provenance.repository.max.storage.size=1 GB
nifi.provenance.repository.rollover.time=30 secs
nifi.provenance.repository.rollover.size=100 MB
# Site to Site properties
nifi.remote.input.socket.port=9990
nifi.remote.input.secure=true
# web properties #
nifi.web.war.directory=./target/lib
nifi.web.http.host=
nifi.web.http.port=
nifi.web.https.host=nifi.nifi.apache.org
nifi.web.https.port=8443
nifi.web.jetty.working.directory=./target/work/jetty
# security properties #
nifi.sensitive.props.key=n2z+tTTbHuZ4V4V2||uWhdasyDXD4ZG2lMAes/vqh6u4vaz4xgL4aEbF4Y/dXevqk3ulRcOwf1vc4RDQ==
nifi.sensitive.props.key.protected=aes/gcm/256
nifi.sensitive.props.algorithm=PBEWITHMD5AND256BITAES-CBC-OPENSSL
nifi.sensitive.props.provider=BC
nifi.sensitive.props.additional.keys=
nifi.security.keystore=/path/to/keystore.jks
nifi.security.keystoreType=JKS
nifi.security.keystorePasswd=oBjT92hIGRElIGOh||MZ6uYuWNBrOA6usq/Jt3DaD2e4otNirZDytac/w/KFe0HOkrJR03vcbo
nifi.security.keystorePasswd.protected=aes/gcm/256
nifi.security.keyPasswd=ac/BaE35SL/esLiJ||+ULRvRLYdIDA2VqpE0eQXDEMjaLBMG2kbKOdOwBk/hGebDKlVg==
nifi.security.keyPasswd.protected=aes/gcm/256
nifi.security.truststore=
nifi.security.truststoreType=
nifi.security.truststorePasswd=/X/RSlNr2QCJ1Kwe||dENJevX5P61ix+97airrtoBQoyasMFS6DG6fHbX+SZtw2VAMllSSnDeT97Q=
nifi.security.truststorePasswd.protected=aes/gcm/256
nifi.security.needClientAuth=
nifi.security.user.authorizer=
# cluster common properties (cluster manager and nodes must have same values) #
nifi.cluster.protocol.heartbeat.interval=5 sec
nifi.cluster.protocol.is.secure=false
nifi.cluster.protocol.socket.timeout=30 sec
nifi.cluster.protocol.connection.handshake.timeout=45 sec
# if multicast is used, then nifi.cluster.protocol.multicast.xxx properties must be configured #
nifi.cluster.protocol.use.multicast=false
nifi.cluster.protocol.multicast.address=
nifi.cluster.protocol.multicast.port=
nifi.cluster.protocol.multicast.service.broadcast.delay=500 ms
nifi.cluster.protocol.multicast.service.locator.attempts=3
nifi.cluster.protocol.multicast.service.locator.attempts.delay=1 sec
# cluster node properties (only configure for cluster nodes) #
nifi.cluster.is.node=false
nifi.cluster.node.address=
nifi.cluster.node.protocol.port=
nifi.cluster.node.protocol.threads=2
# if multicast is not used, nifi.cluster.node.unicast.xxx must have same values as nifi.cluster.manager.xxx #
nifi.cluster.node.unicast.manager.address=
nifi.cluster.node.unicast.manager.protocol.port=
nifi.cluster.node.unicast.manager.authority.provider.port=
# cluster manager properties (only configure for cluster manager) #
nifi.cluster.is.manager=false
nifi.cluster.manager.address=
nifi.cluster.manager.protocol.port=
nifi.cluster.manager.authority.provider.port=
nifi.cluster.manager.authority.provider.threads=10
nifi.cluster.manager.node.firewall.file=
nifi.cluster.manager.node.event.history.size=10
nifi.cluster.manager.node.api.connection.timeout=30 sec
nifi.cluster.manager.node.api.read.timeout=30 sec
nifi.cluster.manager.node.api.request.threads=10
nifi.cluster.manager.flow.retrieval.delay=5 sec
nifi.cluster.manager.protocol.threads=10
nifi.cluster.manager.safemode.duration=0 sec

125
nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-properties-loader/src/test/resources/conf/nifi_with_recursive_additional_sensitive_keys.properties Normal file
View File

@ -0,0 +1,125 @@
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# Core Properties #
nifi.version=nifi-test 3.0.0
nifi.flow.configuration.file=./target/flow.xml.gz
nifi.flow.configuration.archive.dir=./target/archive/
nifi.flowcontroller.autoResumeState=true
nifi.flowcontroller.graceful.shutdown.period=10 sec
nifi.flowservice.writedelay.interval=2 sec
nifi.administrative.yield.duration=30 sec
nifi.reporting.task.configuration.file=./target/reporting-tasks.xml
nifi.controller.service.configuration.file=./target/controller-services.xml
nifi.templates.directory=./target/templates
nifi.ui.banner.text=UI Banner Text
nifi.ui.autorefresh.interval=30 sec
nifi.nar.library.directory=./target/resources/NiFiProperties/lib/
nifi.nar.library.directory.alt=./target/resources/NiFiProperties/lib2/
nifi.nar.working.directory=./target/work/nar/
# H2 Settings
nifi.database.directory=./target/database_repository
nifi.h2.url.append=;LOCK_TIMEOUT=25000;WRITE_DELAY=0;AUTO_SERVER=FALSE
# FlowFile Repository
nifi.flowfile.repository.directory=./target/test-repo
nifi.flowfile.repository.partitions=1
nifi.flowfile.repository.checkpoint.interval=2 mins
nifi.queue.swap.threshold=20000
nifi.swap.storage.directory=./target/test-repo/swap
nifi.swap.in.period=5 sec
nifi.swap.in.threads=1
nifi.swap.out.period=5 sec
nifi.swap.out.threads=4
# Content Repository
nifi.content.claim.max.appendable.size=10 MB
nifi.content.claim.max.flow.files=100
nifi.content.repository.directory.default=./target/content_repository
# Provenance Repository Properties
nifi.provenance.repository.storage.directory=./target/provenance_repository
nifi.provenance.repository.max.storage.time=24 hours
nifi.provenance.repository.max.storage.size=1 GB
nifi.provenance.repository.rollover.time=30 secs
nifi.provenance.repository.rollover.size=100 MB
# Site to Site properties
nifi.remote.input.socket.port=9990
nifi.remote.input.secure=true
# web properties #
nifi.web.war.directory=./target/lib
nifi.web.http.host=
nifi.web.http.port=8080
nifi.web.https.host=
nifi.web.https.port=
nifi.web.jetty.working.directory=./target/work/jetty
# security properties #
nifi.sensitive.props.key=key
nifi.sensitive.props.algorithm=PBEWITHMD5AND256BITAES-CBC-OPENSSL
nifi.sensitive.props.provider=BC
nifi.sensitive.props.additional.keys=nifi.ui.banner.text, nifi.version, nifi.sensitive.props.additional.keys
nifi.security.keystore=
nifi.security.keystoreType=
nifi.security.keystorePasswd=
nifi.security.keyPasswd=
nifi.security.truststore=
nifi.security.truststoreType=
nifi.security.truststorePasswd=
nifi.security.needClientAuth=
nifi.security.user.authorizer=
# cluster common properties (cluster manager and nodes must have same values) #
nifi.cluster.protocol.heartbeat.interval=5 sec
nifi.cluster.protocol.is.secure=false
nifi.cluster.protocol.socket.timeout=30 sec
nifi.cluster.protocol.connection.handshake.timeout=45 sec
# if multicast is used, then nifi.cluster.protocol.multicast.xxx properties must be configured #
nifi.cluster.protocol.use.multicast=false
nifi.cluster.protocol.multicast.address=
nifi.cluster.protocol.multicast.port=
nifi.cluster.protocol.multicast.service.broadcast.delay=500 ms
nifi.cluster.protocol.multicast.service.locator.attempts=3
nifi.cluster.protocol.multicast.service.locator.attempts.delay=1 sec
# cluster node properties (only configure for cluster nodes) #
nifi.cluster.is.node=false
nifi.cluster.node.address=
nifi.cluster.node.protocol.port=
nifi.cluster.node.protocol.threads=2
# if multicast is not used, nifi.cluster.node.unicast.xxx must have same values as nifi.cluster.manager.xxx #
nifi.cluster.node.unicast.manager.address=
nifi.cluster.node.unicast.manager.protocol.port=
nifi.cluster.node.unicast.manager.authority.provider.port=
# cluster manager properties (only configure for cluster manager) #
nifi.cluster.is.manager=false
nifi.cluster.manager.address=
nifi.cluster.manager.protocol.port=
nifi.cluster.manager.authority.provider.port=
nifi.cluster.manager.authority.provider.threads=10
nifi.cluster.manager.node.firewall.file=
nifi.cluster.manager.node.event.history.size=10
nifi.cluster.manager.node.api.connection.timeout=30 sec
nifi.cluster.manager.node.api.read.timeout=30 sec
nifi.cluster.manager.node.api.request.threads=10
nifi.cluster.manager.flow.retrieval.delay=5 sec
nifi.cluster.manager.protocol.threads=10
nifi.cluster.manager.safemode.duration=0 sec

128
nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-properties-loader/src/test/resources/conf/nifi_with_sensitive_properties_protected_aes.properties Normal file
View File

@ -0,0 +1,128 @@
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# Core Properties #
nifi.version=nifi-test 3.0.0
nifi.flow.configuration.file=./target/flow.xml.gz
nifi.flow.configuration.archive.dir=./target/archive/
nifi.flowcontroller.autoResumeState=true
nifi.flowcontroller.graceful.shutdown.period=10 sec
nifi.flowservice.writedelay.interval=2 sec
nifi.administrative.yield.duration=30 sec
nifi.reporting.task.configuration.file=./target/reporting-tasks.xml
nifi.controller.service.configuration.file=./target/controller-services.xml
nifi.templates.directory=./target/templates
nifi.ui.banner.text=UI Banner Text
nifi.ui.autorefresh.interval=30 sec
nifi.nar.library.directory=./target/resources/NiFiProperties/lib/
nifi.nar.library.directory.alt=./target/resources/NiFiProperties/lib2/
nifi.nar.working.directory=./target/work/nar/
# H2 Settings
nifi.database.directory=./target/database_repository
nifi.h2.url.append=;LOCK_TIMEOUT=25000;WRITE_DELAY=0;AUTO_SERVER=FALSE
# FlowFile Repository
nifi.flowfile.repository.directory=./target/test-repo
nifi.flowfile.repository.partitions=1
nifi.flowfile.repository.checkpoint.interval=2 mins
nifi.queue.swap.threshold=20000
nifi.swap.storage.directory=./target/test-repo/swap
nifi.swap.in.period=5 sec
nifi.swap.in.threads=1
nifi.swap.out.period=5 sec
nifi.swap.out.threads=4
# Content Repository
nifi.content.claim.max.appendable.size=10 MB
nifi.content.claim.max.flow.files=100
nifi.content.repository.directory.default=./target/content_repository
# Provenance Repository Properties
nifi.provenance.repository.storage.directory=./target/provenance_repository
nifi.provenance.repository.max.storage.time=24 hours
nifi.provenance.repository.max.storage.size=1 GB
nifi.provenance.repository.rollover.time=30 secs
nifi.provenance.repository.rollover.size=100 MB
# Site to Site properties
nifi.remote.input.socket.port=9990
nifi.remote.input.secure=true
# web properties #
nifi.web.war.directory=./target/lib
nifi.web.http.host=
nifi.web.http.port=
nifi.web.https.host=nifi.nifi.apache.org
nifi.web.https.port=8443
nifi.web.jetty.working.directory=./target/work/jetty
# security properties #
nifi.sensitive.props.key=n2z+tTTbHuZ4V4V2||uWhdasyDXD4ZG2lMAes/vqh6u4vaz4xgL4aEbF4Y/dXevqk3ulRcOwf1vc4RDQ==
nifi.sensitive.props.key.protected=aes/gcm/256
nifi.sensitive.props.algorithm=PBEWITHMD5AND256BITAES-CBC-OPENSSL
nifi.sensitive.props.provider=BC
nifi.sensitive.props.additional.keys=nifi.ui.banner.text, nifi.version
nifi.security.keystore=/path/to/keystore.jks
nifi.security.keystoreType=JKS
nifi.security.keystorePasswd=oBjT92hIGRElIGOh||MZ6uYuWNBrOA6usq/Jt3DaD2e4otNirZDytac/w/KFe0HOkrJR03vcbo
nifi.security.keystorePasswd.protected=aes/gcm/256
nifi.security.keyPasswd=ac/BaE35SL/esLiJ||+ULRvRLYdIDA2VqpE0eQXDEMjaLBMG2kbKOdOwBk/hGebDKlVg==
nifi.security.keyPasswd.protected=aes/gcm/256
nifi.security.truststore=
nifi.security.truststoreType=
nifi.security.truststorePasswd=
nifi.security.needClientAuth=
nifi.security.user.authorizer=
# cluster common properties (cluster manager and nodes must have same values) #
nifi.cluster.protocol.heartbeat.interval=5 sec
nifi.cluster.protocol.is.secure=false
nifi.cluster.protocol.socket.timeout=30 sec
nifi.cluster.protocol.connection.handshake.timeout=45 sec
# if multicast is used, then nifi.cluster.protocol.multicast.xxx properties must be configured #
nifi.cluster.protocol.use.multicast=false
nifi.cluster.protocol.multicast.address=
nifi.cluster.protocol.multicast.port=
nifi.cluster.protocol.multicast.service.broadcast.delay=500 ms
nifi.cluster.protocol.multicast.service.locator.attempts=3
nifi.cluster.protocol.multicast.service.locator.attempts.delay=1 sec
# cluster node properties (only configure for cluster nodes) #
nifi.cluster.is.node=false
nifi.cluster.node.address=
nifi.cluster.node.protocol.port=
nifi.cluster.node.protocol.threads=2
# if multicast is not used, nifi.cluster.node.unicast.xxx must have same values as nifi.cluster.manager.xxx #
nifi.cluster.node.unicast.manager.address=
nifi.cluster.node.unicast.manager.protocol.port=
nifi.cluster.node.unicast.manager.authority.provider.port=
# cluster manager properties (only configure for cluster manager) #
nifi.cluster.is.manager=false
nifi.cluster.manager.address=
nifi.cluster.manager.protocol.port=
nifi.cluster.manager.authority.provider.port=
nifi.cluster.manager.authority.provider.threads=10
nifi.cluster.manager.node.firewall.file=
nifi.cluster.manager.node.event.history.size=10
nifi.cluster.manager.node.api.connection.timeout=30 sec
nifi.cluster.manager.node.api.read.timeout=30 sec
nifi.cluster.manager.node.api.request.threads=10
nifi.cluster.manager.flow.retrieval.delay=5 sec
nifi.cluster.manager.protocol.threads=10
nifi.cluster.manager.safemode.duration=0 sec

128
nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-properties-loader/src/test/resources/conf/nifi_with_sensitive_properties_protected_aes_multiple_malformed.properties Normal file
View File

@ -0,0 +1,128 @@
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# Core Properties #
nifi.version=nifi-test 3.0.0
nifi.flow.configuration.file=./target/flow.xml.gz
nifi.flow.configuration.archive.dir=./target/archive/
nifi.flowcontroller.autoResumeState=true
nifi.flowcontroller.graceful.shutdown.period=10 sec
nifi.flowservice.writedelay.interval=2 sec
nifi.administrative.yield.duration=30 sec
nifi.reporting.task.configuration.file=./target/reporting-tasks.xml
nifi.controller.service.configuration.file=./target/controller-services.xml
nifi.templates.directory=./target/templates
nifi.ui.banner.text=UI Banner Text
nifi.ui.autorefresh.interval=30 sec
nifi.nar.library.directory=./target/resources/NiFiProperties/lib/
nifi.nar.library.directory.alt=./target/resources/NiFiProperties/lib2/
nifi.nar.working.directory=./target/work/nar/
# H2 Settings
nifi.database.directory=./target/database_repository
nifi.h2.url.append=;LOCK_TIMEOUT=25000;WRITE_DELAY=0;AUTO_SERVER=FALSE
# FlowFile Repository
nifi.flowfile.repository.directory=./target/test-repo
nifi.flowfile.repository.partitions=1
nifi.flowfile.repository.checkpoint.interval=2 mins
nifi.queue.swap.threshold=20000
nifi.swap.storage.directory=./target/test-repo/swap
nifi.swap.in.period=5 sec
nifi.swap.in.threads=1
nifi.swap.out.period=5 sec
nifi.swap.out.threads=4
# Content Repository
nifi.content.claim.max.appendable.size=10 MB
nifi.content.claim.max.flow.files=100
nifi.content.repository.directory.default=./target/content_repository
# Provenance Repository Properties
nifi.provenance.repository.storage.directory=./target/provenance_repository
nifi.provenance.repository.max.storage.time=24 hours
nifi.provenance.repository.max.storage.size=1 GB
nifi.provenance.repository.rollover.time=30 secs
nifi.provenance.repository.rollover.size=100 MB
# Site to Site properties
nifi.remote.input.socket.port=9990
nifi.remote.input.secure=true
# web properties #
nifi.web.war.directory=./target/lib
nifi.web.http.host=
nifi.web.http.port=
nifi.web.https.host=nifi.nifi.apache.org
nifi.web.https.port=8443
nifi.web.jetty.working.directory=./target/work/jetty
# security properties #
nifi.sensitive.props.key=n2z+tTTbHuZ4V4V2||uWhdasyDXD4ZG2lMAes/vqh6u4vaz4xgL4aEbF4y/dXevqk3ulRcOwf1vc4RDQ==
nifi.sensitive.props.key.protected=aes/gcm/256
nifi.sensitive.props.algorithm=PBEWITHMD5AND256BITAES-CBC-OPENSSL
nifi.sensitive.props.provider=BC
nifi.sensitive.props.additional.keys=nifi.ui.banner.text, nifi.version
nifi.security.keystore=/path/to/keystore.jks
nifi.security.keystoreType=JKS
nifi.security.keystorePasswd=oBjT92hIGRElIGOh||MZ6uYuWNBrOA6usq/Jt3DaD2e4otNirZDytaC/w/KFe0HOkrJR03vcbo
nifi.security.keystorePasswd.protected=aes/gcm/256
nifi.security.keyPasswd=ac/BaE35SL/esLiJ||+ULRvRLYdIDA2VqpE0eQXDEMjaLBMG2kbKOdOwBK/hGebDKlVg==
nifi.security.keyPasswd.protected=aes/gcm/256
nifi.security.truststore=
nifi.security.truststoreType=
nifi.security.truststorePasswd=
nifi.security.needClientAuth=
nifi.security.user.authorizer=
# cluster common properties (cluster manager and nodes must have same values) #
nifi.cluster.protocol.heartbeat.interval=5 sec
nifi.cluster.protocol.is.secure=false
nifi.cluster.protocol.socket.timeout=30 sec
nifi.cluster.protocol.connection.handshake.timeout=45 sec
# if multicast is used, then nifi.cluster.protocol.multicast.xxx properties must be configured #
nifi.cluster.protocol.use.multicast=false
nifi.cluster.protocol.multicast.address=
nifi.cluster.protocol.multicast.port=
nifi.cluster.protocol.multicast.service.broadcast.delay=500 ms
nifi.cluster.protocol.multicast.service.locator.attempts=3
nifi.cluster.protocol.multicast.service.locator.attempts.delay=1 sec
# cluster node properties (only configure for cluster nodes) #
nifi.cluster.is.node=false
nifi.cluster.node.address=
nifi.cluster.node.protocol.port=
nifi.cluster.node.protocol.threads=2
# if multicast is not used, nifi.cluster.node.unicast.xxx must have same values as nifi.cluster.manager.xxx #
nifi.cluster.node.unicast.manager.address=
nifi.cluster.node.unicast.manager.protocol.port=
nifi.cluster.node.unicast.manager.authority.provider.port=
# cluster manager properties (only configure for cluster manager) #
nifi.cluster.is.manager=false
nifi.cluster.manager.address=
nifi.cluster.manager.protocol.port=
nifi.cluster.manager.authority.provider.port=
nifi.cluster.manager.authority.provider.threads=10
nifi.cluster.manager.node.firewall.file=
nifi.cluster.manager.node.event.history.size=10
nifi.cluster.manager.node.api.connection.timeout=30 sec
nifi.cluster.manager.node.api.read.timeout=30 sec
nifi.cluster.manager.node.api.request.threads=10
nifi.cluster.manager.flow.retrieval.delay=5 sec
nifi.cluster.manager.protocol.threads=10
nifi.cluster.manager.safemode.duration=0 sec

128
nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-properties-loader/src/test/resources/conf/nifi_with_sensitive_properties_protected_aes_single_malformed.properties Normal file
View File

@ -0,0 +1,128 @@
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# Core Properties #
nifi.version=nifi-test 3.0.0
nifi.flow.configuration.file=./target/flow.xml.gz
nifi.flow.configuration.archive.dir=./target/archive/
nifi.flowcontroller.autoResumeState=true
nifi.flowcontroller.graceful.shutdown.period=10 sec
nifi.flowservice.writedelay.interval=2 sec
nifi.administrative.yield.duration=30 sec
nifi.reporting.task.configuration.file=./target/reporting-tasks.xml
nifi.controller.service.configuration.file=./target/controller-services.xml
nifi.templates.directory=./target/templates
nifi.ui.banner.text=UI Banner Text
nifi.ui.autorefresh.interval=30 sec
nifi.nar.library.directory=./target/resources/NiFiProperties/lib/
nifi.nar.library.directory.alt=./target/resources/NiFiProperties/lib2/
nifi.nar.working.directory=./target/work/nar/
# H2 Settings
nifi.database.directory=./target/database_repository
nifi.h2.url.append=;LOCK_TIMEOUT=25000;WRITE_DELAY=0;AUTO_SERVER=FALSE
# FlowFile Repository
nifi.flowfile.repository.directory=./target/test-repo
nifi.flowfile.repository.partitions=1
nifi.flowfile.repository.checkpoint.interval=2 mins
nifi.queue.swap.threshold=20000
nifi.swap.storage.directory=./target/test-repo/swap
nifi.swap.in.period=5 sec
nifi.swap.in.threads=1
nifi.swap.out.period=5 sec
nifi.swap.out.threads=4
# Content Repository
nifi.content.claim.max.appendable.size=10 MB
nifi.content.claim.max.flow.files=100
nifi.content.repository.directory.default=./target/content_repository
# Provenance Repository Properties
nifi.provenance.repository.storage.directory=./target/provenance_repository
nifi.provenance.repository.max.storage.time=24 hours
nifi.provenance.repository.max.storage.size=1 GB
nifi.provenance.repository.rollover.time=30 secs
nifi.provenance.repository.rollover.size=100 MB
# Site to Site properties
nifi.remote.input.socket.port=9990
nifi.remote.input.secure=true
# web properties #
nifi.web.war.directory=./target/lib
nifi.web.http.host=
nifi.web.http.port=
nifi.web.https.host=nifi.nifi.apache.org
nifi.web.https.port=8443
nifi.web.jetty.working.directory=./target/work/jetty
# security properties #
nifi.sensitive.props.key=n2z+tTTbHuZ4V4V2||uWhdasyDXD4ZG2lMAes/vqh6u4vaz4xgL4aEbF4Y/dXevqk3ulRcOwf1vc4RDQ==
nifi.sensitive.props.key.protected=aes/gcm/256
nifi.sensitive.props.algorithm=PBEWITHMD5AND256BITAES-CBC-OPENSSL
nifi.sensitive.props.provider=BC
nifi.sensitive.props.additional.keys=nifi.ui.banner.text, nifi.version
nifi.security.keystore=/path/to/keystore.jks
nifi.security.keystoreType=JKS
nifi.security.keystorePasswd=oBjT92hIGRElIGOh||MZ6uYuWNBrOA6usq/Jt3DaD2e4otNirZDytac/w/KFe0HOkrJR03
nifi.security.keystorePasswd.protected=aes/gcm/256
nifi.security.keyPasswd=ac/BaE35SL/esLiJ||+ULRvRLYdIDA2VqpE0eQXDEMjaLBMG2kbKOdOwBk/hGebDKlVg==
nifi.security.keyPasswd.protected=aes/gcm/256
nifi.security.truststore=
nifi.security.truststoreType=
nifi.security.truststorePasswd=
nifi.security.needClientAuth=
nifi.security.user.authorizer=
# cluster common properties (cluster manager and nodes must have same values) #
nifi.cluster.protocol.heartbeat.interval=5 sec
nifi.cluster.protocol.is.secure=false
nifi.cluster.protocol.socket.timeout=30 sec
nifi.cluster.protocol.connection.handshake.timeout=45 sec
# if multicast is used, then nifi.cluster.protocol.multicast.xxx properties must be configured #
nifi.cluster.protocol.use.multicast=false
nifi.cluster.protocol.multicast.address=
nifi.cluster.protocol.multicast.port=
nifi.cluster.protocol.multicast.service.broadcast.delay=500 ms
nifi.cluster.protocol.multicast.service.locator.attempts=3
nifi.cluster.protocol.multicast.service.locator.attempts.delay=1 sec
# cluster node properties (only configure for cluster nodes) #
nifi.cluster.is.node=false
nifi.cluster.node.address=
nifi.cluster.node.protocol.port=
nifi.cluster.node.protocol.threads=2
# if multicast is not used, nifi.cluster.node.unicast.xxx must have same values as nifi.cluster.manager.xxx #
nifi.cluster.node.unicast.manager.address=
nifi.cluster.node.unicast.manager.protocol.port=
nifi.cluster.node.unicast.manager.authority.provider.port=
# cluster manager properties (only configure for cluster manager) #
nifi.cluster.is.manager=false
nifi.cluster.manager.address=
nifi.cluster.manager.protocol.port=
nifi.cluster.manager.authority.provider.port=
nifi.cluster.manager.authority.provider.threads=10
nifi.cluster.manager.node.firewall.file=
nifi.cluster.manager.node.event.history.size=10
nifi.cluster.manager.node.api.connection.timeout=30 sec
nifi.cluster.manager.node.api.read.timeout=30 sec
nifi.cluster.manager.node.api.request.threads=10
nifi.cluster.manager.flow.retrieval.delay=5 sec
nifi.cluster.manager.protocol.threads=10
nifi.cluster.manager.safemode.duration=0 sec

128
nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-properties-loader/src/test/resources/conf/nifi_with_sensitive_properties_protected_unknown.properties Normal file
View File

@ -0,0 +1,128 @@
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# Core Properties #
nifi.version=nifi-test 3.0.0
nifi.flow.configuration.file=./target/flow.xml.gz
nifi.flow.configuration.archive.dir=./target/archive/
nifi.flowcontroller.autoResumeState=true
nifi.flowcontroller.graceful.shutdown.period=10 sec
nifi.flowservice.writedelay.interval=2 sec
nifi.administrative.yield.duration=30 sec
nifi.reporting.task.configuration.file=./target/reporting-tasks.xml
nifi.controller.service.configuration.file=./target/controller-services.xml
nifi.templates.directory=./target/templates
nifi.ui.banner.text=UI Banner Text
nifi.ui.autorefresh.interval=30 sec
nifi.nar.library.directory=./target/resources/NiFiProperties/lib/
nifi.nar.library.directory.alt=./target/resources/NiFiProperties/lib2/
nifi.nar.working.directory=./target/work/nar/
# H2 Settings
nifi.database.directory=./target/database_repository
nifi.h2.url.append=;LOCK_TIMEOUT=25000;WRITE_DELAY=0;AUTO_SERVER=FALSE
# FlowFile Repository
nifi.flowfile.repository.directory=./target/test-repo
nifi.flowfile.repository.partitions=1
nifi.flowfile.repository.checkpoint.interval=2 mins
nifi.queue.swap.threshold=20000
nifi.swap.storage.directory=./target/test-repo/swap
nifi.swap.in.period=5 sec
nifi.swap.in.threads=1
nifi.swap.out.period=5 sec
nifi.swap.out.threads=4
# Content Repository
nifi.content.claim.max.appendable.size=10 MB
nifi.content.claim.max.flow.files=100
nifi.content.repository.directory.default=./target/content_repository
# Provenance Repository Properties
nifi.provenance.repository.storage.directory=./target/provenance_repository
nifi.provenance.repository.max.storage.time=24 hours
nifi.provenance.repository.max.storage.size=1 GB
nifi.provenance.repository.rollover.time=30 secs
nifi.provenance.repository.rollover.size=100 MB
# Site to Site properties
nifi.remote.input.socket.port=9990
nifi.remote.input.secure=true
# web properties #
nifi.web.war.directory=./target/lib
nifi.web.http.host=
nifi.web.http.port=
nifi.web.https.host=nifi.nifi.apache.org
nifi.web.https.port=8443
nifi.web.jetty.working.directory=./target/work/jetty
# security properties #
nifi.sensitive.props.key=n2z+tTTbHuZ4V4V2||uWhdasyDXD4ZG2lMAes/vqh6u4vaz4xgL4aEbF4Y/dXevqk3ulRcOwf1vc4RDQ==
nifi.sensitive.props.key.protected=unknown
nifi.sensitive.props.algorithm=PBEWITHMD5AND256BITAES-CBC-OPENSSL
nifi.sensitive.props.provider=BC
nifi.sensitive.props.additional.keys=nifi.ui.banner.text, nifi.version
nifi.security.keystore=/path/to/keystore.jks
nifi.security.keystoreType=JKS
nifi.security.keystorePasswd=oBjT92hIGRElIGOh||MZ6uYuWNBrOA6usq/Jt3DaD2e4otNirZDytac/w/KFe0HOkrJR03vcbo
nifi.security.keystorePasswd.protected=unknown
nifi.security.keyPasswd=ac/BaE35SL/esLiJ||+ULRvRLYdIDA2VqpE0eQXDEMjaLBMG2kbKOdOwBk/hGebDKlVg==
nifi.security.keyPasswd.protected=unknown
nifi.security.truststore=
nifi.security.truststoreType=
nifi.security.truststorePasswd=
nifi.security.needClientAuth=
nifi.security.user.authorizer=
# cluster common properties (cluster manager and nodes must have same values) #
nifi.cluster.protocol.heartbeat.interval=5 sec
nifi.cluster.protocol.is.secure=false
nifi.cluster.protocol.socket.timeout=30 sec
nifi.cluster.protocol.connection.handshake.timeout=45 sec
# if multicast is used, then nifi.cluster.protocol.multicast.xxx properties must be configured #
nifi.cluster.protocol.use.multicast=false
nifi.cluster.protocol.multicast.address=
nifi.cluster.protocol.multicast.port=
nifi.cluster.protocol.multicast.service.broadcast.delay=500 ms
nifi.cluster.protocol.multicast.service.locator.attempts=3
nifi.cluster.protocol.multicast.service.locator.attempts.delay=1 sec
# cluster node properties (only configure for cluster nodes) #
nifi.cluster.is.node=false
nifi.cluster.node.address=
nifi.cluster.node.protocol.port=
nifi.cluster.node.protocol.threads=2
# if multicast is not used, nifi.cluster.node.unicast.xxx must have same values as nifi.cluster.manager.xxx #
nifi.cluster.node.unicast.manager.address=
nifi.cluster.node.unicast.manager.protocol.port=
nifi.cluster.node.unicast.manager.authority.provider.port=
# cluster manager properties (only configure for cluster manager) #
nifi.cluster.is.manager=false
nifi.cluster.manager.address=
nifi.cluster.manager.protocol.port=
nifi.cluster.manager.authority.provider.port=
nifi.cluster.manager.authority.provider.threads=10
nifi.cluster.manager.node.firewall.file=
nifi.cluster.manager.node.event.history.size=10
nifi.cluster.manager.node.api.connection.timeout=30 sec
nifi.cluster.manager.node.api.read.timeout=30 sec
nifi.cluster.manager.node.api.request.threads=10
nifi.cluster.manager.flow.retrieval.delay=5 sec
nifi.cluster.manager.protocol.threads=10
nifi.cluster.manager.safemode.duration=0 sec

125
nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-properties-loader/src/test/resources/conf/nifi_with_sensitive_properties_unprotected.properties Normal file
View File

@ -0,0 +1,125 @@
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# Core Properties #
nifi.version=nifi-test 3.0.0
nifi.flow.configuration.file=./target/flow.xml.gz
nifi.flow.configuration.archive.dir=./target/archive/
nifi.flowcontroller.autoResumeState=true
nifi.flowcontroller.graceful.shutdown.period=10 sec
nifi.flowservice.writedelay.interval=2 sec
nifi.administrative.yield.duration=30 sec
nifi.reporting.task.configuration.file=./target/reporting-tasks.xml
nifi.controller.service.configuration.file=./target/controller-services.xml
nifi.templates.directory=./target/templates
nifi.ui.banner.text=UI Banner Text
nifi.ui.autorefresh.interval=30 sec
nifi.nar.library.directory=./target/resources/NiFiProperties/lib/
nifi.nar.library.directory.alt=./target/resources/NiFiProperties/lib2/
nifi.nar.working.directory=./target/work/nar/
# H2 Settings
nifi.database.directory=./target/database_repository
nifi.h2.url.append=;LOCK_TIMEOUT=25000;WRITE_DELAY=0;AUTO_SERVER=FALSE
# FlowFile Repository
nifi.flowfile.repository.directory=./target/test-repo
nifi.flowfile.repository.partitions=1
nifi.flowfile.repository.checkpoint.interval=2 mins
nifi.queue.swap.threshold=20000
nifi.swap.storage.directory=./target/test-repo/swap
nifi.swap.in.period=5 sec
nifi.swap.in.threads=1
nifi.swap.out.period=5 sec
nifi.swap.out.threads=4
# Content Repository
nifi.content.claim.max.appendable.size=10 MB
nifi.content.claim.max.flow.files=100
nifi.content.repository.directory.default=./target/content_repository
# Provenance Repository Properties
nifi.provenance.repository.storage.directory=./target/provenance_repository
nifi.provenance.repository.max.storage.time=24 hours
nifi.provenance.repository.max.storage.size=1 GB
nifi.provenance.repository.rollover.time=30 secs
nifi.provenance.repository.rollover.size=100 MB
# Site to Site properties
nifi.remote.input.socket.port=9990
nifi.remote.input.secure=true
# web properties #
nifi.web.war.directory=./target/lib
nifi.web.http.host=
nifi.web.http.port=
nifi.web.https.host=nifi.nifi.apache.org
nifi.web.https.port=8443
nifi.web.jetty.working.directory=./target/work/jetty
# security properties #
nifi.sensitive.props.key=thisIsABadSensitiveKeyPassword
nifi.sensitive.props.algorithm=PBEWITHMD5AND256BITAES-CBC-OPENSSL
nifi.sensitive.props.provider=BC
nifi.sensitive.props.additional.keys=nifi.ui.banner.text, nifi.version
nifi.security.keystore=/path/to/keystore.jks
nifi.security.keystoreType=JKS
nifi.security.keystorePasswd=thisIsABadKeystorePassword
nifi.security.keyPasswd=thisIsABadKeyPassword
nifi.security.truststore=
nifi.security.truststoreType=
nifi.security.truststorePasswd=
nifi.security.needClientAuth=
nifi.security.user.authorizer=
# cluster common properties (cluster manager and nodes must have same values) #
nifi.cluster.protocol.heartbeat.interval=5 sec
nifi.cluster.protocol.is.secure=false
nifi.cluster.protocol.socket.timeout=30 sec
nifi.cluster.protocol.connection.handshake.timeout=45 sec
# if multicast is used, then nifi.cluster.protocol.multicast.xxx properties must be configured #
nifi.cluster.protocol.use.multicast=false
nifi.cluster.protocol.multicast.address=
nifi.cluster.protocol.multicast.port=
nifi.cluster.protocol.multicast.service.broadcast.delay=500 ms
nifi.cluster.protocol.multicast.service.locator.attempts=3
nifi.cluster.protocol.multicast.service.locator.attempts.delay=1 sec
# cluster node properties (only configure for cluster nodes) #
nifi.cluster.is.node=false
nifi.cluster.node.address=
nifi.cluster.node.protocol.port=
nifi.cluster.node.protocol.threads=2
# if multicast is not used, nifi.cluster.node.unicast.xxx must have same values as nifi.cluster.manager.xxx #
nifi.cluster.node.unicast.manager.address=
nifi.cluster.node.unicast.manager.protocol.port=
nifi.cluster.node.unicast.manager.authority.provider.port=
# cluster manager properties (only configure for cluster manager) #
nifi.cluster.is.manager=false
nifi.cluster.manager.address=
nifi.cluster.manager.protocol.port=
nifi.cluster.manager.authority.provider.port=
nifi.cluster.manager.authority.provider.threads=10
nifi.cluster.manager.node.firewall.file=
nifi.cluster.manager.node.event.history.size=10
nifi.cluster.manager.node.api.connection.timeout=30 sec
nifi.cluster.manager.node.api.read.timeout=30 sec
nifi.cluster.manager.node.api.request.threads=10
nifi.cluster.manager.flow.retrieval.delay=5 sec
nifi.cluster.manager.protocol.threads=10
nifi.cluster.manager.safemode.duration=0 sec

126
nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-properties-loader/src/test/resources/conf/nifi_with_sensitive_properties_unprotected_extra_line.properties Normal file
View File

@ -0,0 +1,126 @@
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# Core Properties #
nifi.version=nifi-test 3.0.0
nifi.flow.configuration.file=./target/flow.xml.gz
nifi.flow.configuration.archive.dir=./target/archive/
nifi.flowcontroller.autoResumeState=true
nifi.flowcontroller.graceful.shutdown.period=10 sec
nifi.flowservice.writedelay.interval=2 sec
nifi.administrative.yield.duration=30 sec
nifi.reporting.task.configuration.file=./target/reporting-tasks.xml
nifi.controller.service.configuration.file=./target/controller-services.xml
nifi.templates.directory=./target/templates
nifi.ui.banner.text=UI Banner Text
nifi.ui.autorefresh.interval=30 sec
nifi.nar.library.directory=./target/resources/NiFiProperties/lib/
nifi.nar.library.directory.alt=./target/resources/NiFiProperties/lib2/
nifi.nar.working.directory=./target/work/nar/
# H2 Settings
nifi.database.directory=./target/database_repository
nifi.h2.url.append=;LOCK_TIMEOUT=25000;WRITE_DELAY=0;AUTO_SERVER=FALSE
# FlowFile Repository
nifi.flowfile.repository.directory=./target/test-repo
nifi.flowfile.repository.partitions=1
nifi.flowfile.repository.checkpoint.interval=2 mins
nifi.queue.swap.threshold=20000
nifi.swap.storage.directory=./target/test-repo/swap
nifi.swap.in.period=5 sec
nifi.swap.in.threads=1
nifi.swap.out.period=5 sec
nifi.swap.out.threads=4
# Content Repository
nifi.content.claim.max.appendable.size=10 MB
nifi.content.claim.max.flow.files=100
nifi.content.repository.directory.default=./target/content_repository
# Provenance Repository Properties
nifi.provenance.repository.storage.directory=./target/provenance_repository
nifi.provenance.repository.max.storage.time=24 hours
nifi.provenance.repository.max.storage.size=1 GB
nifi.provenance.repository.rollover.time=30 secs
nifi.provenance.repository.rollover.size=100 MB
# Site to Site properties
nifi.remote.input.socket.port=9990
nifi.remote.input.secure=true
# web properties #
nifi.web.war.directory=./target/lib
nifi.web.http.host=
nifi.web.http.port=
nifi.web.https.host=nifi.nifi.apache.org
nifi.web.https.port=8443
nifi.web.jetty.working.directory=./target/work/jetty
# security properties #
nifi.sensitive.props.key=thisIsABadSensitiveKeyPassword
nifi.sensitive.props.key.protected=
nifi.sensitive.props.algorithm=PBEWITHMD5AND256BITAES-CBC-OPENSSL
nifi.sensitive.props.provider=BC
nifi.sensitive.props.additional.keys=nifi.ui.banner.text, nifi.version
nifi.security.keystore=/path/to/keystore.jks
nifi.security.keystoreType=JKS
nifi.security.keystorePasswd=thisIsABadKeystorePassword
nifi.security.keyPasswd=thisIsABadKeyPassword
nifi.security.truststore=
nifi.security.truststoreType=
nifi.security.truststorePasswd=
nifi.security.needClientAuth=
nifi.security.user.authorizer=
# cluster common properties (cluster manager and nodes must have same values) #
nifi.cluster.protocol.heartbeat.interval=5 sec
nifi.cluster.protocol.is.secure=false
nifi.cluster.protocol.socket.timeout=30 sec
nifi.cluster.protocol.connection.handshake.timeout=45 sec
# if multicast is used, then nifi.cluster.protocol.multicast.xxx properties must be configured #
nifi.cluster.protocol.use.multicast=false
nifi.cluster.protocol.multicast.address=
nifi.cluster.protocol.multicast.port=
nifi.cluster.protocol.multicast.service.broadcast.delay=500 ms
nifi.cluster.protocol.multicast.service.locator.attempts=3
nifi.cluster.protocol.multicast.service.locator.attempts.delay=1 sec
# cluster node properties (only configure for cluster nodes) #
nifi.cluster.is.node=false
nifi.cluster.node.address=
nifi.cluster.node.protocol.port=
nifi.cluster.node.protocol.threads=2
# if multicast is not used, nifi.cluster.node.unicast.xxx must have same values as nifi.cluster.manager.xxx #
nifi.cluster.node.unicast.manager.address=
nifi.cluster.node.unicast.manager.protocol.port=
nifi.cluster.node.unicast.manager.authority.provider.port=
# cluster manager properties (only configure for cluster manager) #
nifi.cluster.is.manager=false
nifi.cluster.manager.address=
nifi.cluster.manager.protocol.port=
nifi.cluster.manager.authority.provider.port=
nifi.cluster.manager.authority.provider.threads=10
nifi.cluster.manager.node.firewall.file=
nifi.cluster.manager.node.event.history.size=10
nifi.cluster.manager.node.api.connection.timeout=30 sec
nifi.cluster.manager.node.api.read.timeout=30 sec
nifi.cluster.manager.node.api.request.threads=10
nifi.cluster.manager.flow.retrieval.delay=5 sec
nifi.cluster.manager.protocol.threads=10
nifi.cluster.manager.safemode.duration=0 sec

3
nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-resources/pom.xml
View File

@ -81,8 +81,11 @@
<nifi.nar.working.directory>./work/nar/</nifi.nar.working.directory>
<nifi.documentation.working.directory>./work/docs/components</nifi.documentation.working.directory>
<nifi.sensitive.props.key.protected />
<nifi.sensitive.props.algorithm>PBEWITHMD5AND256BITAES-CBC-OPENSSL</nifi.sensitive.props.algorithm>
<nifi.sensitive.props.provider>BC</nifi.sensitive.props.provider>
<nifi.sensitive.props.additional.keys />
<nifi.h2.url.append>;LOCK_TIMEOUT=25000;WRITE_DELAY=0;AUTO_SERVER=FALSE</nifi.h2.url.append>
<nifi.remote.input.socket.port>9990</nifi.remote.input.socket.port>

2
nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-resources/src/main/resources/conf/bootstrap.conf
View File

@ -51,6 +51,8 @@ java.arg.13=-XX:+UseG1GC
#Set headless mode by default
java.arg.14=-Djava.awt.headless=true
#Set sensitive properties key
nifi.bootstrap.sensitive.key=
###
# Notification Services for notifying interested parties when NiFi is stopped, started, dies

2
nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-resources/src/main/resources/conf/nifi.properties
View File

@ -131,8 +131,10 @@ nifi.web.jetty.threads=${nifi.web.jetty.threads}
# security properties #
nifi.sensitive.props.key=
nifi.sensitive.props.key.protected=${nifi.sensitive.props.key.protected}
nifi.sensitive.props.algorithm=${nifi.sensitive.props.algorithm}
nifi.sensitive.props.provider=${nifi.sensitive.props.provider}
nifi.sensitive.props.additional.keys=${nifi.sensitive.props.additional.keys}
nifi.security.keystore=${nifi.security.keystore}
nifi.security.keystoreType=${nifi.security.keystoreType}

10
nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-runtime/pom.xml
View File

@ -32,6 +32,11 @@
<artifactId>nifi-properties</artifactId>
<scope>compile</scope>
</dependency>
<dependency>
<groupId>org.apache.nifi</groupId>
<artifactId>nifi-properties-loader</artifactId>
<scope>compile</scope>
</dependency>
<dependency>
<groupId>org.apache.nifi</groupId>
<artifactId>nifi-documentation</artifactId>
@ -41,5 +46,10 @@
<groupId>org.slf4j</groupId>
<artifactId>jul-to-slf4j</artifactId>
</dependency>
<dependency>
<groupId>ch.qos.logback</groupId>
<artifactId>logback-classic</artifactId>
<scope>test</scope>
</dependency>
</dependencies>
</project>

104
nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-runtime/src/main/java/org/apache/nifi/NiFi.java
View File

@ -21,6 +21,9 @@ import java.io.IOException;
import java.lang.Thread.UncaughtExceptionHandler;
import java.lang.reflect.Constructor;
import java.lang.reflect.InvocationTargetException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import java.util.Timer;
import java.util.TimerTask;
import java.util.concurrent.Executors;
@ -30,12 +33,13 @@ import java.util.concurrent.ThreadFactory;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.atomic.AtomicLong;
import org.apache.nifi.documentation.DocGenerator;
import org.apache.nifi.nar.ExtensionManager;
import org.apache.nifi.nar.ExtensionMapping;
import org.apache.nifi.nar.NarClassLoaders;
import org.apache.nifi.nar.NarUnpacker;
import org.apache.nifi.properties.NiFiPropertiesLoader;
import org.apache.nifi.properties.SensitivePropertyProtectionException;
import org.apache.nifi.util.FileUtils;
import org.apache.nifi.util.NiFiProperties;
import org.slf4j.Logger;
@ -45,6 +49,7 @@ import org.slf4j.bridge.SLF4JBridgeHandler;
public class NiFi {
private static final Logger logger = LoggerFactory.getLogger(NiFi.class);
private static final String KEY_FLAG = "-k";
private final NiFiServer nifiServer;
private final BootstrapListener bootstrapListener;
@ -158,7 +163,7 @@ public class NiFi {
}
logger.info("Jetty web server shutdown completed (nicely or otherwise).");
} catch (final Throwable t) {
logger.warn("Problem occured ensuring Jetty web server was properly terminated due to " + t);
logger.warn("Problem occurred ensuring Jetty web server was properly terminated due to " + t);
}
}
@ -183,14 +188,14 @@ public class NiFi {
});
final AtomicInteger occurrencesOutOfRange = new AtomicInteger(0);
final AtomicInteger occurences = new AtomicInteger(0);
final AtomicInteger occurrences = new AtomicInteger(0);
final Runnable command = new Runnable() {
@Override
public void run() {
final long curMillis = System.currentTimeMillis();
final long difference = curMillis - lastTriggerMillis.get();
final long millisOff = Math.abs(difference - 2000L);
occurences.incrementAndGet();
occurrences.incrementAndGet();
if (millisOff > 500L) {
occurrencesOutOfRange.incrementAndGet();
}
@ -206,7 +211,7 @@ public class NiFi {
future.cancel(true);
service.shutdownNow();
if (occurences.get() < minRequiredOccurrences || occurrencesOutOfRange.get() > maxOccurrencesOutOfRange) {
if (occurrences.get() < minRequiredOccurrences || occurrencesOutOfRange.get() > maxOccurrencesOutOfRange) {
logger.warn("NiFi has detected that this box is not responding within the expected timing interval, which may cause "
+ "Processors to be scheduled erratically. Please see the NiFi documentation for more information.");
}
@ -224,9 +229,96 @@ public class NiFi {
public static void main(String[] args) {
logger.info("Launching NiFi...");
try {
new NiFi(NiFiProperties.createBasicNiFiProperties(null, null));
NiFiProperties properties = initializeProperties(args);
new NiFi(properties);
} catch (final Throwable t) {
logger.error("Failure to launch NiFi due to " + t, t);
}
}
private static NiFiProperties initializeProperties(String[] args) {
// Try to get key
// If key doesn't exist, instantiate without
// Load properties
// If properties are protected and key missing, throw RuntimeException
try {
String key = loadFormattedKey(args);
// The key might be empty or null when it is passed to the loader
try {
NiFiProperties properties = NiFiPropertiesLoader.withKey(key).get();
logger.info("Loaded {} properties", properties.size());
return properties;
} catch (SensitivePropertyProtectionException e) {
final String msg = "There was an issue decrypting protected properties";
logger.error(msg, e);
throw new IllegalArgumentException(msg);
}
} catch (IllegalArgumentException e) {
final String msg = "The bootstrap process did not provide a valid key and there are protected properties present in the properties file";
logger.error(msg, e);
throw new IllegalArgumentException(msg);
}
}
private static String loadFormattedKey(String[] args) {
String key = null;
List<String> parsedArgs = parseArgs(args);
// Check if args contain protection key
if (parsedArgs.contains(KEY_FLAG)) {
key = getKeyFromArgs(parsedArgs);
// Format the key (check hex validity and remove spaces)
key = formatHexKey(key);
if (!isHexKeyValid(key)) {
throw new IllegalArgumentException("The key was not provided in valid hex format and of the correct length");
}
return key;
} else {
// throw new IllegalStateException("No key provided from bootstrap");
return "";
}
}
private static String getKeyFromArgs(List<String> parsedArgs) {
String key;
logger.debug("The bootstrap process provided the " + KEY_FLAG + " flag");
int i = parsedArgs.indexOf(KEY_FLAG);
if (parsedArgs.size() <= i + 1) {
logger.error("The bootstrap process passed the {} flag without a key", KEY_FLAG);
throw new IllegalArgumentException("The bootstrap process provided the " + KEY_FLAG + " flag but no key");
}
key = parsedArgs.get(i + 1);
logger.info("Read property protection key from bootstrap process");
return key;
}
private static List<String> parseArgs(String[] args) {
List<String> parsedArgs = new ArrayList<>(Arrays.asList(args));
for (int i = 0; i < parsedArgs.size(); i++) {
if (parsedArgs.get(i).startsWith(KEY_FLAG + " ")) {
String[] split = parsedArgs.get(i).split(" ", 2);
parsedArgs.set(i, split[0]);
parsedArgs.add(i + 1, split[1]);
break;
}
}
return parsedArgs;
}
private static String formatHexKey(String input) {
if (input == null || input.trim().isEmpty()) {
return "";
}
return input.replaceAll("[^0-9a-fA-F]", "").toLowerCase();
}
private static boolean isHexKeyValid(String key) {
if (key == null || key.trim().isEmpty()) {
return false;
}
// Key length is in "nibbles" (i.e. one hex char = 4 bits)
return Arrays.asList(128, 196, 256).contains(key.length() * 4) && key.matches("^[0-9a-fA-F]*$");
}
}

277
nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-runtime/src/test/groovy/org/apache/nifi/NiFiGroovyTest.groovy Normal file
View File

@ -0,0 +1,277 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.nifi
import ch.qos.logback.classic.spi.LoggingEvent
import ch.qos.logback.core.AppenderBase
import org.apache.nifi.properties.AESSensitivePropertyProvider
import org.apache.nifi.properties.NiFiPropertiesLoader
import org.apache.nifi.properties.StandardNiFiProperties
import org.apache.nifi.util.NiFiProperties
import org.bouncycastle.jce.provider.BouncyCastleProvider
import org.junit.After
import org.junit.AfterClass
import org.junit.Before
import org.junit.BeforeClass
import org.junit.Test
import org.junit.runner.RunWith
import org.junit.runners.JUnit4
import org.slf4j.Logger
import org.slf4j.LoggerFactory
import org.slf4j.bridge.SLF4JBridgeHandler
import java.security.Security
@RunWith(JUnit4.class)
class NiFiGroovyTest extends GroovyTestCase {
private static final Logger logger = LoggerFactory.getLogger(NiFiGroovyTest.class)
private static String originalPropertiesPath = System.getProperty(NiFiProperties.PROPERTIES_FILE_PATH)
private static final String TEST_RES_PATH = NiFiGroovyTest.getClassLoader().getResource(".").toURI().getPath()
private static final File workDir = new File("./target/work/jetty/")
@BeforeClass
public static void setUpOnce() throws Exception {
Security.addProvider(new BouncyCastleProvider())
SLF4JBridgeHandler.install()
logger.metaClass.methodMissing = { String name, args ->
logger.info("[${name?.toUpperCase()}] ${(args as List).join(" ")}")
}
logger.info("Identified test resources path as ${TEST_RES_PATH}")
}
@Before
public void setUp() throws Exception {
if (!workDir.exists()) {
workDir.mkdirs()
}
}
@After
public void tearDown() throws Exception {
NiFiPropertiesLoader.@sensitivePropertyProviderFactory = null
TestAppender.reset()
System.setIn(System.in)
}
@AfterClass
public static void tearDownOnce() {
if (originalPropertiesPath) {
System.setProperty(NiFiProperties.PROPERTIES_FILE_PATH, originalPropertiesPath)
}
}
@Test
public void testInitializePropertiesShouldHandleNoBootstrapKey() throws Exception {
// Arrange
def args = [] as String[]
String plainPropertiesPath = "${TEST_RES_PATH}/NiFiProperties/conf/nifi.properties"
System.setProperty(NiFiProperties.PROPERTIES_FILE_PATH, plainPropertiesPath)
// Act
NiFiProperties loadedProperties = NiFi.initializeProperties(args)
// Assert
assert loadedProperties.size() > 0
}
@Test
public void testMainShouldHandleNoBootstrapKeyWithProtectedProperties() throws Exception {
// Arrange
def args = [] as String[]
System.setProperty(NiFiProperties.PROPERTIES_FILE_PATH, "${TEST_RES_PATH}/NiFiProperties/conf/nifi_with_sensitive_properties_protected_aes_different_key.properties")
// Act
NiFi.main(args)
// Assert
assert TestAppender.events.last().getMessage() == "Failure to launch NiFi due to java.lang.IllegalArgumentException: The bootstrap process did not provide a valid key and there are protected properties present in the properties file"
}
@Test
public void testParseArgsShouldSplitCombinedArgs() throws Exception {
// Arrange
final String DIFFERENT_KEY = "0" * 64
def args = ["-k ${DIFFERENT_KEY}"] as String[]
// Act
def parsedArgs = NiFi.parseArgs(args)
// Assert
assert parsedArgs.size() == 2
assert parsedArgs == args.join(" ").split(" ") as List
}
@Test
public void testMainShouldHandleBadArgs() throws Exception {
// Arrange
def args = ["-k"] as String[]
System.setProperty(NiFiProperties.PROPERTIES_FILE_PATH, "${TEST_RES_PATH}/NiFiProperties/conf/nifi_with_sensitive_properties_protected_aes.properties")
// Act
NiFi.main(args)
// Assert
assert TestAppender.events.collect {
it.getFormattedMessage()
}.contains("The bootstrap process passed the -k flag without a key")
assert TestAppender.events.last().getMessage() == "Failure to launch NiFi due to java.lang.IllegalArgumentException: The bootstrap process did not provide a valid key and there are protected properties present in the properties file"
}
@Test
public void testMainShouldHandleMalformedBootstrapKey() throws Exception {
// Arrange
def args = ["-k", "BAD KEY"] as String[]
System.setProperty(NiFiProperties.PROPERTIES_FILE_PATH, "${TEST_RES_PATH}/NiFiProperties/conf/nifi_with_sensitive_properties_protected_aes.properties")
// Act
NiFi.main(args)
// Assert
assert TestAppender.events.last().getMessage() == "Failure to launch NiFi due to java.lang.IllegalArgumentException: The bootstrap process did not provide a valid key and there are protected properties present in the properties file"
}
@Test
public void testInitializePropertiesShouldSetBootstrapKeyFromArgs() throws Exception {
// Arrange
final String DIFFERENT_KEY = "0" * 64
def args = ["-k", DIFFERENT_KEY] as String[]
String testPropertiesPath = "${TEST_RES_PATH}/NiFiProperties/conf/nifi_with_sensitive_properties_protected_aes_different_key.properties"
System.setProperty(NiFiProperties.PROPERTIES_FILE_PATH, testPropertiesPath)
NiFiProperties unprocessedProperties = new NiFiPropertiesLoader().loadRaw(new File(testPropertiesPath))
def protectedKeys = getProtectedKeys(unprocessedProperties)
logger.info("Reading from raw properties file gives protected properties: ${protectedKeys}")
// Act
NiFiProperties properties = NiFi.initializeProperties(args)
// Assert
// Ensure that there were protected properties, they were encrypted using AES/GCM (128/256 bit key), and they were decrypted (raw value != retrieved value)
assert !hasProtectedKeys(properties)
def unprotectedProperties = decrypt(unprocessedProperties, DIFFERENT_KEY)
getProtectedPropertyKeys(unprocessedProperties).every { k, v ->
String rawValue = unprocessedProperties.getProperty(k)
logger.raw("${k} -> ${rawValue}")
String retrievedValue = properties.getProperty(k)
logger.decrypted("${k} -> ${retrievedValue}")
assert v =~ "aes/gcm"
logger.assert("${retrievedValue} != ${rawValue}")
assert retrievedValue != rawValue
String decryptedProperty = unprotectedProperties.getProperty(k)
logger.assert("${retrievedValue} == ${decryptedProperty}")
assert retrievedValue == decryptedProperty
}
}
private static boolean hasProtectedKeys(NiFiProperties properties) {
properties.getPropertyKeys().any { it.endsWith(".protected") }
}
private static Map<String, String> getProtectedPropertyKeys(NiFiProperties properties) {
getProtectedKeys(properties).collectEntries { String key ->
[(key): properties.getProperty(key + ".protected")]
}
}
private static Set<String> getProtectedKeys(NiFiProperties properties) {
properties.getPropertyKeys().findAll { it.endsWith(".protected") }
}
private static NiFiProperties decrypt(NiFiProperties encryptedProperties, String keyHex) {
AESSensitivePropertyProvider spp = new AESSensitivePropertyProvider(keyHex)
def map = encryptedProperties.getPropertyKeys().collectEntries { String key ->
if (encryptedProperties.getProperty(key + ".protected") == spp.getIdentifierKey()) {
[(key): spp.unprotect(encryptedProperties.getProperty(key))]
} else if (!key.endsWith(".protected")) {
[(key): encryptedProperties.getProperty(key)]
}
}
new StandardNiFiProperties(map as Properties)
}
@Test
public void testShouldValidateKeys() {
// Arrange
final List<String> VALID_KEYS = [
"0" * 64, // 256 bit keys
"ABCDEF01" * 8,
"0123" * 8, // 128 bit keys
"0123456789ABCDEFFEDCBA9876543210",
"0123456789ABCDEFFEDCBA9876543210".toLowerCase(),
]
// Act
def isValid = VALID_KEYS.collectEntries { String key -> [(key): NiFi.isHexKeyValid(key)] }
logger.info("Key validity: ${isValid}")
// Assert
assert isValid.every { k, v -> v }
}
@Test
public void testShouldNotValidateInvalidKeys() {
// Arrange
final List<String> VALID_KEYS = [
"0" * 63,
"ABCDEFG1" * 8,
"0123" * 9,
"0123456789ABCDEFFEDCBA987654321",
"0123456789ABCDEF FEDCBA9876543210".toLowerCase(),
null,
"",
" "
]
// Act
def isValid = VALID_KEYS.collectEntries { String key -> [(key): NiFi.isHexKeyValid(key)] }
logger.info("Key validity: ${isValid}")
// Assert
assert isValid.every { k, v -> !v }
}
}
public class TestAppender extends AppenderBase<LoggingEvent> {
static List<LoggingEvent> events = new ArrayList<>();
@Override
protected void append(LoggingEvent e) {
synchronized (events) {
events.add(e);
}
}
public static void reset() {
synchronized (events) {
events.clear();
}
}
}

188
nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-runtime/src/test/resources/NiFiProperties/conf/nifi.properties Normal file
View File

@ -0,0 +1,188 @@
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# Core Properties #
nifi.version=1.0.0-SNAPSHOT
nifi.flow.configuration.file=./target/conf/flow.xml.gz
nifi.flow.configuration.archive.enabled=true
nifi.flow.configuration.archive.dir=./target/conf/archive/
nifi.flow.configuration.archive.max.time=30 days
nifi.flow.configuration.archive.max.storage=500 MB
nifi.flowcontroller.autoResumeState=true
nifi.flowcontroller.graceful.shutdown.period=10 sec
nifi.flowservice.writedelay.interval=500 ms
nifi.administrative.yield.duration=30 sec
# If a component has no work to do (is "bored"), how long should we wait before checking again for work?
nifi.bored.yield.duration=10 millis
nifi.authorizer.configuration.file=./target/conf/authorizers.xml
nifi.login.identity.provider.configuration.file=./target/conf/login-identity-providers.xml
nifi.templates.directory=./target/conf/templates
nifi.ui.banner.text=
nifi.ui.autorefresh.interval=30 sec
nifi.nar.library.directory=./target/lib
nifi.nar.working.directory=./target/work/nar/
nifi.documentation.working.directory=./target/work/docs/components
####################
# State Management #
####################
nifi.state.management.configuration.file=./target/conf/state-management.xml
# The ID of the local state provider
nifi.state.management.provider.local=local-provider
# The ID of the cluster-wide state provider. This will be ignored if NiFi is not clustered but must be populated if running in a cluster.
nifi.state.management.provider.cluster=zk-provider
# Specifies whether or not this instance of NiFi should run an embedded ZooKeeper server
nifi.state.management.embedded.zookeeper.start=false
# Properties file that provides the ZooKeeper properties to use if <nifi.state.management.embedded.zookeeper.start> is set to true
nifi.state.management.embedded.zookeeper.properties=./target/conf/zookeeper.properties
# H2 Settings
nifi.database.directory=./target/database_repository
nifi.h2.url.append=;LOCK_TIMEOUT=25000;WRITE_DELAY=0;AUTO_SERVER=FALSE
# FlowFile Repository
nifi.flowfile.repository.implementation=org.apache.nifi.controller.repository.WriteAheadFlowFileRepository
nifi.flowfile.repository.directory=./target/flowfile_repository
nifi.flowfile.repository.partitions=256
nifi.flowfile.repository.checkpoint.interval=2 mins
nifi.flowfile.repository.always.sync=false
nifi.swap.manager.implementation=org.apache.nifi.controller.FileSystemSwapManager
nifi.queue.swap.threshold=20000
nifi.swap.in.period=5 sec
nifi.swap.in.threads=1
nifi.swap.out.period=5 sec
nifi.swap.out.threads=4
# Content Repository
nifi.content.repository.implementation=org.apache.nifi.controller.repository.FileSystemRepository
nifi.content.claim.max.appendable.size=10 MB
nifi.content.claim.max.flow.files=100
nifi.content.repository.directory.default=./target/content_repository
nifi.content.repository.archive.max.retention.period=12 hours
nifi.content.repository.archive.max.usage.percentage=50%
nifi.content.repository.archive.enabled=true
nifi.content.repository.always.sync=false
nifi.content.viewer.url=/nifi-content-viewer/
# Provenance Repository Properties
nifi.provenance.repository.implementation=org.apache.nifi.provenance.PersistentProvenanceRepository
# Persistent Provenance Repository Properties
nifi.provenance.repository.directory.default=./target/provenance_repository
nifi.provenance.repository.max.storage.time=24 hours
nifi.provenance.repository.max.storage.size=1 GB
nifi.provenance.repository.rollover.time=30 secs
nifi.provenance.repository.rollover.size=100 MB
nifi.provenance.repository.query.threads=2
nifi.provenance.repository.index.threads=1
nifi.provenance.repository.compress.on.rollover=true
nifi.provenance.repository.always.sync=false
nifi.provenance.repository.journal.count=16
# Comma-separated list of fields. Fields that are not indexed will not be searchable. Valid fields are:
# EventType, FlowFileUUID, Filename, TransitURI, ProcessorID, AlternateIdentifierURI, Relationship, Details
nifi.provenance.repository.indexed.fields=EventType, FlowFileUUID, Filename, ProcessorID, Relationship
# FlowFile Attributes that should be indexed and made searchable. Some examples to consider are filename, uuid, mime.type
nifi.provenance.repository.indexed.attributes=
# Large values for the shard size will result in more Java heap usage when searching the Provenance Repository
# but should provide better performance
nifi.provenance.repository.index.shard.size=500 MB
# Indicates the maximum length that a FlowFile attribute can be when retrieving a Provenance Event from
# the repository. If the length of any attribute exceeds this value, it will be truncated when the event is retrieved.
nifi.provenance.repository.max.attribute.length=65536
# Volatile Provenance Respository Properties
nifi.provenance.repository.buffer.size=100000
# Component Status Repository
nifi.components.status.repository.implementation=org.apache.nifi.controller.status.history.VolatileComponentStatusRepository
nifi.components.status.repository.buffer.size=1440
nifi.components.status.snapshot.frequency=1 min
# Site to Site properties
nifi.remote.input.host=
nifi.remote.input.secure=false
nifi.remote.input.socket.port=
nifi.remote.input.http.enabled=true
nifi.remote.input.http.transaction.ttl=30 sec
# web properties #
nifi.web.war.directory=./target/lib
nifi.web.http.host=
nifi.web.http.port=8080
nifi.web.https.host=
nifi.web.https.port=
nifi.web.jetty.working.directory=./target/work/jetty
nifi.web.jetty.threads=200
# security properties #
nifi.sensitive.props.key=
nifi.sensitive.props.algorithm=PBEWITHMD5AND256BITAES-CBC-OPENSSL
nifi.sensitive.props.provider=BC
nifi.sensitive.props.additional.keys=
nifi.security.keystore=
nifi.security.keystoreType=
nifi.security.keystorePasswd=
nifi.security.keyPasswd=
nifi.security.truststore=
nifi.security.truststoreType=
nifi.security.truststorePasswd=
nifi.security.needClientAuth=
nifi.security.user.authorizer=file-provider
nifi.security.user.login.identity.provider=
nifi.security.ocsp.responder.url=
nifi.security.ocsp.responder.certificate=
# Identity Mapping Properties #
# These properties allow normalizing user identities such that identities coming from different identity providers
# (certificates, LDAP, Kerberos) can be treated the same internally in NiFi. The following example demonstrates normalizing
# DNs from certificates and principals from Kerberos into a common identity string:
#
# nifi.security.identity.mapping.pattern.dn=^CN=(.*?), OU=(.*?), O=(.*?), L=(.*?), ST=(.*?), C=(.*?)$
# nifi.security.identity.mapping.value.dn=$1@$2
# nifi.security.identity.mapping.pattern.kerb=^(.*?)/instance@(.*?)$
# nifi.security.identity.mapping.value.kerb=$1@$2
# cluster common properties (all nodes must have same values) #
nifi.cluster.protocol.heartbeat.interval=5 sec
nifi.cluster.protocol.is.secure=false
# cluster node properties (only configure for cluster nodes) #
nifi.cluster.is.node=false
nifi.cluster.node.address=
nifi.cluster.node.protocol.port=
nifi.cluster.node.protocol.threads=10
nifi.cluster.node.event.history.size=25
nifi.cluster.node.connection.timeout=5 sec
nifi.cluster.node.read.timeout=5 sec
nifi.cluster.firewall.file=
# How long a request should be allowed to hold a 'lock' on a component. #
nifi.cluster.request.replication.claim.timeout=15 secs
# zookeeper properties, used for cluster management #
nifi.zookeeper.connect.string=
nifi.zookeeper.connect.timeout=3 secs
nifi.zookeeper.session.timeout=3 secs
nifi.zookeeper.root.node=/nifi
# kerberos #
nifi.kerberos.krb5.file=
nifi.kerberos.service.principal=
nifi.kerberos.keytab.location=
nifi.kerberos.authentication.expiration=12 hours

189
nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-runtime/src/test/resources/NiFiProperties/conf/nifi_with_sensitive_properties_protected_aes.properties Normal file
View File

@ -0,0 +1,189 @@
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# Core Properties #
nifi.version=1.0.0-SNAPSHOT
nifi.flow.configuration.file=./target/conf/flow.xml.gz
nifi.flow.configuration.archive.enabled=true
nifi.flow.configuration.archive.dir=./target/conf/archive/
nifi.flow.configuration.archive.max.time=30 days
nifi.flow.configuration.archive.max.storage=500 MB
nifi.flowcontroller.autoResumeState=true
nifi.flowcontroller.graceful.shutdown.period=10 sec
nifi.flowservice.writedelay.interval=500 ms
nifi.administrative.yield.duration=30 sec
# If a component has no work to do (is "bored"), how long should we wait before checking again for work?
nifi.bored.yield.duration=10 millis
nifi.authorizer.configuration.file=./target/conf/authorizers.xml
nifi.login.identity.provider.configuration.file=./target/conf/login-identity-providers.xml
nifi.templates.directory=./target/conf/templates
nifi.ui.banner.text=n8hL1zgQcYpG70Vm||e1hYsrc7FLvi1E9LcHM1VYeN5atWJIGg/WCsyuxxNqN1lK1ASGEZR8040NFZNqwsnbx+
nifi.ui.banner.text.protected=aes/gcm/256
nifi.ui.autorefresh.interval=30 sec
nifi.nar.library.directory=./target/lib
nifi.nar.working.directory=./target/work/nar/
nifi.documentation.working.directory=./target/work/docs/components
####################
# State Management #
####################
nifi.state.management.configuration.file=./target/conf/state-management.xml
# The ID of the local state provider
nifi.state.management.provider.local=local-provider
# The ID of the cluster-wide state provider. This will be ignored if NiFi is not clustered but must be populated if running in a cluster.
nifi.state.management.provider.cluster=zk-provider
# Specifies whether or not this instance of NiFi should run an embedded ZooKeeper server
nifi.state.management.embedded.zookeeper.start=false
# Properties file that provides the ZooKeeper properties to use if <nifi.state.management.embedded.zookeeper.start> is set to true
nifi.state.management.embedded.zookeeper.properties=./target/conf/zookeeper.properties
# H2 Settings
nifi.database.directory=./target/database_repository
nifi.h2.url.append=;LOCK_TIMEOUT=25000;WRITE_DELAY=0;AUTO_SERVER=FALSE
# FlowFile Repository
nifi.flowfile.repository.implementation=org.apache.nifi.controller.repository.WriteAheadFlowFileRepository
nifi.flowfile.repository.directory=./target/flowfile_repository
nifi.flowfile.repository.partitions=256
nifi.flowfile.repository.checkpoint.interval=2 mins
nifi.flowfile.repository.always.sync=false
nifi.swap.manager.implementation=org.apache.nifi.controller.FileSystemSwapManager
nifi.queue.swap.threshold=20000
nifi.swap.in.period=5 sec
nifi.swap.in.threads=1
nifi.swap.out.period=5 sec
nifi.swap.out.threads=4
# Content Repository
nifi.content.repository.implementation=org.apache.nifi.controller.repository.FileSystemRepository
nifi.content.claim.max.appendable.size=10 MB
nifi.content.claim.max.flow.files=100
nifi.content.repository.directory.default=./target/content_repository
nifi.content.repository.archive.max.retention.period=12 hours
nifi.content.repository.archive.max.usage.percentage=50%
nifi.content.repository.archive.enabled=true
nifi.content.repository.always.sync=false
nifi.content.viewer.url=/nifi-content-viewer/
# Provenance Repository Properties
nifi.provenance.repository.implementation=org.apache.nifi.provenance.PersistentProvenanceRepository
# Persistent Provenance Repository Properties
nifi.provenance.repository.directory.default=./target/provenance_repository
nifi.provenance.repository.max.storage.time=24 hours
nifi.provenance.repository.max.storage.size=1 GB
nifi.provenance.repository.rollover.time=30 secs
nifi.provenance.repository.rollover.size=100 MB
nifi.provenance.repository.query.threads=2
nifi.provenance.repository.index.threads=1
nifi.provenance.repository.compress.on.rollover=true
nifi.provenance.repository.always.sync=false
nifi.provenance.repository.journal.count=16
# Comma-separated list of fields. Fields that are not indexed will not be searchable. Valid fields are:
# EventType, FlowFileUUID, Filename, TransitURI, ProcessorID, AlternateIdentifierURI, Relationship, Details
nifi.provenance.repository.indexed.fields=EventType, FlowFileUUID, Filename, ProcessorID, Relationship
# FlowFile Attributes that should be indexed and made searchable. Some examples to consider are filename, uuid, mime.type
nifi.provenance.repository.indexed.attributes=
# Large values for the shard size will result in more Java heap usage when searching the Provenance Repository
# but should provide better performance
nifi.provenance.repository.index.shard.size=500 MB
# Indicates the maximum length that a FlowFile attribute can be when retrieving a Provenance Event from
# the repository. If the length of any attribute exceeds this value, it will be truncated when the event is retrieved.
nifi.provenance.repository.max.attribute.length=65536
# Volatile Provenance Respository Properties
nifi.provenance.repository.buffer.size=100000
# Component Status Repository
nifi.components.status.repository.implementation=org.apache.nifi.controller.status.history.VolatileComponentStatusRepository
nifi.components.status.repository.buffer.size=1440
nifi.components.status.snapshot.frequency=1 min
# Site to Site properties
nifi.remote.input.host=
nifi.remote.input.secure=false
nifi.remote.input.socket.port=
nifi.remote.input.http.enabled=true
nifi.remote.input.http.transaction.ttl=30 sec
# web properties #
nifi.web.war.directory=./target/lib
nifi.web.http.host=
nifi.web.http.port=8080
nifi.web.https.host=
nifi.web.https.port=
nifi.web.jetty.working.directory=./target/work/jetty
nifi.web.jetty.threads=200
# security properties #
nifi.sensitive.props.key=
nifi.sensitive.props.algorithm=PBEWITHMD5AND256BITAES-CBC-OPENSSL
nifi.sensitive.props.provider=BC
nifi.sensitive.props.additional.keys=nifi.ui.banner.text
nifi.security.keystore=
nifi.security.keystoreType=
nifi.security.keystorePasswd=
nifi.security.keyPasswd=
nifi.security.truststore=
nifi.security.truststoreType=
nifi.security.truststorePasswd=
nifi.security.needClientAuth=
nifi.security.user.authorizer=file-provider
nifi.security.user.login.identity.provider=
nifi.security.ocsp.responder.url=
nifi.security.ocsp.responder.certificate=
# Identity Mapping Properties #
# These properties allow normalizing user identities such that identities coming from different identity providers
# (certificates, LDAP, Kerberos) can be treated the same internally in NiFi. The following example demonstrates normalizing
# DNs from certificates and principals from Kerberos into a common identity string:
#
# nifi.security.identity.mapping.pattern.dn=^CN=(.*?), OU=(.*?), O=(.*?), L=(.*?), ST=(.*?), C=(.*?)$
# nifi.security.identity.mapping.value.dn=$1@$2
# nifi.security.identity.mapping.pattern.kerb=^(.*?)/instance@(.*?)$
# nifi.security.identity.mapping.value.kerb=$1@$2
# cluster common properties (all nodes must have same values) #
nifi.cluster.protocol.heartbeat.interval=5 sec
nifi.cluster.protocol.is.secure=false
# cluster node properties (only configure for cluster nodes) #
nifi.cluster.is.node=false
nifi.cluster.node.address=
nifi.cluster.node.protocol.port=
nifi.cluster.node.protocol.threads=10
nifi.cluster.node.event.history.size=25
nifi.cluster.node.connection.timeout=5 sec
nifi.cluster.node.read.timeout=5 sec
nifi.cluster.firewall.file=
# How long a request should be allowed to hold a 'lock' on a component. #
nifi.cluster.request.replication.claim.timeout=15 secs
# zookeeper properties, used for cluster management #
nifi.zookeeper.connect.string=
nifi.zookeeper.connect.timeout=3 secs
nifi.zookeeper.session.timeout=3 secs
nifi.zookeeper.root.node=/nifi
# kerberos #
nifi.kerberos.krb5.file=
nifi.kerberos.service.principal=
nifi.kerberos.keytab.location=
nifi.kerberos.authentication.expiration=12 hours

192
nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-runtime/src/test/resources/NiFiProperties/conf/nifi_with_sensitive_properties_protected_aes_different_key.properties Normal file
View File

@ -0,0 +1,192 @@
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# Core Properties #
nifi.version=1.0.0-SNAPSHOT
nifi.flow.configuration.file=./target/conf/flow.xml.gz
nifi.flow.configuration.archive.enabled=true
nifi.flow.configuration.archive.dir=./target/conf/archive/
nifi.flow.configuration.archive.max.time=30 days
nifi.flow.configuration.archive.max.storage=500 MB
nifi.flowcontroller.autoResumeState=true
nifi.flowcontroller.graceful.shutdown.period=10 sec
nifi.flowservice.writedelay.interval=500 ms
nifi.administrative.yield.duration=30 sec
# If a component has no work to do (is "bored"), how long should we wait before checking again for work?
nifi.bored.yield.duration=10 millis
nifi.authorizer.configuration.file=./target/conf/authorizers.xml
nifi.login.identity.provider.configuration.file=./target/conf/login-identity-providers.xml
nifi.templates.directory=./target/conf/templates
nifi.ui.banner.text=dXwnu9mLyPETJrq1||n9e5dk5+HSTBCGOA/Sy6VYzwPw3baeRNvglalA1Pr1PcToyc4/qT6md24YOP4xVz14jd
nifi.ui.banner.text.protected=aes/gcm/256
nifi.ui.autorefresh.interval=30 sec
nifi.nar.library.directory=./target/lib
nifi.nar.working.directory=./target/work/nar/
nifi.documentation.working.directory=./target/work/docs/components
####################
# State Management #
####################
nifi.state.management.configuration.file=./target/conf/state-management.xml
# The ID of the local state provider
nifi.state.management.provider.local=local-provider
# The ID of the cluster-wide state provider. This will be ignored if NiFi is not clustered but must be populated if running in a cluster.
nifi.state.management.provider.cluster=zk-provider
# Specifies whether or not this instance of NiFi should run an embedded ZooKeeper server
nifi.state.management.embedded.zookeeper.start=false
# Properties file that provides the ZooKeeper properties to use if <nifi.state.management.embedded.zookeeper.start> is set to true
nifi.state.management.embedded.zookeeper.properties=./target/conf/zookeeper.properties
# H2 Settings
nifi.database.directory=./target/database_repository
nifi.h2.url.append=;LOCK_TIMEOUT=25000;WRITE_DELAY=0;AUTO_SERVER=FALSE
# FlowFile Repository
nifi.flowfile.repository.implementation=org.apache.nifi.controller.repository.WriteAheadFlowFileRepository
nifi.flowfile.repository.directory=./target/flowfile_repository
nifi.flowfile.repository.partitions=256
nifi.flowfile.repository.checkpoint.interval=2 mins
nifi.flowfile.repository.always.sync=false
nifi.swap.manager.implementation=org.apache.nifi.controller.FileSystemSwapManager
nifi.queue.swap.threshold=20000
nifi.swap.in.period=5 sec
nifi.swap.in.threads=1
nifi.swap.out.period=5 sec
nifi.swap.out.threads=4
# Content Repository
nifi.content.repository.implementation=org.apache.nifi.controller.repository.FileSystemRepository
nifi.content.claim.max.appendable.size=10 MB
nifi.content.claim.max.flow.files=100
nifi.content.repository.directory.default=./target/content_repository
nifi.content.repository.archive.max.retention.period=12 hours
nifi.content.repository.archive.max.usage.percentage=50%
nifi.content.repository.archive.enabled=true
nifi.content.repository.always.sync=false
nifi.content.viewer.url=/nifi-content-viewer/
# Provenance Repository Properties
nifi.provenance.repository.implementation=org.apache.nifi.provenance.PersistentProvenanceRepository
# Persistent Provenance Repository Properties
nifi.provenance.repository.directory.default=./target/provenance_repository
nifi.provenance.repository.max.storage.time=24 hours
nifi.provenance.repository.max.storage.size=1 GB
nifi.provenance.repository.rollover.time=30 secs
nifi.provenance.repository.rollover.size=100 MB
nifi.provenance.repository.query.threads=2
nifi.provenance.repository.index.threads=1
nifi.provenance.repository.compress.on.rollover=true
nifi.provenance.repository.always.sync=false
nifi.provenance.repository.journal.count=16
# Comma-separated list of fields. Fields that are not indexed will not be searchable. Valid fields are:
# EventType, FlowFileUUID, Filename, TransitURI, ProcessorID, AlternateIdentifierURI, Relationship, Details
nifi.provenance.repository.indexed.fields=EventType, FlowFileUUID, Filename, ProcessorID, Relationship
# FlowFile Attributes that should be indexed and made searchable. Some examples to consider are filename, uuid, mime.type
nifi.provenance.repository.indexed.attributes=
# Large values for the shard size will result in more Java heap usage when searching the Provenance Repository
# but should provide better performance
nifi.provenance.repository.index.shard.size=500 MB
# Indicates the maximum length that a FlowFile attribute can be when retrieving a Provenance Event from
# the repository. If the length of any attribute exceeds this value, it will be truncated when the event is retrieved.
nifi.provenance.repository.max.attribute.length=65536
# Volatile Provenance Respository Properties
nifi.provenance.repository.buffer.size=100000
# Component Status Repository
nifi.components.status.repository.implementation=org.apache.nifi.controller.status.history.VolatileComponentStatusRepository
nifi.components.status.repository.buffer.size=1440
nifi.components.status.snapshot.frequency=1 min
# Site to Site properties
nifi.remote.input.host=
nifi.remote.input.secure=false
nifi.remote.input.socket.port=
nifi.remote.input.http.enabled=true
nifi.remote.input.http.transaction.ttl=30 sec
# web properties #
nifi.web.war.directory=./target/lib
nifi.web.http.host=
nifi.web.http.port=8080
nifi.web.https.host=
nifi.web.https.port=
nifi.web.jetty.working.directory=./target/work/jetty
nifi.web.jetty.threads=200
# security properties #
nifi.sensitive.props.key=dQU402Mz4J+t+e18||6+ictR0Nssq3/rR/d8fq5CFAKmpakr9jCyPIJYxG7n6D86gxsu2TRp4M48ugUw==
nifi.sensitive.props.key.protected=aes/gcm/256
nifi.sensitive.props.algorithm=PBEWITHMD5AND256BITAES-CBC-OPENSSL
nifi.sensitive.props.provider=BC
nifi.sensitive.props.additional.keys=
nifi.security.keystore=/path/to/keystore.jks
nifi.security.keystoreType=JKS
nifi.security.keystorePasswd=Q8T3wv+Xl2ie98GV||qsuY9wa/Rt27cqFXs8ebX25E1iSbFAEFcD0cjCwrl3Tw6HghQjBIaCzQ
nifi.security.keystorePasswd.protected=aes/gcm/256
nifi.security.keyPasswd=1S0XmoiAr379B8rg||PPZzjdw9BAJSon9g4xm9uscFhCCyk734FTjXtRnBXUy819zsoQ==
nifi.security.keyPasswd.protected=aes/gcm/256
nifi.security.truststore=
nifi.security.truststoreType=
nifi.security.truststorePasswd=
nifi.security.needClientAuth=
nifi.security.user.authorizer=file-provider
nifi.security.user.login.identity.provider=
nifi.security.ocsp.responder.url=
nifi.security.ocsp.responder.certificate=
# Identity Mapping Properties #
# These properties allow normalizing user identities such that identities coming from different identity providers
# (certificates, LDAP, Kerberos) can be treated the same internally in NiFi. The following example demonstrates normalizing
# DNs from certificates and principals from Kerberos into a common identity string:
#
# nifi.security.identity.mapping.pattern.dn=^CN=(.*?), OU=(.*?), O=(.*?), L=(.*?), ST=(.*?), C=(.*?)$
# nifi.security.identity.mapping.value.dn=$1@$2
# nifi.security.identity.mapping.pattern.kerb=^(.*?)/instance@(.*?)$
# nifi.security.identity.mapping.value.kerb=$1@$2
# cluster common properties (all nodes must have same values) #
nifi.cluster.protocol.heartbeat.interval=5 sec
nifi.cluster.protocol.is.secure=false
# cluster node properties (only configure for cluster nodes) #
nifi.cluster.is.node=false
nifi.cluster.node.address=
nifi.cluster.node.protocol.port=
nifi.cluster.node.protocol.threads=10
nifi.cluster.node.event.history.size=25
nifi.cluster.node.connection.timeout=5 sec
nifi.cluster.node.read.timeout=5 sec
nifi.cluster.firewall.file=
# How long a request should be allowed to hold a 'lock' on a component. #
nifi.cluster.request.replication.claim.timeout=15 secs
# zookeeper properties, used for cluster management #
nifi.zookeeper.connect.string=
nifi.zookeeper.connect.timeout=3 secs
nifi.zookeeper.session.timeout=3 secs
nifi.zookeeper.root.node=/nifi
# kerberos #
nifi.kerberos.krb5.file=
nifi.kerberos.service.principal=
nifi.kerberos.keytab.location=
nifi.kerberos.authentication.expiration=12 hours

34
nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-runtime/src/test/resources/logback-test.xml Normal file
View File

@ -0,0 +1,34 @@
<?xml version="1.0" encoding="UTF-8"?>
<!--
Licensed to the Apache Software Foundation (ASF) under one or more
contributor license agreements. See the NOTICE file distributed with
this work for additional information regarding copyright ownership.
The ASF licenses this file to You under the Apache License, Version 2.0
(the "License"); you may not use this file except in compliance with
the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
-->
<configuration>
<appender name="CONSOLE" class="ch.qos.logback.core.ConsoleAppender">
<encoder class="ch.qos.logback.classic.encoder.PatternLayoutEncoder">
<pattern>%-4r [%t] %-5p %c - %m%n</pattern>
</encoder>
</appender>
<appender name="TEST" class="org.apache.nifi.TestAppender">
<encoder class="ch.qos.logback.classic.encoder.PatternLayoutEncoder">
<pattern>%-4r [%t] %-5p %c - %m%n</pattern>
</encoder>
</appender>
<logger name="org.apache.nifi.processor" level="DEBUG"/>
<root level="DEBUG">
<appender-ref ref="CONSOLE"/>
<appender-ref ref="TEST"/>
</root>
</configuration>

14
nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-site-to-site/src/main/java/org/apache/nifi/remote/protocol/http/StandardHttpFlowFileServerProtocol.java
View File

@ -93,7 +93,7 @@ public class StandardHttpFlowFileServerProtocol extends AbstractFlowFileServerPr
HttpServerCommunicationsSession commSession = (HttpServerCommunicationsSession) commsSession;
commSession.setResponseCode(response);
if (isTransfer) {
if(isTransfer){
switch (response) {
case NO_MORE_DATA:
logger.debug("{} There's no data to send.", this);
@ -138,8 +138,8 @@ public class StandardHttpFlowFileServerProtocol extends AbstractFlowFileServerPr
ByteArrayOutputStream bos = new ByteArrayOutputStream();
Transaction.TransactionState currentStatus = commSession.getStatus();
if (isTransfer) {
switch (currentStatus) {
if(isTransfer){
switch (currentStatus){
case DATA_EXCHANGED:
String clientChecksum = commSession.getChecksum();
logger.debug("readTransactionResponse. clientChecksum={}", clientChecksum);
@ -151,7 +151,7 @@ public class StandardHttpFlowFileServerProtocol extends AbstractFlowFileServerPr
break;
}
} else {
switch (currentStatus) {
switch (currentStatus){
case TRANSACTION_STARTED:
logger.debug("readTransactionResponse. returning CONTINUE_TRANSACTION.");
// We don't know if there's more data to receive, so just continue it.
@ -161,7 +161,7 @@ public class StandardHttpFlowFileServerProtocol extends AbstractFlowFileServerPr
// Checksum was successfully validated at client side, or BAD_CHECKSUM is returned.
ResponseCode responseCode = commSession.getResponseCode();
logger.debug("readTransactionResponse. responseCode={}", responseCode);
if (responseCode.containsMessage()) {
if(responseCode.containsMessage()){
responseCode.writeResponse(new DataOutputStream(bos), "");
} else {
responseCode.writeResponse(new DataOutputStream(bos));
@ -228,8 +228,8 @@ public class StandardHttpFlowFileServerProtocol extends AbstractFlowFileServerPr
}
@Override
public void sendPeerList(Peer peer, Optional<ClusterNodeInformation> clusterNodeInfo, String remoteInputHost,
int remoteInputPort, int remoteInputHttpPort, boolean isSiteToSiteSecure) throws IOException {
public void sendPeerList(Peer peer, Optional<ClusterNodeInformation> clusterNodeInfo, String remoteInputHost, int remoteInputPort, int remoteInputHttpPort,
boolean isSiteToSiteSecure) throws IOException {
}
@Override

27
nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-site-to-site/src/main/java/org/apache/nifi/remote/protocol/socket/SocketFlowFileServerProtocol.java
View File

@ -16,6 +16,16 @@
*/
package org.apache.nifi.remote.protocol.socket;
import java.io.DataInputStream;
import java.io.DataOutputStream;
import java.io.IOException;
import java.net.InetAddress;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import org.apache.nifi.remote.Peer;
import org.apache.nifi.remote.RemoteResourceFactory;
import org.apache.nifi.remote.StandardVersionNegotiator;
@ -31,17 +41,6 @@ import org.apache.nifi.remote.protocol.HandshakeProperties;
import org.apache.nifi.remote.protocol.RequestType;
import org.apache.nifi.remote.protocol.ResponseCode;
import java.io.DataInputStream;
import java.io.DataOutputStream;
import java.io.IOException;
import java.net.InetAddress;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Optional;
public class SocketFlowFileServerProtocol extends AbstractFlowFileServerProtocol {
public static final String RESOURCE_NAME = "SocketFlowFileProtocol";
@ -83,7 +82,7 @@ public class SocketFlowFileServerProtocol extends AbstractFlowFileServerProtocol
validateHandshakeRequest(confirmed, peer, properties);
} catch (HandshakeException e) {
ResponseCode handshakeResult = e.getResponseCode();
if (handshakeResult.containsMessage()) {
if(handshakeResult.containsMessage()){
handshakeResult.writeResponse(dos, e.getMessage());
} else {
handshakeResult.writeResponse(dos);
@ -135,6 +134,7 @@ public class SocketFlowFileServerProtocol extends AbstractFlowFileServerProtocol
}
}
@Override
public RequestType getRequestType(final Peer peer) throws IOException {
if (!handshakeCompleted) {
@ -181,7 +181,7 @@ public class SocketFlowFileServerProtocol extends AbstractFlowFileServerProtocol
nodeInfos = new ArrayList<>(clusterNodeInfo.get().getNodeInformation());
} else {
final NodeInformation self = new NodeInformation(remoteInputHostVal, remoteInputPort, remoteInputHttpPort, remoteInputHttpPort,
isSiteToSiteSecure, 0);
isSiteToSiteSecure, 0);
nodeInfos = Collections.singletonList(self);
}
@ -214,6 +214,7 @@ public class SocketFlowFileServerProtocol extends AbstractFlowFileServerProtocol
return RESOURCE_NAME;
}
@Override
public VersionNegotiator getVersionNegotiator() {
return versionNegotiator;

198
nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/api/DataTransferResource.java
View File

@ -16,12 +16,47 @@
*/
package org.apache.nifi.web.api;
import static org.apache.commons.lang3.StringUtils.isEmpty;
import static org.apache.nifi.remote.protocol.HandshakeProperty.BATCH_COUNT;
import static org.apache.nifi.remote.protocol.HandshakeProperty.BATCH_DURATION;
import static org.apache.nifi.remote.protocol.HandshakeProperty.BATCH_SIZE;
import static org.apache.nifi.remote.protocol.HandshakeProperty.REQUEST_EXPIRATION_MILLIS;
import static org.apache.nifi.remote.protocol.http.HttpHeaders.HANDSHAKE_PROPERTY_BATCH_COUNT;
import static org.apache.nifi.remote.protocol.http.HttpHeaders.HANDSHAKE_PROPERTY_BATCH_DURATION;
import static org.apache.nifi.remote.protocol.http.HttpHeaders.HANDSHAKE_PROPERTY_BATCH_SIZE;
import static org.apache.nifi.remote.protocol.http.HttpHeaders.HANDSHAKE_PROPERTY_REQUEST_EXPIRATION;
import static org.apache.nifi.remote.protocol.http.HttpHeaders.HANDSHAKE_PROPERTY_USE_COMPRESSION;
import com.wordnik.swagger.annotations.Api;
import com.wordnik.swagger.annotations.ApiOperation;
import com.wordnik.swagger.annotations.ApiParam;
import com.wordnik.swagger.annotations.ApiResponse;
import com.wordnik.swagger.annotations.ApiResponses;
import com.wordnik.swagger.annotations.Authorization;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.util.HashMap;
import java.util.Map;
import javax.servlet.ServletContext;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
import javax.ws.rs.Consumes;
import javax.ws.rs.DELETE;
import javax.ws.rs.DefaultValue;
import javax.ws.rs.GET;
import javax.ws.rs.POST;
import javax.ws.rs.PUT;
import javax.ws.rs.Path;
import javax.ws.rs.PathParam;
import javax.ws.rs.Produces;
import javax.ws.rs.QueryParam;
import javax.ws.rs.WebApplicationException;
import javax.ws.rs.core.Context;
import javax.ws.rs.core.MediaType;
import javax.ws.rs.core.Response;
import javax.ws.rs.core.StreamingOutput;
import javax.ws.rs.core.UriInfo;
import org.apache.commons.lang3.StringUtils;
import org.apache.nifi.authorization.AccessDeniedException;
import org.apache.nifi.authorization.AuthorizationRequest;
@ -53,47 +88,11 @@ import org.apache.nifi.remote.protocol.ResponseCode;
import org.apache.nifi.remote.protocol.http.HttpFlowFileServerProtocol;
import org.apache.nifi.remote.protocol.http.StandardHttpFlowFileServerProtocol;
import org.apache.nifi.stream.io.ByteArrayOutputStream;
import org.apache.nifi.util.NiFiProperties;
import org.apache.nifi.web.api.entity.TransactionResultEntity;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import javax.servlet.ServletContext;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
import javax.ws.rs.Consumes;
import javax.ws.rs.DELETE;
import javax.ws.rs.DefaultValue;
import javax.ws.rs.GET;
import javax.ws.rs.POST;
import javax.ws.rs.PUT;
import javax.ws.rs.Path;
import javax.ws.rs.PathParam;
import javax.ws.rs.Produces;
import javax.ws.rs.QueryParam;
import javax.ws.rs.WebApplicationException;
import javax.ws.rs.core.Context;
import javax.ws.rs.core.MediaType;
import javax.ws.rs.core.Response;
import javax.ws.rs.core.StreamingOutput;
import javax.ws.rs.core.UriInfo;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.util.HashMap;
import java.util.Map;
import static org.apache.commons.lang3.StringUtils.isEmpty;
import static org.apache.nifi.remote.protocol.HandshakeProperty.BATCH_COUNT;
import static org.apache.nifi.remote.protocol.HandshakeProperty.BATCH_DURATION;
import static org.apache.nifi.remote.protocol.HandshakeProperty.BATCH_SIZE;
import static org.apache.nifi.remote.protocol.HandshakeProperty.REQUEST_EXPIRATION_MILLIS;
import static org.apache.nifi.remote.protocol.http.HttpHeaders.HANDSHAKE_PROPERTY_BATCH_COUNT;
import static org.apache.nifi.remote.protocol.http.HttpHeaders.HANDSHAKE_PROPERTY_BATCH_DURATION;
import static org.apache.nifi.remote.protocol.http.HttpHeaders.HANDSHAKE_PROPERTY_BATCH_SIZE;
import static org.apache.nifi.remote.protocol.http.HttpHeaders.HANDSHAKE_PROPERTY_REQUEST_EXPIRATION;
import static org.apache.nifi.remote.protocol.http.HttpHeaders.HANDSHAKE_PROPERTY_USE_COMPRESSION;
import org.apache.nifi.util.NiFiProperties;
/**
* RESTful endpoint for managing a SiteToSite connection.
*/
@ -109,6 +108,7 @@ public class DataTransferResource extends ApplicationResource {
public static final String CHECK_SUM = "checksum";
public static final String RESPONSE_CODE = "responseCode";
private static final String PORT_TYPE_INPUT = "input-ports";
private static final String PORT_TYPE_OUTPUT = "output-ports";
@ -116,8 +116,10 @@ public class DataTransferResource extends ApplicationResource {
private final ResponseCreator responseCreator = new ResponseCreator();
private final VersionNegotiator transportProtocolVersionNegotiator = new TransportProtocolVersionNegotiator(1);
private final HttpRemoteSiteListener transactionManager;
private final NiFiProperties nifiProperties;
public DataTransferResource(final NiFiProperties nifiProperties) {
public DataTransferResource(final NiFiProperties nifiProperties){
this.nifiProperties = nifiProperties;
transactionManager = HttpRemoteSiteListener.getInstance(nifiProperties);
}
@ -165,17 +167,18 @@ public class DataTransferResource extends ApplicationResource {
value = "Create a transaction to the specified output port or input port",
response = TransactionResultEntity.class,
authorizations = {
@Authorization(value = "Write - /data-transfer/{component-type}/{uuid}", type = "")
@Authorization(value = "Write - /data-transfer/{component-type}/{uuid}", type = "")
}
)
@ApiResponses(
value = {
@ApiResponse(code = 400, message = "NiFi was unable to complete the request because it was invalid. The request should not be retried without modification."),
@ApiResponse(code = 401, message = "Client could not be authenticated."),
@ApiResponse(code = 403, message = "Client is not authorized to make this request."),
@ApiResponse(code = 404, message = "The specified resource could not be found."),
@ApiResponse(code = 409, message = "The request was valid but NiFi was not in the appropriate state to process it. Retrying the same request later may be successful."),
@ApiResponse(code = 503, message = "NiFi instance is not ready for serving request, or temporarily overloaded. Retrying the same request later may be successful"),}
@ApiResponse(code = 400, message = "NiFi was unable to complete the request because it was invalid. The request should not be retried without modification."),
@ApiResponse(code = 401, message = "Client could not be authenticated."),
@ApiResponse(code = 403, message = "Client is not authorized to make this request."),
@ApiResponse(code = 404, message = "The specified resource could not be found."),
@ApiResponse(code = 409, message = "The request was valid but NiFi was not in the appropriate state to process it. Retrying the same request later may be successful."),
@ApiResponse(code = 503, message = "NiFi instance is not ready for serving request, or temporarily overloaded. Retrying the same request later may be successful"),
}
)
public Response createPortTransaction(
@ApiParam(
@ -190,6 +193,7 @@ public class DataTransferResource extends ApplicationResource {
@Context UriInfo uriInfo,
InputStream inputStream) {
if (!PORT_TYPE_INPUT.equals(portType) && !PORT_TYPE_OUTPUT.equals(portType)) {
return responseCreator.wrongPortTypeResponse(portType, portId);
}
@ -237,17 +241,18 @@ public class DataTransferResource extends ApplicationResource {
value = "Transfer flow files to the input port",
response = String.class,
authorizations = {
@Authorization(value = "Write - /data-transfer/input-ports/{uuid}", type = "")
@Authorization(value = "Write - /data-transfer/input-ports/{uuid}", type = "")
}
)
@ApiResponses(
value = {
@ApiResponse(code = 400, message = "NiFi was unable to complete the request because it was invalid. The request should not be retried without modification."),
@ApiResponse(code = 401, message = "Client could not be authenticated."),
@ApiResponse(code = 403, message = "Client is not authorized to make this request."),
@ApiResponse(code = 404, message = "The specified resource could not be found."),
@ApiResponse(code = 409, message = "The request was valid but NiFi was not in the appropriate state to process it. Retrying the same request later may be successful."),
@ApiResponse(code = 503, message = "NiFi instance is not ready for serving request, or temporarily overloaded. Retrying the same request later may be successful"),}
@ApiResponse(code = 400, message = "NiFi was unable to complete the request because it was invalid. The request should not be retried without modification."),
@ApiResponse(code = 401, message = "Client could not be authenticated."),
@ApiResponse(code = 403, message = "Client is not authorized to make this request."),
@ApiResponse(code = 404, message = "The specified resource could not be found."),
@ApiResponse(code = 409, message = "The request was valid but NiFi was not in the appropriate state to process it. Retrying the same request later may be successful."),
@ApiResponse(code = 503, message = "NiFi instance is not ready for serving request, or temporarily overloaded. Retrying the same request later may be successful"),
}
)
public Response receiveFlowFiles(
@ApiParam(
@ -300,26 +305,26 @@ public class DataTransferResource extends ApplicationResource {
}
private HttpFlowFileServerProtocol initiateServerProtocol(final HttpServletRequest req, final Peer peer,
final Integer transportProtocolVersion) throws IOException {
final Integer transportProtocolVersion) throws IOException {
// Switch transaction protocol version based on transport protocol version.
TransportProtocolVersionNegotiator negotiatedTransportProtocolVersion = new TransportProtocolVersionNegotiator(transportProtocolVersion);
VersionNegotiator versionNegotiator = new StandardVersionNegotiator(negotiatedTransportProtocolVersion.getTransactionProtocolVersion());
final String dataTransferUrl = req.getRequestURL().toString();
((HttpCommunicationsSession) peer.getCommunicationsSession()).setDataTransferUrl(dataTransferUrl);
((HttpCommunicationsSession)peer.getCommunicationsSession()).setDataTransferUrl(dataTransferUrl);
HttpFlowFileServerProtocol serverProtocol = getHttpFlowFileServerProtocol(versionNegotiator);
HttpRemoteSiteListener.getInstance(getProperties()).setupServerProtocol(serverProtocol);
HttpRemoteSiteListener.getInstance(nifiProperties).setupServerProtocol(serverProtocol);
serverProtocol.handshake(peer);
return serverProtocol;
}
HttpFlowFileServerProtocol getHttpFlowFileServerProtocol(final VersionNegotiator versionNegotiator) {
return new StandardHttpFlowFileServerProtocol(versionNegotiator, getProperties());
return new StandardHttpFlowFileServerProtocol(versionNegotiator, nifiProperties);
}
private Peer constructPeer(final HttpServletRequest req, final InputStream inputStream,
final OutputStream outputStream, final String portId, final String transactionId) {
final OutputStream outputStream, final String portId, final String transactionId) {
final String clientHostName = req.getRemoteHost();
final int clientPort = req.getRemotePort();
@ -377,17 +382,18 @@ public class DataTransferResource extends ApplicationResource {
value = "Commit or cancel the specified transaction",
response = TransactionResultEntity.class,
authorizations = {
@Authorization(value = "Write - /data-transfer/output-ports/{uuid}", type = "")
@Authorization(value = "Write - /data-transfer/output-ports/{uuid}", type = "")
}
)
@ApiResponses(
value = {
@ApiResponse(code = 400, message = "NiFi was unable to complete the request because it was invalid. The request should not be retried without modification."),
@ApiResponse(code = 401, message = "Client could not be authenticated."),
@ApiResponse(code = 403, message = "Client is not authorized to make this request."),
@ApiResponse(code = 404, message = "The specified resource could not be found."),
@ApiResponse(code = 409, message = "The request was valid but NiFi was not in the appropriate state to process it. Retrying the same request later may be successful."),
@ApiResponse(code = 503, message = "NiFi instance is not ready for serving request, or temporarily overloaded. Retrying the same request later may be successful"),}
@ApiResponse(code = 400, message = "NiFi was unable to complete the request because it was invalid. The request should not be retried without modification."),
@ApiResponse(code = 401, message = "Client could not be authenticated."),
@ApiResponse(code = 403, message = "Client is not authorized to make this request."),
@ApiResponse(code = 404, message = "The specified resource could not be found."),
@ApiResponse(code = 409, message = "The request was valid but NiFi was not in the appropriate state to process it. Retrying the same request later may be successful."),
@ApiResponse(code = 503, message = "NiFi instance is not ready for serving request, or temporarily overloaded. Retrying the same request later may be successful"),
}
)
public Response commitOutputPortTransaction(
@ApiParam(
@ -474,6 +480,7 @@ public class DataTransferResource extends ApplicationResource {
return clusterContext(noCache(setCommonHeaders(Response.ok(entity), transportProtocolVersion, transactionManager))).build();
}
@DELETE
@Consumes(MediaType.APPLICATION_OCTET_STREAM)
@Produces(MediaType.APPLICATION_JSON)
@ -482,17 +489,18 @@ public class DataTransferResource extends ApplicationResource {
value = "Commit or cancel the specified transaction",
response = TransactionResultEntity.class,
authorizations = {
@Authorization(value = "Write - /data-transfer/input-ports/{uuid}", type = "")
@Authorization(value = "Write - /data-transfer/input-ports/{uuid}", type = "")
}
)
@ApiResponses(
value = {
@ApiResponse(code = 400, message = "NiFi was unable to complete the request because it was invalid. The request should not be retried without modification."),
@ApiResponse(code = 401, message = "Client could not be authenticated."),
@ApiResponse(code = 403, message = "Client is not authorized to make this request."),
@ApiResponse(code = 404, message = "The specified resource could not be found."),
@ApiResponse(code = 409, message = "The request was valid but NiFi was not in the appropriate state to process it. Retrying the same request later may be successful."),
@ApiResponse(code = 503, message = "NiFi instance is not ready for serving request, or temporarily overloaded. Retrying the same request later may be successful"),}
@ApiResponse(code = 400, message = "NiFi was unable to complete the request because it was invalid. The request should not be retried without modification."),
@ApiResponse(code = 401, message = "Client could not be authenticated."),
@ApiResponse(code = 403, message = "Client is not authorized to make this request."),
@ApiResponse(code = 404, message = "The specified resource could not be found."),
@ApiResponse(code = 409, message = "The request was valid but NiFi was not in the appropriate state to process it. Retrying the same request later may be successful."),
@ApiResponse(code = 503, message = "NiFi instance is not ready for serving request, or temporarily overloaded. Retrying the same request later may be successful"),
}
)
public Response commitInputPortTransaction(
@ApiParam(
@ -590,6 +598,7 @@ public class DataTransferResource extends ApplicationResource {
return Response.ok(entity).build();
}
@GET
@Consumes(MediaType.WILDCARD)
@Produces(MediaType.APPLICATION_OCTET_STREAM)
@ -598,18 +607,19 @@ public class DataTransferResource extends ApplicationResource {
value = "Transfer flow files from the output port",
response = StreamingOutput.class,
authorizations = {
@Authorization(value = "Write - /data-transfer/output-ports/{uuid}", type = "")
@Authorization(value = "Write - /data-transfer/output-ports/{uuid}", type = "")
}
)
@ApiResponses(
value = {
@ApiResponse(code = 200, message = "There is no flow file to return."),
@ApiResponse(code = 400, message = "NiFi was unable to complete the request because it was invalid. The request should not be retried without modification."),
@ApiResponse(code = 401, message = "Client could not be authenticated."),
@ApiResponse(code = 403, message = "Client is not authorized to make this request."),
@ApiResponse(code = 404, message = "The specified resource could not be found."),
@ApiResponse(code = 409, message = "The request was valid but NiFi was not in the appropriate state to process it. Retrying the same request later may be successful."),
@ApiResponse(code = 503, message = "NiFi instance is not ready for serving request, or temporarily overloaded. Retrying the same request later may be successful"),}
@ApiResponse(code = 200, message = "There is no flow file to return."),
@ApiResponse(code = 400, message = "NiFi was unable to complete the request because it was invalid. The request should not be retried without modification."),
@ApiResponse(code = 401, message = "Client could not be authenticated."),
@ApiResponse(code = 403, message = "Client is not authorized to make this request."),
@ApiResponse(code = 404, message = "The specified resource could not be found."),
@ApiResponse(code = 409, message = "The request was valid but NiFi was not in the appropriate state to process it. Retrying the same request later may be successful."),
@ApiResponse(code = 503, message = "NiFi instance is not ready for serving request, or temporarily overloaded. Retrying the same request later may be successful"),
}
)
public Response transferFlowFiles(
@ApiParam(
@ -681,16 +691,16 @@ public class DataTransferResource extends ApplicationResource {
value = "Extend transaction TTL",
response = TransactionResultEntity.class,
authorizations = {
@Authorization(value = "Write - /data-transfer/input-ports/{uuid}", type = "")
@Authorization(value = "Write - /data-transfer/input-ports/{uuid}", type = "")
}
)
@ApiResponses(
value = {
@ApiResponse(code = 400, message = "NiFi was unable to complete the request because it was invalid. The request should not be retried without modification."),
@ApiResponse(code = 401, message = "Client could not be authenticated."),
@ApiResponse(code = 403, message = "Client is not authorized to make this request."),
@ApiResponse(code = 404, message = "The specified resource could not be found."),
@ApiResponse(code = 409, message = "The request was valid but NiFi was not in the appropriate state to process it. Retrying the same request later may be successful.")
@ApiResponse(code = 400, message = "NiFi was unable to complete the request because it was invalid. The request should not be retried without modification."),
@ApiResponse(code = 401, message = "Client could not be authenticated."),
@ApiResponse(code = 403, message = "Client is not authorized to make this request."),
@ApiResponse(code = 404, message = "The specified resource could not be found."),
@ApiResponse(code = 409, message = "The request was valid but NiFi was not in the appropriate state to process it. Retrying the same request later may be successful.")
}
)
public Response extendInputPortTransactionTTL(
@ -716,17 +726,18 @@ public class DataTransferResource extends ApplicationResource {
value = "Extend transaction TTL",
response = TransactionResultEntity.class,
authorizations = {
@Authorization(value = "Write - /data-transfer/output-ports/{uuid}", type = "")
@Authorization(value = "Write - /data-transfer/output-ports/{uuid}", type = "")
}
)
@ApiResponses(
value = {
@ApiResponse(code = 400, message = "NiFi was unable to complete the request because it was invalid. The request should not be retried without modification."),
@ApiResponse(code = 401, message = "Client could not be authenticated."),
@ApiResponse(code = 403, message = "Client is not authorized to make this request."),
@ApiResponse(code = 404, message = "The specified resource could not be found."),
@ApiResponse(code = 409, message = "The request was valid but NiFi was not in the appropriate state to process it. Retrying the same request later may be successful."),
@ApiResponse(code = 503, message = "NiFi instance is not ready for serving request, or temporarily overloaded. Retrying the same request later may be successful"),}
@ApiResponse(code = 400, message = "NiFi was unable to complete the request because it was invalid. The request should not be retried without modification."),
@ApiResponse(code = 401, message = "Client could not be authenticated."),
@ApiResponse(code = 403, message = "Client is not authorized to make this request."),
@ApiResponse(code = 404, message = "The specified resource could not be found."),
@ApiResponse(code = 409, message = "The request was valid but NiFi was not in the appropriate state to process it. Retrying the same request later may be successful."),
@ApiResponse(code = 503, message = "NiFi instance is not ready for serving request, or temporarily overloaded. Retrying the same request later may be successful"),
}
)
public Response extendOutputPortTransactionTTL(
@PathParam("portId") String portId,
@ -789,7 +800,6 @@ public class DataTransferResource extends ApplicationResource {
}
private class ValidateRequestResult {
private Integer transportProtocolVersion;
private Response errResponse;
}
@ -820,7 +830,9 @@ public class DataTransferResource extends ApplicationResource {
return result;
}
// setters
public void setAuthorizer(Authorizer authorizer) {
this.authorizer = authorizer;
}

7
nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/api/SiteToSiteResource.java
View File

@ -16,6 +16,10 @@
*/
package org.apache.nifi.web.api;
import java.net.InetAddress;
import java.net.UnknownHostException;
import com.wordnik.swagger.annotations.Api;
import com.wordnik.swagger.annotations.ApiOperation;
import com.wordnik.swagger.annotations.ApiResponse;
@ -56,8 +60,6 @@ import javax.ws.rs.Produces;
import javax.ws.rs.core.Context;
import javax.ws.rs.core.MediaType;
import javax.ws.rs.core.Response;
import java.net.InetAddress;
import java.net.UnknownHostException;
import java.util.ArrayList;
import java.util.List;
import java.util.Set;
@ -171,6 +173,7 @@ public class SiteToSiteResource extends ApplicationResource {
@Path("/peers")
@Consumes(MediaType.WILDCARD)
@Produces({MediaType.APPLICATION_JSON, MediaType.APPLICATION_XML})
// TODO: @PreAuthorize("hasRole('ROLE_NIFI')")
@ApiOperation(
value = "Returns the available Peers and its status of this NiFi",
response = PeersEntity.class,

1
nifi-nar-bundles/nifi-framework-bundle/nifi-framework/pom.xml
View File

@ -40,6 +40,7 @@
<module>nifi-resources</module>
<module>nifi-documentation</module>
<module>nifi-authorizer</module>
<module>nifi-properties-loader</module>
</modules>
<dependencies>
<dependency>

5
nifi-nar-bundles/nifi-framework-bundle/pom.xml
View File

@ -73,6 +73,11 @@
<artifactId>nifi-framework-core</artifactId>
<version>1.0.0-SNAPSHOT</version>
</dependency>
<dependency>
<groupId>org.apache.nifi</groupId>
<artifactId>nifi-properties-loader</artifactId>
<version>1.0.0-SNAPSHOT</version>
</dependency>
<dependency>
<groupId>org.apache.nifi</groupId>
<artifactId>nifi-framework-authorization</artifactId>

1
nifi-nar-bundles/nifi-provenance-repository-bundle/nifi-volatile-provenance-repository/src/test/java/org/apache/nifi/provenance/TestVolatileProvenanceRepository.java
View File

@ -44,7 +44,6 @@ public class TestVolatileProvenanceRepository {
@Test
public void testAddAndGet() throws IOException, InterruptedException {
repo = new VolatileProvenanceRepository(NiFiProperties.createBasicNiFiProperties(null, null));
final Map<String, String> attributes = new HashMap<>();

14
nifi-toolkit/nifi-toolkit-assembly/NOTICE
View File

@ -90,5 +90,15 @@ The following binary components are provided under the Apache Software License v
(ASLv2) Jetty
The following NOTICE information applies:
Jetty Web Container
Copyright 1995-2015 Mort Bay Consulting Pty Ltd.
Jetty Web Container
Copyright 1995-2015 Mort Bay Consulting Pty Ltd.
(ASLv2) Groovy (org.codehaus.groovy:groovy-all:jar:2.4.5 - http://www.groovy-lang.org)
The following NOTICE information applies:
Groovy Language
Copyright 2003-2015 The respective authors and developers
Developers and Contributors are listed in the project POM file
and Gradle build
This product includes software developed by
The Groovy community (http://groovy.codehaus.org/).

6
nifi-toolkit/nifi-toolkit-assembly/pom.xml
View File

@ -18,7 +18,7 @@ language governing permissions and limitations under the License. -->
</parent>
<artifactId>nifi-toolkit-assembly</artifactId>
<packaging>pom</packaging>
<description>This is the assembly Apache NiFi Toolkit</description>
<description>This is the assembly for the Apache NiFi Toolkit</description>
<build>
<plugins>
<plugin>
@ -64,6 +64,10 @@ language governing permissions and limitations under the License. -->
<groupId>org.apache.nifi</groupId>
<artifactId>nifi-toolkit-tls</artifactId>
</dependency>
<dependency>
<groupId>org.apache.nifi</groupId>
<artifactId>nifi-toolkit-encrypt-config</artifactId>
</dependency>
<dependency>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-api</artifactId>

9
nifi-toolkit/nifi-toolkit-assembly/src/main/assembly/dependencies.xml
View File

@ -45,6 +45,15 @@
<outputDirectory>conf/</outputDirectory>
<fileMode>0600</fileMode>
</fileSet>
<!--<fileSet>-->
<!--<directory>${project.build.directory}/nifi-resources/src/main/resources/conf</directory>-->
<!--<outputDirectory>conf/</outputDirectory>-->
<!--<fileMode>0600</fileMode>-->
<!--<includes>-->
<!--<include>nifi.properties</include>-->
<!--<include>bootstrap.conf</include>-->
<!--</includes>-->
<!--</fileSet>-->
<fileSet>
<directory>${project.basedir}/src/main/resources/classpath</directory>
<outputDirectory>classpath/</outputDirectory>

40
nifi-toolkit/nifi-toolkit-assembly/src/main/resources/bin/encrypt-config.bat Normal file
View File

@ -0,0 +1,40 @@
@echo off
rem
rem Licensed to the Apache Software Foundation (ASF) under one or more
rem contributor license agreements. See the NOTICE file distributed with
rem this work for additional information regarding copyright ownership.
rem The ASF licenses this file to You under the Apache License, Version 2.0
rem (the "License"); you may not use this file except in compliance with
rem the License. You may obtain a copy of the License at
rem
rem http://www.apache.org/licenses/LICENSE-2.0
rem
rem Unless required by applicable law or agreed to in writing, software
rem distributed under the License is distributed on an "AS IS" BASIS,
rem WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
rem See the License for the specific language governing permissions and
rem limitations under the License.
rem
rem Use JAVA_HOME if it's set; otherwise, just use java
if "%JAVA_HOME%" == "" goto noJavaHome
if not exist "%JAVA_HOME%\bin\java.exe" goto noJavaHome
set JAVA_EXE=%JAVA_HOME%\bin\java.exe
goto startConfig
:noJavaHome
echo The JAVA_HOME environment variable is not defined correctly.
echo Instead the PATH will be used to find the java executable.
echo.
set JAVA_EXE=java
goto startConfig
:startConfig
set LIB_DIR=%~dp0..\classpath;%~dp0..\lib
SET JAVA_PARAMS=-cp %LIB_DIR%\* -Xms12m -Xmx24m %JAVA_ARGS% org.apache.nifi.util.config.ConfigEncryptionTool
cmd.exe /C "%JAVA_EXE%" %JAVA_PARAMS% %*
popd

120
nifi-toolkit/nifi-toolkit-assembly/src/main/resources/bin/encrypt-config.sh Normal file
View File

@ -0,0 +1,120 @@
#!/bin/sh
#
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
#
# Script structure inspired from Apache Karaf and other Apache projects with similar startup approaches
SCRIPT_DIR=$(dirname "$0")
SCRIPT_NAME=$(basename "$0")
NIFI_TOOLKIT_HOME=$(cd "${SCRIPT_DIR}" && cd .. && pwd)
PROGNAME=$(basename "$0")
warn() {
echo "${PROGNAME}: $*"
}
die() {
warn "$*"
exit 1
}
detectOS() {
# OS specific support (must be 'true' or 'false').
cygwin=false;
aix=false;
os400=false;
darwin=false;
case "$(uname)" in
CYGWIN*)
cygwin=true
;;
AIX*)
aix=true
;;
OS400*)
os400=true
;;
Darwin)
darwin=true
;;
esac
# For AIX, set an environment variable
if ${aix}; then
export LDR_CNTRL=MAXDATA=0xB0000000@DSA
echo ${LDR_CNTRL}
fi
}
locateJava() {
# Setup the Java Virtual Machine
if $cygwin ; then
[ -n "${JAVA}" ] && JAVA=$(cygpath --unix "${JAVA}")
[ -n "${JAVA_HOME}" ] && JAVA_HOME=$(cygpath --unix "${JAVA_HOME}")
fi
if [ "x${JAVA}" = "x" ] && [ -r /etc/gentoo-release ] ; then
JAVA_HOME=$(java-config --jre-home)
fi
if [ "x${JAVA}" = "x" ]; then
if [ "x${JAVA_HOME}" != "x" ]; then
if [ ! -d "${JAVA_HOME}" ]; then
die "JAVA_HOME is not valid: ${JAVA_HOME}"
fi
JAVA="${JAVA_HOME}/bin/java"
else
warn "JAVA_HOME not set; results may vary"
JAVA=$(type java)
JAVA=$(expr "${JAVA}" : '.* \(/.*\)$')
if [ "x${JAVA}" = "x" ]; then
die "java command not found"
fi
fi
fi
}
init() {
# Determine if there is special OS handling we must perform
detectOS
# Locate the Java VM to execute
locateJava
}
run() {
LIBS="${NIFI_TOOLKIT_HOME}/lib/*"
sudo_cmd_prefix=""
if $cygwin; then
NIFI_TOOLKIT_HOME=$(cygpath --path --windows "${NIFI_TOOLKIT_HOME}")
CLASSPATH="$NIFI_TOOLKIT_HOME/classpath";$(cygpath --path --windows "${LIBS}")
else
CLASSPATH="$NIFI_TOOLKIT_HOME/classpath:${LIBS}"
fi
export JAVA_HOME="$JAVA_HOME"
export NIFI_TOOLKIT_HOME="$NIFI_TOOLKIT_HOME"
umask 0077
"${JAVA}" -cp "${CLASSPATH}" -Xms128m -Xmx256m org.apache.nifi.properties.ConfigEncryptionTool "$@"
return $?
}
init
run "$@"

225
nifi-toolkit/nifi-toolkit-encrypt-config/LICENSE Normal file
View File

@ -0,0 +1,225 @@
Apache License
Version 2.0, January 2004
http://www.apache.org/licenses/
TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
1. Definitions.
"License" shall mean the terms and conditions for use, reproduction,
and distribution as defined by Sections 1 through 9 of this document.
"Licensor" shall mean the copyright owner or entity authorized by
the copyright owner that is granting the License.
"Legal Entity" shall mean the union of the acting entity and all
other entities that control, are controlled by, or are under common
control with that entity. For the purposes of this definition,
"control" means (i) the power, direct or indirect, to cause the
direction or management of such entity, whether by contract or
otherwise, or (ii) ownership of fifty percent (50%) or more of the
outstanding shares, or (iii) beneficial ownership of such entity.
"You" (or "Your") shall mean an individual or Legal Entity
exercising permissions granted by this License.
"Source" form shall mean the preferred form for making modifications,
including but not limited to software source code, documentation
source, and configuration files.
"Object" form shall mean any form resulting from mechanical
transformation or translation of a Source form, including but
not limited to compiled object code, generated documentation,
and conversions to other media types.
"Work" shall mean the work of authorship, whether in Source or
Object form, made available under the License, as indicated by a
copyright notice that is included in or attached to the work
(an example is provided in the Appendix below).
"Derivative Works" shall mean any work, whether in Source or Object
form, that is based on (or derived from) the Work and for which the
editorial revisions, annotations, elaborations, or other modifications
represent, as a whole, an original work of authorship. For the purposes
of this License, Derivative Works shall not include works that remain
separable from, or merely link (or bind by name) to the interfaces of,
the Work and Derivative Works thereof.
"Contribution" shall mean any work of authorship, including
the original version of the Work and any modifications or additions
to that Work or Derivative Works thereof, that is intentionally
submitted to Licensor for inclusion in the Work by the copyright owner
or by an individual or Legal Entity authorized to submit on behalf of
the copyright owner. For the purposes of this definition, "submitted"
means any form of electronic, verbal, or written communication sent
to the Licensor or its representatives, including but not limited to
communication on electronic mailing lists, source code control systems,
and issue tracking systems that are managed by, or on behalf of, the
Licensor for the purpose of discussing and improving the Work, but
excluding communication that is conspicuously marked or otherwise
designated in writing by the copyright owner as "Not a Contribution."
"Contributor" shall mean Licensor and any individual or Legal Entity
on behalf of whom a Contribution has been received by Licensor and
subsequently incorporated within the Work.
2. Grant of Copyright License. Subject to the terms and conditions of
this License, each Contributor hereby grants to You a perpetual,
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
copyright license to reproduce, prepare Derivative Works of,
publicly display, publicly perform, sublicense, and distribute the
Work and such Derivative Works in Source or Object form.
3. Grant of Patent License. Subject to the terms and conditions of
this License, each Contributor hereby grants to You a perpetual,
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
(except as stated in this section) patent license to make, have made,
use, offer to sell, sell, import, and otherwise transfer the Work,
where such license applies only to those patent claims licensable
by such Contributor that are necessarily infringed by their
Contribution(s) alone or by combination of their Contribution(s)
with the Work to which such Contribution(s) was submitted. If You
institute patent litigation against any entity (including a
cross-claim or counterclaim in a lawsuit) alleging that the Work
or a Contribution incorporated within the Work constitutes direct
or contributory patent infringement, then any patent licenses
granted to You under this License for that Work shall terminate
as of the date such litigation is filed.
4. Redistribution. You may reproduce and distribute copies of the
Work or Derivative Works thereof in any medium, with or without
modifications, and in Source or Object form, provided that You
meet the following conditions:
(a) You must give any other recipients of the Work or
Derivative Works a copy of this License; and
(b) You must cause any modified files to carry prominent notices
stating that You changed the files; and
(c) You must retain, in the Source form of any Derivative Works
that You distribute, all copyright, patent, trademark, and
attribution notices from the Source form of the Work,
excluding those notices that do not pertain to any part of
the Derivative Works; and
(d) If the Work includes a "NOTICE" text file as part of its
distribution, then any Derivative Works that You distribute must
include a readable copy of the attribution notices contained
within such NOTICE file, excluding those notices that do not
pertain to any part of the Derivative Works, in at least one
of the following places: within a NOTICE text file distributed
as part of the Derivative Works; within the Source form or
documentation, if provided along with the Derivative Works; or,
within a display generated by the Derivative Works, if and
wherever such third-party notices normally appear. The contents
of the NOTICE file are for informational purposes only and
do not modify the License. You may add Your own attribution
notices within Derivative Works that You distribute, alongside
or as an addendum to the NOTICE text from the Work, provided
that such additional attribution notices cannot be construed
as modifying the License.
You may add Your own copyright statement to Your modifications and
may provide additional or different license terms and conditions
for use, reproduction, or distribution of Your modifications, or
for any such Derivative Works as a whole, provided Your use,
reproduction, and distribution of the Work otherwise complies with
the conditions stated in this License.
5. Submission of Contributions. Unless You explicitly state otherwise,
any Contribution intentionally submitted for inclusion in the Work
by You to the Licensor shall be under the terms and conditions of
this License, without any additional terms or conditions.
Notwithstanding the above, nothing herein shall supersede or modify
the terms of any separate license agreement you may have executed
with Licensor regarding such Contributions.
6. Trademarks. This License does not grant permission to use the trade
names, trademarks, service marks, or product names of the Licensor,
except as required for reasonable and customary use in describing the
origin of the Work and reproducing the content of the NOTICE file.
7. Disclaimer of Warranty. Unless required by applicable law or
agreed to in writing, Licensor provides the Work (and each
Contributor provides its Contributions) on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
implied, including, without limitation, any warranties or conditions
of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
PARTICULAR PURPOSE. You are solely responsible for determining the
appropriateness of using or redistributing the Work and assume any
risks associated with Your exercise of permissions under this License.
8. Limitation of Liability. In no event and under no legal theory,
whether in tort (including negligence), contract, or otherwise,
unless required by applicable law (such as deliberate and grossly
negligent acts) or agreed to in writing, shall any Contributor be
liable to You for damages, including any direct, indirect, special,
incidental, or consequential damages of any character arising as a
result of this License or out of the use or inability to use the
Work (including but not limited to damages for loss of goodwill,
work stoppage, computer failure or malfunction, or any and all
other commercial damages or losses), even if such Contributor
has been advised of the possibility of such damages.
9. Accepting Warranty or Additional Liability. While redistributing
the Work or Derivative Works thereof, You may choose to offer,
and charge a fee for, acceptance of support, warranty, indemnity,
or other liability obligations and/or rights consistent with this
License. However, in accepting such obligations, You may act only
on Your own behalf and on Your sole responsibility, not on behalf
of any other Contributor, and only if You agree to indemnify,
defend, and hold each Contributor harmless for any liability
incurred by, or claims asserted against, such Contributor by reason
of your accepting any such warranty or additional liability.
END OF TERMS AND CONDITIONS
APPENDIX: How to apply the Apache License to your work.
To apply the Apache License to your work, attach the following
boilerplate notice, with the fields enclosed by brackets "[]"
replaced with your own identifying information. (Don't include
the brackets!) The text should be enclosed in the appropriate
comment syntax for the file format. We also recommend that a
file or class name and description of purpose be included on the
same "printed page" as the copyright notice for easier
identification within third-party archives.
Copyright [yyyy] [name of copyright owner]
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
This product bundles source from 'AbstractingTheJavaConsole'. The source is available under an MIT LICENSE.
Copyright (C) 2010 McDowell
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in
all copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
THE SOFTWARE.

162
nifi-toolkit/nifi-toolkit-encrypt-config/pom.xml Normal file
View File

@ -0,0 +1,162 @@
<?xml version="1.0" encoding="UTF-8"?>
<!--
Licensed to the Apache Software Foundation (ASF) under one or more
contributor license agreements. See the NOTICE file distributed with
this work for additional information regarding copyright ownership.
The ASF licenses this file to You under the Apache License, Version 2.0
(the "License"); you may not use this file except in compliance with
the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
-->
<project xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<parent>
<groupId>org.apache.nifi</groupId>
<artifactId>nifi-toolkit</artifactId>
<version>1.0.0-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
<artifactId>nifi-toolkit-encrypt-config</artifactId>
<description>Tool to encrypt sensitive configuration values</description>
<dependencies>
<dependency>
<groupId>org.apache.nifi</groupId>
<artifactId>nifi-properties</artifactId>
</dependency>
<dependency>
<groupId>org.apache.nifi</groupId>
<artifactId>nifi-properties-loader</artifactId>
</dependency>
<dependency>
<groupId>org.apache.nifi</groupId>
<artifactId>nifi-toolkit-tls</artifactId>
<exclusions>
<exclusion>
<groupId>commons-logging</groupId>
<artifactId>commons-logging</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>commons-cli</groupId>
<artifactId>commons-cli</artifactId>
</dependency>
<!--<dependency>-->
<!--<groupId>org.codehaus.groovy</groupId>-->
<!--<artifactId>groovy-all</artifactId>-->
<!--<scope>compile</scope>-->
<!--</dependency>-->
<dependency>
<groupId>ch.qos.logback</groupId>
<artifactId>logback-classic</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>com.github.stefanbirkner</groupId>
<artifactId>system-rules</artifactId>
<version>1.16.0</version>
<scope>test</scope>
</dependency>
</dependencies>
<build>
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-compiler-plugin</artifactId>
<executions>
<execution>
<goals>
<goal>compile</goal>
<goal>testCompile</goal>
</goals>
<configuration>
<compilerId>groovy-eclipse-compiler</compilerId>
</configuration>
</execution>
</executions>
<configuration>
<source>1.8</source>
<target>1.8</target>
</configuration>
<dependencies>
<dependency>
<groupId>org.codehaus.groovy</groupId>
<artifactId>groovy-eclipse-compiler</artifactId>
<version>2.9.2-01</version>
</dependency>
<dependency>
<groupId>org.codehaus.groovy</groupId>
<artifactId>groovy-eclipse-batch</artifactId>
<version>2.4.3-01</version>
</dependency>
</dependencies>
</plugin>
<!--<plugin>-->
<!--<groupId>org.codehaus.groovy</groupId>-->
<!--<artifactId>groovy-eclipse-compiler</artifactId>-->
<!--<version>2.9.2-01</version>-->
<!--<extensions>true</extensions>-->
<!--</plugin>-->
<!--<plugin>-->
<!--<groupId>org.apache.maven.plugins</groupId>-->
<!--<artifactId>maven-shade-plugin</artifactId>-->
<!--<version>2.1</version>-->
<!--<executions>-->
<!--<execution>-->
<!--<phase>package</phase>-->
<!--<goals>-->
<!--<goal>shade</goal>-->
<!--</goals>-->
<!--</execution>-->
<!--</executions>-->
<!--<configuration>-->
<!--<transformers>-->
<!--<transformer implementation="org.apache.maven.plugins.shade.resource.ManifestResourceTransformer">-->
<!--<mainClass>ConfigEncryptionTool</mainClass>-->
<!--</transformer>-->
<!--</transformers>-->
<!--</configuration>-->
<!--</plugin>-->
<plugin>
<groupId>org.codehaus.mojo</groupId>
<artifactId>build-helper-maven-plugin</artifactId>
<version>1.5</version>
<executions>
<execution>
<id>add-source</id>
<phase>generate-sources</phase>
<goals>
<goal>add-source</goal>
</goals>
<configuration>
<sources>
<source>src/main/groovy</source>
</sources>
</configuration>
</execution>
<execution>
<id>add-test-source</id>
<phase>generate-test-sources</phase>
<goals>
<goal>add-test-source</goal>
</goals>
<configuration>
<sources>
<source>src/test/groovy</source>
</sources>
</configuration>
</execution>
</executions>
</plugin>
</plugins>
</build>
</project>

578
nifi-toolkit/nifi-toolkit-encrypt-config/src/main/groovy/org/apache/nifi/properties/ConfigEncryptionTool.groovy Normal file
View File

@ -0,0 +1,578 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.nifi.properties
import groovy.io.GroovyPrintWriter
import org.apache.commons.cli.CommandLine
import org.apache.commons.cli.CommandLineParser
import org.apache.commons.cli.DefaultParser
import org.apache.commons.cli.HelpFormatter
import org.apache.commons.cli.Options
import org.apache.commons.cli.ParseException
import org.apache.commons.codec.binary.Hex
import org.apache.nifi.toolkit.tls.commandLine.CommandLineParseException
import org.apache.nifi.toolkit.tls.commandLine.ExitCode
import org.apache.nifi.util.NiFiProperties
import org.apache.nifi.util.console.TextDevice
import org.apache.nifi.util.console.TextDevices
import org.bouncycastle.crypto.generators.SCrypt
import org.bouncycastle.jce.provider.BouncyCastleProvider
import org.slf4j.Logger
import org.slf4j.LoggerFactory
import javax.crypto.Cipher
import java.nio.charset.StandardCharsets
import java.security.KeyException
import java.security.Security
class ConfigEncryptionTool {
private static final Logger logger = LoggerFactory.getLogger(ConfigEncryptionTool.class)
public String bootstrapConfPath
public String niFiPropertiesPath
public String outputNiFiPropertiesPath
public String loginIdentityProvidersPath
private String keyHex
private String password
private NiFiProperties niFiProperties
private boolean usingPassword = true
private boolean isVerbose = false
private static final String HELP_ARG = "help"
private static final String VERBOSE_ARG = "verbose"
private static final String BOOTSTRAP_CONF_ARG = "bootstrapConf"
private static final String NIFI_PROPERTIES_ARG = "niFiProperties"
private static final String OUTPUT_NIFI_PROPERTIES_ARG = "outputNiFiProperties"
private static final String KEY_ARG = "key"
private static final String PASSWORD_ARG = "password"
private static final String USE_KEY_ARG = "useRawKey"
private static final int MIN_PASSWORD_LENGTH = 12
// Strong parameters as of 12 Aug 2016
private static final int SCRYPT_N = 2**16
private static final int SCRYPT_R = 8
private static final int SCRYPT_P = 1
private static
final String BOOTSTRAP_KEY_COMMENT = "# Master key in hexadecimal format for encrypted sensitive configuration values"
private static final String BOOTSTRAP_KEY_PREFIX = "nifi.bootstrap.sensitive.key="
private static final String JAVA_HOME = "JAVA_HOME"
private static final String NIFI_TOOLKIT_HOME = "NIFI_TOOLKIT_HOME"
private static final String SEP = System.lineSeparator()
private static final String FOOTER = buildFooter()
private static
final String DEFAULT_DESCRIPTION = "This tool reads from a nifi.properties file with plain sensitive configuration values, prompts the user for a master key, and encrypts each value. It will replace the plain value with the protected value in the same file (or write to a new nifi.properties file if specified)."
private static String buildHeader(String description = DEFAULT_DESCRIPTION) {
"${SEP}${description}${SEP * 2}"
}
private static String buildFooter() {
"${SEP}Java home: ${System.getenv(JAVA_HOME)}${SEP}NiFi Toolkit home: ${System.getenv(NIFI_TOOLKIT_HOME)}"
}
private final Options options;
private final String header;
public ConfigEncryptionTool() {
this(DEFAULT_DESCRIPTION)
}
public ConfigEncryptionTool(String description) {
this.header = buildHeader(description)
this.options = new Options()
options.addOption("h", HELP_ARG, false, "Prints this usage message")
options.addOption("v", VERBOSE_ARG, false, "Sets verbose mode (default false)")
options.addOption("n", NIFI_PROPERTIES_ARG, true, "The nifi.properties file containing unprotected config values (will be overwritten)")
options.addOption("b", BOOTSTRAP_CONF_ARG, true, "The bootstrap.conf file to persist master key")
options.addOption("o", OUTPUT_NIFI_PROPERTIES_ARG, true, "The destination nifi.properties file containing protected config values (will not modify input nifi.properties)")
options.addOption("k", KEY_ARG, true, "The raw hexadecimal key to use to encrypt the sensitive properties")
options.addOption("p", PASSWORD_ARG, true, "The password from which to derive the key to use to encrypt the sensitive properties")
options.addOption("r", USE_KEY_ARG, false, "If provided, the secure console will prompt for the raw key value in hexadecimal form")
}
/**
* Prints the usage message and available arguments for this tool (along with a specific error message if provided).
*
* @param errorMessage the optional error message
*/
public void printUsage(String errorMessage) {
if (errorMessage) {
System.out.println(errorMessage)
System.out.println()
}
HelpFormatter helpFormatter = new HelpFormatter()
helpFormatter.setWidth(160)
helpFormatter.printHelp(ConfigEncryptionTool.class.getCanonicalName(), header, options, FOOTER, true)
}
protected void printUsageAndThrow(String errorMessage, ExitCode exitCode) throws CommandLineParseException {
printUsage(errorMessage);
throw new CommandLineParseException(errorMessage, exitCode);
}
protected CommandLine parse(String[] args) throws CommandLineParseException {
CommandLineParser parser = new DefaultParser()
CommandLine commandLine
try {
commandLine = parser.parse(options, args)
if (commandLine.hasOption(HELP_ARG)) {
printUsageAndThrow(null, ExitCode.HELP)
}
isVerbose = commandLine.hasOption(VERBOSE_ARG)
bootstrapConfPath = commandLine.getOptionValue(BOOTSTRAP_CONF_ARG, determineDefaultBootstrapConfPath())
niFiPropertiesPath = commandLine.getOptionValue(NIFI_PROPERTIES_ARG, determineDefaultNiFiPropertiesPath())
outputNiFiPropertiesPath = commandLine.getOptionValue(OUTPUT_NIFI_PROPERTIES_ARG, niFiPropertiesPath)
if (niFiPropertiesPath == outputNiFiPropertiesPath) {
// TODO: Add confirmation pause and provide -y flag to offer no-interaction mode?
logger.warn("The source nifi.properties and destination nifi.properties are identical [${outputNiFiPropertiesPath}] so the original will be overwritten")
}
if (commandLine.hasOption(PASSWORD_ARG)) {
usingPassword = true
if (commandLine.hasOption(KEY_ARG)) {
printUsageAndThrow("Only one of ${PASSWORD_ARG} and ${KEY_ARG} can be used", ExitCode.INVALID_ARGS)
} else {
password = commandLine.getOptionValue(PASSWORD_ARG)
}
} else {
keyHex = commandLine.getOptionValue(KEY_ARG)
usingPassword = !keyHex
}
if (commandLine.hasOption(USE_KEY_ARG)) {
if (keyHex || password) {
logger.warn("If the key or password is provided in the arguments, '-r'/'--${USE_KEY_ARG}' is ignored")
} else {
usingPassword = false
}
}
} catch (ParseException e) {
if (isVerbose) {
logger.error("Encountered an error", e)
}
printUsageAndThrow("Error parsing command line. (" + e.getMessage() + ")", ExitCode.ERROR_PARSING_COMMAND_LINE)
}
return commandLine
}
private String getKey(TextDevice device = TextDevices.defaultTextDevice()) {
if (usingPassword) {
if (!password) {
password = readPasswordFromConsole(device)
}
keyHex = deriveKeyFromPassword(password)
password = null
usingPassword = false
return keyHex
} else {
if (!keyHex) {
keyHex = readKeyFromConsole(device)
}
return keyHex
}
}
private static String readKeyFromConsole(TextDevice textDevice) {
textDevice.printf("Enter the master key in hexadecimal format (spaces acceptable): ")
new String(textDevice.readPassword())
}
private static String readPasswordFromConsole(TextDevice textDevice) {
textDevice.printf("Enter the password: ")
new String(textDevice.readPassword())
}
/**
* Returns the key in uppercase hexadecimal format with delimiters (spaces, '-', etc.) removed. All non-hex chars are removed. If the result is not a valid length (32, 48, 64 chars depending on the JCE), an exception is thrown.
*
* @param rawKey the unprocessed key input
* @return the formatted hex string in uppercase
* @throws KeyException if the key is not a valid length after parsing
*/
private static String parseKey(String rawKey) throws KeyException {
String hexKey = rawKey.replaceAll("[^0-9a-fA-F]", "")
def validKeyLengths = getValidKeyLengths()
if (!validKeyLengths.contains(hexKey.size() * 4)) {
throw new KeyException("The key (${hexKey.size()} hex chars) must be of length ${validKeyLengths} bits (${validKeyLengths.collect { it / 4 }} hex characters)")
}
hexKey.toUpperCase()
}
/**
* Returns the list of acceptable key lengths in bits based on the current JCE policies.
*
* @return 128 , [192, 256]
*/
public static List<Integer> getValidKeyLengths() {
Cipher.getMaxAllowedKeyLength("AES") > 128 ? [128, 192, 256] : [128]
}
/**
* Loads the {@link NiFiProperties} instance from the provided file path (restoring the original value of the System property {@code nifi.properties.file.path} after loading this instance).
*
* @return the NiFiProperties instance
* @throw IOException if the nifi.properties file cannot be read
*/
private NiFiProperties loadNiFiProperties() throws IOException {
File niFiPropertiesFile
if (niFiPropertiesPath && (niFiPropertiesFile = new File(niFiPropertiesPath)).exists()) {
String oldNiFiPropertiesPath = System.getProperty(NiFiProperties.PROPERTIES_FILE_PATH)
logger.debug("Saving existing NiFiProperties file path ${oldNiFiPropertiesPath}")
System.setProperty(NiFiProperties.PROPERTIES_FILE_PATH, niFiPropertiesFile.absolutePath)
logger.debug("Temporarily set NiFiProperties file path to ${niFiPropertiesFile.absolutePath}")
NiFiProperties properties
try {
properties = NiFiPropertiesLoader.withKey(keyHex).load(niFiPropertiesFile)
logger.info("Loaded NiFiProperties instance with ${properties.size()} properties")
return properties
} catch (RuntimeException e) {
if (isVerbose) {
logger.error("Encountered an error", e)
}
throw new IOException("Cannot load NiFiProperties from [${niFiPropertiesPath}]", e)
} finally {
// Can't set a system property to null
if (oldNiFiPropertiesPath) {
System.setProperty(NiFiProperties.PROPERTIES_FILE_PATH, oldNiFiPropertiesPath)
} else {
System.clearProperty(NiFiProperties.PROPERTIES_FILE_PATH)
}
logger.debug("Restored system variable ${NiFiProperties.PROPERTIES_FILE_PATH} to ${oldNiFiPropertiesPath}")
}
} else {
printUsageAndThrow("Cannot load NiFiProperties from [${niFiPropertiesPath}]", ExitCode.ERROR_READING_NIFI_PROPERTIES)
}
}
/**
* Accepts a {@link NiFiProperties} instance, iterates over all non-empty sensitive properties which are not already marked as protected, encrypts them using the master key, and updates the property with the protected value. Additionally, adds a new sibling property {@code x.y.z.protected=aes/gcm/{128,256}} for each indicating the encryption scheme used.
*
* @param plainProperties the NiFiProperties instance containing the raw values
* @return the NiFiProperties containing protected values
*/
private NiFiProperties encryptSensitiveProperties(NiFiProperties plainProperties) {
if (!plainProperties) {
throw new IllegalArgumentException("Cannot encrypt empty NiFiProperties")
}
ProtectedNiFiProperties protectedWrapper = new ProtectedNiFiProperties(plainProperties)
List<String> sensitivePropertyKeys = protectedWrapper.getSensitivePropertyKeys()
if (sensitivePropertyKeys.isEmpty()) {
logger.info("No sensitive properties to encrypt")
return plainProperties
}
// Holder for encrypted properties and protection schemes
Properties encryptedProperties = new Properties()
AESSensitivePropertyProvider spp = new AESSensitivePropertyProvider(keyHex)
protectedWrapper.addSensitivePropertyProvider(spp)
List<String> keysToSkip = []
// Iterate over each -- encrypt and add .protected if populated
sensitivePropertyKeys.each { String key ->
if (!plainProperties.getProperty(key)) {
logger.debug("Skipping encryption of ${key} because it is empty")
} else {
String protectedValue = spp.protect(plainProperties.getProperty(key))
// Add the encrypted value
encryptedProperties.setProperty(key, protectedValue)
logger.info("Protected ${key} with ${spp.getIdentifierKey()} -> \t${protectedValue}")
// Add the protection key ("x.y.z.protected" -> "aes/gcm/{128,256}")
String protectionKey = protectedWrapper.getProtectionKey(key)
encryptedProperties.setProperty(protectionKey, spp.getIdentifierKey())
logger.info("Updated protection key ${protectionKey}")
keysToSkip << key << protectionKey
}
}
// Combine the original raw NiFiProperties and the newly-encrypted properties
// Memory-wasteful but NiFiProperties are immutable -- no setter available (unless we monkey-patch...)
Set<String> nonSensitiveKeys = plainProperties.getPropertyKeys() - keysToSkip
nonSensitiveKeys.each { String key ->
encryptedProperties.setProperty(key, plainProperties.getProperty(key))
}
NiFiProperties mergedProperties = new StandardNiFiProperties(encryptedProperties)
logger.info("Final result: ${mergedProperties.size()} keys including ${ProtectedNiFiProperties.countProtectedProperties(mergedProperties)} protected keys")
mergedProperties
}
/**
* Reads the existing {@code bootstrap.conf} file, updates it to contain the master key, and persists it back to the same location.
*
* @throw IOException if there is a problem reading or writing the bootstrap.conf file
*/
private void writeKeyToBootstrapConf() throws IOException {
File bootstrapConfFile
if (bootstrapConfPath && (bootstrapConfFile = new File(bootstrapConfPath)).exists() && bootstrapConfFile.canRead() && bootstrapConfFile.canWrite()) {
try {
List<String> lines = bootstrapConfFile.readLines()
updateBootstrapContentsWithKey(lines)
// Write the updated values back to the file
bootstrapConfFile.text = lines.join("\n")
} catch (IOException e) {
def msg = "Encountered an exception updating the bootstrap.conf file with the master key"
logger.error(msg, e)
throw e
}
} else {
throw new IOException("The bootstrap.conf file at ${bootstrapConfPath} must exist and be readable and writable by the user running this tool")
}
}
/**
* Accepts the lines of the {@code bootstrap.conf} file as a {@code List <String>} and updates or adds the key property (and associated comment).
*
* @param lines the lines of the bootstrap file
* @return the updated lines
*/
private List<String> updateBootstrapContentsWithKey(List<String> lines) {
String keyLine = "${BOOTSTRAP_KEY_PREFIX}${keyHex}"
// Try to locate the key property line
int keyLineIndex = lines.findIndexOf { it.startsWith(BOOTSTRAP_KEY_PREFIX) }
// If it was found, update inline
if (keyLineIndex != -1) {
logger.debug("The key property was detected in bootstrap.conf")
lines[keyLineIndex] = keyLine
logger.debug("The bootstrap key value was updated")
// Ensure the comment explaining the property immediately precedes it (check for edge case where key is first line)
int keyCommentLineIndex = keyLineIndex > 0 ? keyLineIndex - 1 : 0
if (lines[keyCommentLineIndex] != BOOTSTRAP_KEY_COMMENT) {
lines.add(keyCommentLineIndex, BOOTSTRAP_KEY_COMMENT)
logger.debug("A comment explaining the bootstrap key property was added")
}
} else {
// If it wasn't present originally, add the comment and key property
lines.addAll(["\n", BOOTSTRAP_KEY_COMMENT, keyLine])
logger.debug("The key property was not detected in bootstrap.conf so it was added along with a comment explaining it")
}
lines
}
/**
* Writes the contents of the {@link NiFiProperties} instance with encrypted values to the output {@code nifi.properties} file.
*
* @throw IOException if there is a problem reading or writing the nifi.properties file
*/
private void writeNiFiProperties() throws IOException {
if (!outputNiFiPropertiesPath) {
throw new IllegalArgumentException("Cannot write encrypted properties to empty nifi.properties path")
}
File outputNiFiPropertiesFile = new File(outputNiFiPropertiesPath)
if (isSafeToWrite(outputNiFiPropertiesFile)) {
try {
List<String> linesToPersist
File niFiPropertiesFile = new File(niFiPropertiesPath)
if (niFiPropertiesFile.exists() && niFiPropertiesFile.canRead()) {
// Instead of just writing the NiFiProperties instance to a properties file, this method attempts to maintain the structure of the original file and preserves comments
linesToPersist = serializeNiFiPropertiesAndPreserveFormat(niFiProperties, niFiPropertiesFile)
} else {
linesToPersist = serializeNiFiProperties(niFiProperties)
}
// Write the updated values back to the file
outputNiFiPropertiesFile.text = linesToPersist.join("\n")
} catch (IOException e) {
def msg = "Encountered an exception updating the nifi.properties file with the encrypted values"
logger.error(msg, e)
throw e
}
} else {
throw new IOException("The nifi.properties file at ${outputNiFiPropertiesPath} must be writable by the user running this tool")
}
}
private
static List<String> serializeNiFiPropertiesAndPreserveFormat(NiFiProperties niFiProperties, File originalPropertiesFile) {
List<String> lines = originalPropertiesFile.readLines()
ProtectedNiFiProperties protectedNiFiProperties = new ProtectedNiFiProperties(niFiProperties)
// Only need to replace the keys that have been protected
Map<String, String> protectedKeys = protectedNiFiProperties.getProtectedPropertyKeys()
protectedKeys.each { String key, String protectionScheme ->
int l = lines.findIndexOf { it.startsWith(key) }
if (l != -1) {
lines[l] = "${key}=${protectedNiFiProperties.getProperty(key)}"
}
// Get the index of the following line (or cap at max)
int p = l + 1 > lines.size() ? lines.size() : l + 1
String protectionLine = "${protectedNiFiProperties.getProtectionKey(key)}=${protectionScheme}"
if (p < lines.size() && lines.get(p).startsWith("${protectedNiFiProperties.getProtectionKey(key)}=")) {
lines.set(p, protectionLine)
} else {
lines.add(p, protectionLine)
}
}
lines
}
private static List<String> serializeNiFiProperties(NiFiProperties nifiProperties) {
OutputStream out = new ByteArrayOutputStream()
Writer writer = new GroovyPrintWriter(out)
// Again, waste of memory, but respecting the interface
Properties properties = new Properties()
nifiProperties.getPropertyKeys().each { String key ->
properties.setProperty(key, nifiProperties.getProperty(key))
}
properties.store(writer, null)
writer.flush()
out.toString().split("\n")
}
/**
* Helper method which returns true if it is "safe" to write to the provided file.
*
* Conditions:
* file does not exist and the parent directory is writable
* -OR-
* file exists and is writable
*
* @param fileToWrite the proposed file to be written to
* @return true if the caller can "safely" write to this file location
*/
private static boolean isSafeToWrite(File fileToWrite) {
fileToWrite && ((!fileToWrite.exists() && fileToWrite.absoluteFile.parentFile.canWrite()) || (fileToWrite.exists() && fileToWrite.canWrite()))
}
private static String determineDefaultBootstrapConfPath() {
String niFiToolkitPath = System.getenv(NIFI_TOOLKIT_HOME) ?: ""
"${niFiToolkitPath ? niFiToolkitPath + "/" : ""}conf/bootstrap.conf"
}
private static String determineDefaultNiFiPropertiesPath() {
String niFiToolkitPath = System.getenv(NIFI_TOOLKIT_HOME) ?: ""
"${niFiToolkitPath ? niFiToolkitPath + "/" : ""}conf/nifi.properties"
}
private static String deriveKeyFromPassword(String password) {
password = password?.trim()
if (!password || password.length() < MIN_PASSWORD_LENGTH) {
throw new KeyException("Cannot derive key from empty/short password -- password must be at least ${MIN_PASSWORD_LENGTH} characters")
}
// Generate a 128 bit salt
byte[] salt = generateScryptSalt()
int keyLengthInBytes = getValidKeyLengths().max() / 8
byte[] derivedKeyBytes = SCrypt.generate(password.getBytes(StandardCharsets.UTF_8), salt, SCRYPT_N, SCRYPT_R, SCRYPT_P, keyLengthInBytes)
Hex.encodeHexString(derivedKeyBytes).toUpperCase()
}
private static byte[] generateScryptSalt() {
// byte[] salt = new byte[16]
// new SecureRandom().nextBytes(salt)
// salt
/* It is not ideal to use a static salt, but the KDF operation must be deterministic
for a given password, and storing and retrieving the salt in bootstrap.conf causes
compatibility concerns
*/
"NIFI_SCRYPT_SALT".getBytes(StandardCharsets.UTF_8)
}
/**
* Runs main tool logic (parsing arguments, reading files, protecting properties, and writing key and properties out to destination files).
*
* @param args the command-line arguments
*/
public static void main(String[] args) {
Security.addProvider(new BouncyCastleProvider())
ConfigEncryptionTool tool = new ConfigEncryptionTool()
try {
try {
tool.parse(args)
tool.keyHex = tool.getKey()
if (!tool.keyHex) {
tool.printUsageAndThrow("Hex key must be provided", ExitCode.INVALID_ARGS)
}
try {
// Validate the length and format
tool.keyHex = parseKey(tool.keyHex)
} catch (KeyException e) {
if (tool.isVerbose) {
logger.error("Encountered an error", e)
}
tool.printUsageAndThrow(e.getMessage(), ExitCode.INVALID_ARGS)
}
tool.niFiProperties = tool.loadNiFiProperties()
tool.niFiProperties = tool.encryptSensitiveProperties(tool.niFiProperties)
} catch (CommandLineParseException e) {
if (e.exitCode == ExitCode.HELP) {
System.exit(ExitCode.HELP.ordinal())
}
throw e
} catch (Exception e) {
if (tool.isVerbose) {
logger.error("Encountered an error", e)
}
tool.printUsageAndThrow(e.message, ExitCode.ERROR_PARSING_COMMAND_LINE)
}
try {
// Do this as part of a transaction?
synchronized (this) {
tool.writeKeyToBootstrapConf()
tool.writeNiFiProperties()
}
} catch (Exception e) {
if (tool.isVerbose) {
logger.error("Encountered an error", e)
}
tool.printUsageAndThrow("Encountered an error writing the master key to the bootstrap.conf file and the encrypted properties to nifi.properties", ExitCode.ERROR_GENERATING_CONFIG)
}
} catch (CommandLineParseException e) {
System.exit(e.exitCode.ordinal())
}
System.exit(ExitCode.SUCCESS.ordinal())
}
}

23
nifi-toolkit/nifi-toolkit-encrypt-config/src/main/java/org/apache/nifi/properties/JavaMain.java Normal file
View File

@ -0,0 +1,23 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.nifi.properties;
public class JavaMain {
public static void main(String[] args) {
System.out.println("The Java class #main ran successfully");
}
}

73
nifi-toolkit/nifi-toolkit-encrypt-config/src/main/java/org/apache/nifi/util/console/CharacterDevice.java Normal file
View File

@ -0,0 +1,73 @@
/*
Copyright (c) 2010 McDowell
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in
all copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
THE SOFTWARE.
*/
package org.apache.nifi.util.console;
import java.io.BufferedReader;
import java.io.IOException;
import java.io.PrintWriter;
import java.io.Reader;
/**
* @{link TextDevice} implementation wrapping character streams.
*
* @author McDowell
*/
class CharacterDevice extends TextDevice {
private final BufferedReader reader;
private final PrintWriter writer;
public CharacterDevice(BufferedReader reader, PrintWriter writer) {
this.reader = reader;
this.writer = writer;
}
@Override
public CharacterDevice printf(String fmt, Object... params)
throws ConsoleException {
writer.printf(fmt, params);
return this;
}
@Override
public String readLine() throws ConsoleException {
try {
return reader.readLine();
} catch (IOException e) {
throw new IllegalStateException();
}
}
@Override
public char[] readPassword() throws ConsoleException {
return readLine().toCharArray();
}
@Override
public Reader reader() throws ConsoleException {
return reader;
}
@Override
public PrintWriter writer() throws ConsoleException {
return writer;
}
}

66
nifi-toolkit/nifi-toolkit-encrypt-config/src/main/java/org/apache/nifi/util/console/ConsoleDevice.java Normal file
View File

@ -0,0 +1,66 @@
/*
Copyright (c) 2010 McDowell
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in
all copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
THE SOFTWARE.
*/
package org.apache.nifi.util.console;
import java.io.Console;
import java.io.PrintWriter;
import java.io.Reader;
/**
* {@link TextDevice} implementation wrapping a {@link Console}.
*
* @author McDowell
*/
class ConsoleDevice extends TextDevice {
private final Console console;
public ConsoleDevice(Console console) {
this.console = console;
}
@Override
public TextDevice printf(String fmt, Object... params)
throws ConsoleException {
console.format(fmt, params);
return this;
}
@Override
public Reader reader() throws ConsoleException {
return console.reader();
}
@Override
public String readLine() throws ConsoleException {
return console.readLine();
}
@Override
public char[] readPassword() throws ConsoleException {
return console.readPassword();
}
@Override
public PrintWriter writer() throws ConsoleException {
return console.writer();
}
}

35
nifi-toolkit/nifi-toolkit-encrypt-config/src/main/java/org/apache/nifi/util/console/ConsoleException.java Normal file
View File

@ -0,0 +1,35 @@
/*
Copyright (c) 2010 McDowell
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in
all copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
THE SOFTWARE.
*/
package org.apache.nifi.util.console;
/**
* Runtime exception for handling console errors.
*
* @author McDowell
*/
public class ConsoleException extends RuntimeException {
private static final long serialVersionUID = 1L;
public ConsoleException(Throwable t) {
super(t);
}
}

43
nifi-toolkit/nifi-toolkit-encrypt-config/src/main/java/org/apache/nifi/util/console/TextDevice.java Normal file
View File

@ -0,0 +1,43 @@
/*
Copyright (c) 2010 McDowell
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in
all copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
THE SOFTWARE.
*/
package org.apache.nifi.util.console;
import java.io.PrintWriter;
import java.io.Reader;
/**
* Abstraction representing a text input/output device.
*
* @author McDowell
*/
public abstract class TextDevice {
public abstract TextDevice printf(String fmt, Object... params)
throws ConsoleException;
public abstract String readLine() throws ConsoleException;
public abstract char[] readPassword() throws ConsoleException;
public abstract Reader reader() throws ConsoleException;
public abstract PrintWriter writer() throws ConsoleException;
}

80
nifi-toolkit/nifi-toolkit-encrypt-config/src/main/java/org/apache/nifi/util/console/TextDevices.java Normal file
View File

@ -0,0 +1,80 @@
/*
Copyright (c) 2010 McDowell
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in
all copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
THE SOFTWARE.
*/
package org.apache.nifi.util.console;
import java.io.BufferedReader;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.io.OutputStream;
import java.io.PrintWriter;
/**
* Convenience class for providing {@link TextDevice} implementations.
*
* @author McDowell
*/
public final class TextDevices {
private TextDevices() {}
private static TextDevice DEFAULT = (System.console() == null) ? streamDevice(
System.in, System.out)
: new ConsoleDevice(System.console());
/**
* The default system text I/O device.
*
* @return the default device
*/
public static TextDevice defaultTextDevice() {
return DEFAULT;
}
/**
* Returns a text I/O device wrapping the given streams. The default system
* encoding is used to decode/encode data.
*
* @param in
* an input source
* @param out
* an output target
* @return a new device
*/
public static TextDevice streamDevice(InputStream in, OutputStream out) {
BufferedReader reader = new BufferedReader(new InputStreamReader(in));
PrintWriter writer = new PrintWriter(out, true);
return new CharacterDevice(reader, writer);
}
/**
* Returns a text I/O device wrapping the given streams.
*
* @param reader
* an input source
* @param writer
* an output target
* @return a new device
*/
public static TextDevice characterDevice(BufferedReader reader,
PrintWriter writer) {
return new CharacterDevice(reader, writer);
}
}

22
nifi-toolkit/nifi-toolkit-encrypt-config/src/main/resources/log4j.properties Normal file
View File

@ -0,0 +1,22 @@
#
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
log4j.rootLogger=INFO,console
log4j.appender.console=org.apache.log4j.ConsoleAppender
log4j.appender.console.layout=org.apache.log4j.PatternLayout
log4j.appender.console.layout.ConversionPattern=%d{yy/MM/dd HH:mm:ss} %p %c{2}: %m%n

Some files were not shown because too many files have changed in this diff Show More