Apache
/
nifi
mirror of https://github.com/apache/nifi.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

NIFI-13592 Removed unused nifi-runtime classes and methods (#9120) 

- Removed unused key processing methods from NiFi runtime class
- Removed unused stateless and embedded NiFi classes along with tests
This commit is contained in:
David Handermann 2024-07-29 13:05:40 -05:00 committed by GitHub
parent 60a03b90cb
commit 25299f8106
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
12 changed files with 7 additions and 1020 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

56
nifi-bootstrap/src/main/java/org/apache/nifi/bootstrap/RunStatelessNiFi.java
View File

@ -1,56 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.nifi.bootstrap;
import java.io.File;
import java.lang.reflect.Method;
import java.net.URL;
import java.net.URLClassLoader;
import java.util.ArrayList;
import java.util.List;
import java.util.Objects;
public class RunStatelessNiFi {
public static void main(final String[] args) throws Exception {
String nifi_home = System.getenv("NIFI_HOME");
if (nifi_home == null || nifi_home.equals("")) {
nifi_home = ".";
}
final List<URL> cpURLs = new ArrayList<>();
final File libDir = new File(nifi_home + "/lib");
if (libDir.exists()) {
for (final File file : Objects.requireNonNull(libDir.listFiles((dir, filename) -> filename.toLowerCase().endsWith(".jar")))) {
cpURLs.add(file.toURI().toURL());
}
}
if (cpURLs.isEmpty()) {
throw new RuntimeException("Could not find lib directory at " + libDir.getAbsolutePath());
}
final URLClassLoader rootClassLoader = new URLClassLoader(cpURLs.toArray(new URL[0]));
Thread.currentThread().setContextClassLoader(rootClassLoader);
final Class<?> programClass = Class.forName("org.apache.nifi.StatelessNiFi", true, rootClassLoader);
final Method launchMethod = programClass.getMethod("main", String[].class);
launchMethod.setAccessible(true);
launchMethod.invoke(null, (Object) args);
}
}

21
nifi-framework-bundle/nifi-framework/nifi-runtime/pom.xml
View File

@ -34,26 +34,5 @@
<groupId>org.apache.nifi</groupId>
<artifactId>nifi-nar-utils</artifactId>
</dependency>
<dependency>
<groupId>ch.qos.logback</groupId>
<artifactId>logback-classic</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>ch.qos.logback</groupId>
<artifactId>logback-core</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.apache.nifi</groupId>
<artifactId>nifi-properties-loader</artifactId>
<version>2.0.0-SNAPSHOT</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.apache.commons</groupId>
<artifactId>commons-lang3</artifactId>
<scope>test</scope>
</dependency>
</dependencies>
</project>

64
nifi-framework-bundle/nifi-framework/nifi-runtime/src/main/java/org/apache/nifi/EmbeddedNiFi.java
View File

@ -1,64 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.nifi;
import java.io.IOException;
import java.lang.reflect.InvocationTargetException;
/**
* <p>
* Starts an instance of NiFi within the <b>same JVM</b>, which can later properly be shut down.
* Intended to be used for testing purposes.</p>
*
*/
public class EmbeddedNiFi extends NiFi {
public EmbeddedNiFi(String[] args, ClassLoader rootClassLoader)
throws ClassNotFoundException, IOException, NoSuchMethodException,
InstantiationException, IllegalAccessException, IllegalArgumentException, InvocationTargetException {
super(convertArgumentsToValidatedNiFiProperties(args), rootClassLoader);
}
public EmbeddedNiFi(String[] args, ClassLoader rootClassLoader, ClassLoader bootstrapClassLoader)
throws ClassNotFoundException, IOException, NoSuchMethodException,
InstantiationException, IllegalAccessException, IllegalArgumentException, InvocationTargetException {
super(convertArgumentsToValidatedNiFiProperties(args, bootstrapClassLoader), rootClassLoader);
}
@Override
protected void initLogging() {
// do nothing when running in embedded mode
}
@Override
protected void setDefaultUncaughtExceptionHandler() {
// do nothing when running in embedded mode
}
@Override
protected void addShutdownHook() {
// do nothing when running in embedded mode
}
@Override
public void shutdown() {
super.shutdown();
}
}

175
nifi-framework-bundle/nifi-framework/nifi-runtime/src/main/java/org/apache/nifi/NiFi.java
View File

@ -34,7 +34,6 @@ import org.slf4j.bridge.SLF4JBridgeHandler;
import java.io.File;
import java.io.FileOutputStream;
import java.io.FileWriter;
import java.io.IOException;
import java.io.OutputStream;
import java.lang.reflect.InvocationTargetException;
@ -42,26 +41,15 @@ import java.lang.reflect.Method;
import java.net.MalformedURLException;
import java.net.URL;
import java.net.URLClassLoader;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.time.LocalDateTime;
import java.time.format.DateTimeFormatter;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import java.util.Random;
import java.util.Set;
import java.util.Timer;
import java.util.TimerTask;
import java.util.concurrent.Executors;
import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.ScheduledFuture;
import java.util.concurrent.ThreadFactory;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.atomic.AtomicLong;
import java.util.stream.Stream;
public class NiFi implements NiFiEntryPoint {
@ -70,7 +58,6 @@ public class NiFi implements NiFiEntryPoint {
public static final DateTimeFormatter DATE_TIME_FORMATTER = DateTimeFormatter.ofPattern("yyyy-MM-dd_HH-mm-ss");
private static final Logger LOGGER = LoggerFactory.getLogger(NiFi.class);
private static final String KEY_FILE_FLAG = "-K";
private final NiFiServer nifiServer;
private final BootstrapListener bootstrapListener;
@ -130,8 +117,6 @@ public class NiFi implements NiFiEntryPoint {
FileUtils.deleteFilesInDirectory(webWorkingDir, null, LOGGER, true, true);
FileUtils.deleteFile(webWorkingDir, LOGGER, 3);
detectTimingIssues();
// redirect JUL log events
initLogging();
@ -187,9 +172,7 @@ public class NiFi implements NiFiEntryPoint {
}
protected void setDefaultUncaughtExceptionHandler() {
Thread.setDefaultUncaughtExceptionHandler((thread, exception) -> {
LOGGER.error("An Unknown Error Occurred in Thread {}", thread, exception);
});
Thread.setDefaultUncaughtExceptionHandler((thread, exception) -> LOGGER.error("An Unknown Error Occurred in Thread {}", thread, exception));
}
protected void addShutdownHook() {
@ -269,57 +252,6 @@ public class NiFi implements NiFiEntryPoint {
LOGGER.info("Application Server shutdown completed");
}
/**
* Determine if the machine we're running on has timing issues.
*/
private void detectTimingIssues() {
final int minRequiredOccurrences = 25;
final int maxOccurrencesOutOfRange = 15;
final AtomicLong lastTriggerMillis = new AtomicLong(System.currentTimeMillis());
final ScheduledExecutorService service = Executors.newScheduledThreadPool(1, new ThreadFactory() {
private final ThreadFactory defaultFactory = Executors.defaultThreadFactory();
@Override
public Thread newThread(final Runnable runnable) {
final Thread t = defaultFactory.newThread(runnable);
t.setDaemon(true);
t.setName("Detect Timing Issues");
return t;
}
});
final AtomicInteger occurrencesOutOfRange = new AtomicInteger(0);
final AtomicInteger occurrences = new AtomicInteger(0);
final Runnable command = () -> {
final long curMillis = System.currentTimeMillis();
final long difference = curMillis - lastTriggerMillis.get();
final long millisOff = Math.abs(difference - 2000L);
occurrences.incrementAndGet();
if (millisOff > 500L) {
occurrencesOutOfRange.incrementAndGet();
}
lastTriggerMillis.set(curMillis);
};
final ScheduledFuture<?> future = service.scheduleWithFixedDelay(command, 2000L, 2000L, TimeUnit.MILLISECONDS);
final TimerTask timerTask = new TimerTask() {
@Override
public void run() {
future.cancel(true);
service.shutdownNow();
if (occurrences.get() < minRequiredOccurrences || occurrencesOutOfRange.get() > maxOccurrencesOutOfRange) {
LOGGER.warn("NiFi has detected that this box is not responding within the expected timing interval, which may cause "
+ "Processors to be scheduled erratically. Please see the NiFi documentation for more information.");
}
}
};
final Timer timer = new Timer(true);
timer.schedule(timerTask, 60000L);
}
/**
* Main entry point of the application.
*
@ -328,24 +260,24 @@ public class NiFi implements NiFiEntryPoint {
public static void main(String[] args) {
LOGGER.info("Launching NiFi...");
try {
NiFiProperties properties = convertArgumentsToValidatedNiFiProperties(args);
NiFiProperties properties = loadProperties();
new NiFi(properties);
} catch (final Throwable t) {
LOGGER.error("Failure to launch NiFi", t);
}
}
protected static NiFiProperties convertArgumentsToValidatedNiFiProperties(String[] args) {
return convertArgumentsToValidatedNiFiProperties(args, createBootstrapClassLoader());
protected static NiFiProperties loadProperties() {
return loadProperties(createBootstrapClassLoader());
}
protected static NiFiProperties convertArgumentsToValidatedNiFiProperties(String[] args, final ClassLoader bootstrapClassLoader) {
NiFiProperties properties = initializeProperties(args, bootstrapClassLoader);
protected static NiFiProperties loadProperties(final ClassLoader bootstrapClassLoader) {
NiFiProperties properties = initializeProperties(bootstrapClassLoader);
properties.validate();
return properties;
}
private static NiFiProperties initializeProperties(final String[] args, final ClassLoader boostrapLoader) {
private static NiFiProperties initializeProperties(final ClassLoader boostrapLoader) {
final ClassLoader contextClassLoader = Thread.currentThread().getContextClassLoader();
Thread.currentThread().setContextClassLoader(boostrapLoader);
@ -369,97 +301,4 @@ public class NiFi implements NiFiEntryPoint {
Thread.currentThread().setContextClassLoader(contextClassLoader);
}
}
private static String loadFormattedKey(String[] args) {
String key = null;
List<String> parsedArgs = parseArgs(args);
// Check if args contain protection key
if (parsedArgs.contains(KEY_FILE_FLAG)) {
key = getKeyFromKeyFileAndPrune(parsedArgs);
// Format the key (check hex validity and remove spaces)
key = formatHexKey(key);
}
if (null == key) {
return "";
} else if (!isHexKeyValid(key)) {
throw new IllegalArgumentException("The key was not provided in valid hex format and of the correct length");
} else {
return key;
}
}
private static String getKeyFromKeyFileAndPrune(List<String> parsedArgs) {
String key = null;
LOGGER.debug("The bootstrap process provided the {} flag", KEY_FILE_FLAG);
int i = parsedArgs.indexOf(KEY_FILE_FLAG);
if (parsedArgs.size() <= i + 1) {
LOGGER.error("The bootstrap process passed the {} flag without a filename", KEY_FILE_FLAG);
throw new IllegalArgumentException("The bootstrap process provided the " + KEY_FILE_FLAG + " flag but no key");
}
try {
String passwordfilePath = parsedArgs.get(i + 1);
// Slurp in the contents of the file:
byte[] encoded = Files.readAllBytes(Paths.get(passwordfilePath));
key = new String(encoded, StandardCharsets.UTF_8);
if (0 == key.length())
throw new IllegalArgumentException("Key in keyfile " + passwordfilePath + " yielded an empty key");
LOGGER.debug("Overwriting temporary bootstrap key file [{}]", passwordfilePath);
// Overwrite the contents of the file (to avoid littering file system
// unlinked with key material):
File passwordFile = new File(passwordfilePath);
FileWriter overwriter = new FileWriter(passwordFile, false);
// Construe a random pad:
Random random = new Random();
StringBuffer sb = new StringBuffer();
// Note on correctness: this pad is longer, but equally sufficient.
while (sb.length() < encoded.length) {
sb.append(Integer.toHexString(random.nextInt()));
}
String pad = sb.toString();
overwriter.write(pad);
overwriter.close();
LOGGER.debug("Removing temporary bootstrap key file [{}]", passwordfilePath);
passwordFile.delete();
} catch (IOException e) {
LOGGER.error("Caught IOException while retrieving the {} -passed keyfile; aborting", KEY_FILE_FLAG, e);
System.exit(1);
}
return key;
}
private static List<String> parseArgs(String[] args) {
List<String> parsedArgs = new ArrayList<>(Arrays.asList(args));
for (int i = 0; i < parsedArgs.size(); i++) {
if (parsedArgs.get(i).startsWith(KEY_FILE_FLAG + " ")) {
String[] split = parsedArgs.get(i).split(" ", 2);
parsedArgs.set(i, split[0]);
parsedArgs.add(i + 1, split[1]);
break;
}
}
return parsedArgs;
}
private static String formatHexKey(String input) {
if (input == null || input.trim().isEmpty()) {
return "";
}
return input.replaceAll("[^0-9a-fA-F]", "").toLowerCase();
}
private static boolean isHexKeyValid(String key) {
if (key == null || key.trim().isEmpty()) {
return false;
}
// Key length is in "nibbles" (i.e. one hex char = 4 bits)
return Arrays.asList(128, 196, 256).contains(key.length() * 4) && key.matches("^[0-9a-fA-F]*$");
}
}

115
nifi-framework-bundle/nifi-framework/nifi-runtime/src/main/java/org/apache/nifi/StatelessNiFi.java
View File

@ -1,115 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.nifi;
import org.apache.nifi.nar.NarUnpacker;
import org.apache.nifi.nar.NarUnpackMode;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.File;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.lang.reflect.InvocationTargetException;
import java.lang.reflect.Method;
import java.net.MalformedURLException;
import java.net.URL;
import java.net.URLClassLoader;
import java.util.ArrayList;
import java.util.List;
import java.util.Objects;
import java.util.concurrent.TimeUnit;
public class StatelessNiFi {
private static final Logger logger = LoggerFactory.getLogger(StatelessNiFi.class);
public static final String PROGRAM_CLASS_NAME = "org.apache.nifi.stateless.runtimes.Program";
public static final String EXTRACT_NARS = "ExtractNars";
public static void main(final String[] args) throws IOException, ClassNotFoundException, NoSuchMethodException, InvocationTargetException, IllegalAccessException {
String nifi_home = System.getenv("NIFI_HOME");
if (nifi_home == null || nifi_home.equals("")) {
nifi_home = ".";
}
final File libDir = new File(nifi_home + "/lib");
final File narWorkingDirectory = new File(nifi_home + "/work/stateless-nars");
if (args.length >= 1 && args[0].equals(EXTRACT_NARS)) {
if (!libDir.exists()) {
System.out.println("Specified lib directory <" + libDir + "> does not exist");
return;
}
final File[] narFiles = libDir.listFiles(file -> file.getName().endsWith(".nar"));
if (narFiles == null) {
System.out.println("Could not obtain listing of lib directory <" + libDir + ">");
return;
}
if (!narWorkingDirectory.exists() && !narWorkingDirectory.mkdirs()) {
throw new IOException("Could not create NAR working directory <" + narWorkingDirectory + ">");
}
logger.info("Unpacking {} NARs", narFiles.length);
final long startUnpack = System.nanoTime();
for (final File narFile : narFiles) {
NarUnpacker.unpackNar(narFile, narWorkingDirectory, false, NarUnpackMode.UNPACK_TO_UBER_JAR);
}
final long millis = TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - startUnpack);
logger.info("Finished unpacking {} NARs in {} millis", narFiles.length, millis);
System.exit(0);
}
File frameworkWorkingDirectory;
try {
frameworkWorkingDirectory = Objects.requireNonNull(narWorkingDirectory.listFiles(file -> file.getName().startsWith("nifi-framework")))[0];
} catch (Exception ex) {
throw new FileNotFoundException("Could not find core stateless dependencies in the working directory <" + narWorkingDirectory + ">");
}
final File bundledDependenciesDir = new File(frameworkWorkingDirectory, NarUnpacker.BUNDLED_DEPENDENCIES_DIRECTORY);
final File[] jarFiles = bundledDependenciesDir.listFiles();
if (jarFiles == null) {
throw new IOException("Could not obtain listing of NiFi-Framework NAR's bundled dependencies in working directory <" + bundledDependenciesDir + ">");
}
final URL[] jarUrls = toURLs(jarFiles);
final ClassLoader rootClassLoader = Thread.currentThread().getContextClassLoader();
final URLClassLoader frameworkClassLoader = new URLClassLoader(jarUrls, rootClassLoader);
Thread.currentThread().setContextClassLoader(frameworkClassLoader);
final Class<?> programClass = Class.forName(PROGRAM_CLASS_NAME, true, frameworkClassLoader);
final Method launchMethod = programClass.getMethod("launch", String[].class, ClassLoader.class, File.class);
launchMethod.setAccessible(true);
launchMethod.invoke(null, args, rootClassLoader, narWorkingDirectory);
}
private static URL[] toURLs(final File[] files) throws MalformedURLException {
final List<URL> urls = new ArrayList<>();
for (final File file : files) {
urls.add(file.toURI().toURL());
}
return urls.toArray(new URL[0]);
}
}

39
nifi-framework-bundle/nifi-framework/nifi-runtime/src/main/java/org/apache/nifi/headless/FlowEnrichmentException.java
View File

@ -1,39 +0,0 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
* <p>
* http://www.apache.org/licenses/LICENSE-2.0
* <p>
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.nifi.headless;
public class FlowEnrichmentException extends Exception {
public FlowEnrichmentException() {
super();
}
public FlowEnrichmentException(String message) {
super(message);
}
public FlowEnrichmentException(String message, Throwable cause) {
super(message, cause);
}
public FlowEnrichmentException(Throwable cause) {
super(cause);
}
protected FlowEnrichmentException(String message, Throwable cause, boolean enableSuppression, boolean writableStackTrace) {
super(message, cause, enableSuppression, writableStackTrace);
}
}

41
nifi-framework-bundle/nifi-framework/nifi-runtime/src/test/java/org/apache/nifi/ListAppender.java
View File

@ -1,41 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.nifi;
import ch.qos.logback.classic.spi.LoggingEvent;
import ch.qos.logback.core.AppenderBase;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
public class ListAppender extends AppenderBase<LoggingEvent> {
private static final List<LoggingEvent> LOGGING_EVENTS = Collections.synchronizedList(new ArrayList<>());
public static List<LoggingEvent> getLoggingEvents() {
return LOGGING_EVENTS;
}
public static void clear() {
LOGGING_EVENTS.clear();
}
@Override
protected void append(final LoggingEvent loggingEvent) {
LOGGING_EVENTS.add(loggingEvent);
}
}

116
nifi-framework-bundle/nifi-framework/nifi-runtime/src/test/java/org/apache/nifi/NiFiTest.java
View File

@ -1,116 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.nifi;
import ch.qos.logback.classic.spi.LoggingEvent;
import org.apache.commons.lang3.StringUtils;
import org.apache.nifi.util.NiFiProperties;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import java.io.File;
import java.io.FileWriter;
import java.io.IOException;
import java.io.PrintWriter;
import java.util.Optional;
import static org.junit.jupiter.api.Assertions.assertTrue;
public class NiFiTest {
private static final String[] ARGUMENTS = new String[]{};
private static final String KEY_ARGUMENT = "-K";
private static final String[] FILE_NOT_SPECIFIED_ARGUMENTS = new String[]{KEY_ARGUMENT};
private static final String FAILURE_TO_LAUNCH = "Failure to launch NiFi";
private static final String PROPERTIES_LOADED = "Application Properties loaded";
private static final String PROPERTIES_PATH = "/NiFiProperties/conf/nifi.properties";
private static final String ENCRYPTED_PROPERTIES_PATH = "/NiFiProperties/conf/encrypted.nifi.properties";
private static final String ROOT_KEY = StringUtils.repeat("0", 64);
@BeforeEach
public void setAppender() {
ListAppender.clear();
}
@AfterEach
public void clearPropertiesFilePath() {
System.setProperty(NiFiProperties.PROPERTIES_FILE_PATH, StringUtils.EMPTY);
}
@Test
public void testMainBootstrapKeyFileNotSpecified() {
setPropertiesFilePath(PROPERTIES_PATH);
NiFi.main(FILE_NOT_SPECIFIED_ARGUMENTS);
assertFailureToLaunch();
}
@Test
public void testMainBootstrapKeyNotSpecified() {
setPropertiesFilePath(PROPERTIES_PATH);
NiFi.main(ARGUMENTS);
assertFailureToLaunch();
}
@Test
public void testMainEncryptedNiFiProperties() throws IOException {
final File rootKeyFile = File.createTempFile(getClass().getSimpleName(), ".root.key");
rootKeyFile.deleteOnExit();
try (final PrintWriter writer = new PrintWriter(new FileWriter(rootKeyFile))) {
writer.println(ROOT_KEY);
}
setPropertiesFilePath(ENCRYPTED_PROPERTIES_PATH);
NiFi.main(new String[]{KEY_ARGUMENT, rootKeyFile.getAbsolutePath()});
assertApplicationPropertiesLoaded();
assertFailureToLaunch();
}
private void assertApplicationPropertiesLoaded() {
final Optional<LoggingEvent> event = ListAppender.getLoggingEvents().stream().filter(
loggingEvent -> loggingEvent.getMessage().startsWith(PROPERTIES_LOADED)
).findFirst();
assertTrue(event.isPresent(), "Properties loaded log not found");
}
private void assertFailureToLaunch() {
final Optional<LoggingEvent> event = ListAppender.getLoggingEvents().stream().filter(
loggingEvent -> loggingEvent.getMessage().startsWith(FAILURE_TO_LAUNCH)
).findFirst();
assertTrue(event.isPresent(), "Failure log not found");
}
private void setPropertiesFilePath(final String relativePath) {
System.setProperty(NiFiProperties.PROPERTIES_FILE_PATH, getResourcesPath(relativePath));
}
private String getResourcesPath(final String relativePath) {
return getClass().getResource(relativePath).getPath();
}
}

16
nifi-framework-bundle/nifi-framework/nifi-runtime/src/test/resources/NiFiProperties/conf/bootstrap.conf
View File

@ -1,16 +0,0 @@
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

183
nifi-framework-bundle/nifi-framework/nifi-runtime/src/test/resources/NiFiProperties/conf/encrypted.nifi.properties
View File

@ -1,183 +0,0 @@
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# Core Properties #
nifi.flow.configuration.file=./target/conf/flow.json.gz
nifi.flow.configuration.archive.enabled=true
nifi.flow.configuration.archive.dir=./target/conf/archive/
nifi.flow.configuration.archive.max.time=30 days
nifi.flow.configuration.archive.max.storage=500 MB
nifi.flowcontroller.autoResumeState=true
nifi.flowcontroller.graceful.shutdown.period=10 sec
nifi.flowservice.writedelay.interval=500 ms
nifi.administrative.yield.duration=30 sec
# If a component has no work to do (is "bored"), how long should we wait before checking again for work?
nifi.bored.yield.duration=10 millis
nifi.authorizer.configuration.file=./target/conf/authorizers.xml
nifi.login.identity.provider.configuration.file=./target/conf/login-identity-providers.xml
nifi.ui.banner.text=dXwnu9mLyPETJrq1||n9e5dk5+HSTBCGOA/Sy6VYzwPw3baeRNvglalA1Pr1PcToyc4/qT6md24YOP4xVz14jd
nifi.ui.banner.text.protected=aes/gcm/256
nifi.nar.library.directory=./target/lib
nifi.nar.working.directory=./target/work/nar/
nifi.documentation.working.directory=./target/work/docs/components
####################
# State Management #
####################
nifi.state.management.configuration.file=./target/conf/state-management.xml
# The ID of the local state provider
nifi.state.management.provider.local=local-provider
# The ID of the cluster-wide state provider. This will be ignored if NiFi is not clustered but must be populated if running in a cluster.
nifi.state.management.provider.cluster=zk-provider
# Specifies whether or not this instance of NiFi should run an embedded ZooKeeper server
nifi.state.management.embedded.zookeeper.start=false
# Properties file that provides the ZooKeeper properties to use if <nifi.state.management.embedded.zookeeper.start> is set to true
nifi.state.management.embedded.zookeeper.properties=./target/conf/zookeeper.properties
# Database Settings
nifi.database.directory=./target/database_repository
# FlowFile Repository
nifi.flowfile.repository.implementation=org.apache.nifi.controller.repository.WriteAheadFlowFileRepository
nifi.flowfile.repository.directory=./target/flowfile_repository
nifi.flowfile.repository.partitions=256
nifi.flowfile.repository.checkpoint.interval=2 mins
nifi.flowfile.repository.always.sync=false
nifi.swap.manager.implementation=org.apache.nifi.controller.FileSystemSwapManager
nifi.queue.swap.threshold=20000
nifi.swap.in.period=5 sec
nifi.swap.in.threads=1
nifi.swap.out.period=5 sec
nifi.swap.out.threads=4
# Content Repository
nifi.content.repository.implementation=org.apache.nifi.controller.repository.FileSystemRepository
nifi.content.claim.max.appendable.size=10 MB
nifi.content.claim.max.flow.files=100
nifi.content.repository.directory.default=./target/content_repository
nifi.content.repository.archive.max.retention.period=12 hours
nifi.content.repository.archive.max.usage.percentage=50%
nifi.content.repository.archive.enabled=true
nifi.content.repository.always.sync=false
nifi.content.viewer.url=/nifi-content-viewer/
# Provenance Repository Properties
nifi.provenance.repository.implementation=org.apache.nifi.provenance.WriteAheadProvenanceRepository
# Persistent Provenance Repository Properties
nifi.provenance.repository.directory.default=./target/provenance_repository
nifi.provenance.repository.max.storage.time=24 hours
nifi.provenance.repository.max.storage.size=1 GB
nifi.provenance.repository.rollover.time=30 secs
nifi.provenance.repository.rollover.size=100 MB
nifi.provenance.repository.query.threads=2
nifi.provenance.repository.index.threads=1
nifi.provenance.repository.compress.on.rollover=true
nifi.provenance.repository.always.sync=false
nifi.provenance.repository.journal.count=16
# Comma-separated list of fields. Fields that are not indexed will not be searchable. Valid fields are:
# EventType, FlowFileUUID, Filename, TransitURI, ProcessorID, AlternateIdentifierURI, Relationship, Details
nifi.provenance.repository.indexed.fields=EventType, FlowFileUUID, Filename, ProcessorID, Relationship
# FlowFile Attributes that should be indexed and made searchable. Some examples to consider are filename, uuid, mime.type
nifi.provenance.repository.indexed.attributes=
# Large values for the shard size will result in more Java heap usage when searching the Provenance Repository
# but should provide better performance
nifi.provenance.repository.index.shard.size=500 MB
# Indicates the maximum length that a FlowFile attribute can be when retrieving a Provenance Event from
# the repository. If the length of any attribute exceeds this value, it will be truncated when the event is retrieved.
nifi.provenance.repository.max.attribute.length=65536
# Volatile Provenance Respository Properties
nifi.provenance.repository.buffer.size=100000
# Component Status Repository
nifi.components.status.repository.implementation=org.apache.nifi.controller.status.history.VolatileComponentStatusRepository
nifi.components.status.repository.buffer.size=1440
nifi.components.status.snapshot.frequency=1 min
# Site to Site properties
nifi.remote.input.host=
nifi.remote.input.secure=false
nifi.remote.input.socket.port=
nifi.remote.input.http.enabled=true
nifi.remote.input.http.transaction.ttl=30 sec
# web properties #
nifi.web.war.directory=./target/lib
nifi.web.http.host=
nifi.web.http.port=8080
nifi.web.https.host=
nifi.web.https.port=
nifi.web.jetty.working.directory=./target/work/jetty
nifi.web.jetty.threads=200
# security properties #
nifi.sensitive.props.key=dQU402Mz4J+t+e18||6+ictR0Nssq3/rR/d8fq5CFAKmpakr9jCyPIJYxG7n6D86gxsu2TRp4M48ugUw==
nifi.sensitive.props.key.protected=aes/gcm/256
nifi.sensitive.props.algorithm=NIFI_PBKDF2_AES_GCM_256
nifi.sensitive.props.additional.keys=nifi.ui.banner.text
nifi.security.keystore=/path/to/keystore.jks
nifi.security.keystoreType=JKS
nifi.security.keystorePasswd=Q8T3wv+Xl2ie98GV||qsuY9wa/Rt27cqFXs8ebX25E1iSbFAEFcD0cjCwrl3Tw6HghQjBIaCzQ
nifi.security.keystorePasswd.protected=aes/gcm/256
nifi.security.keyPasswd=1S0XmoiAr379B8rg||PPZzjdw9BAJSon9g4xm9uscFhCCyk734FTjXtRnBXUy819zsoQ==
nifi.security.keyPasswd.protected=aes/gcm/256
nifi.security.truststore=
nifi.security.truststoreType=
nifi.security.truststorePasswd=
nifi.security.user.authorizer=file-provider
nifi.security.user.login.identity.provider=
nifi.security.ocsp.responder.url=
nifi.security.ocsp.responder.certificate=
# Identity Mapping Properties #
# These properties allow normalizing user identities such that identities coming from different identity providers
# (certificates, LDAP, Kerberos) can be treated the same internally in NiFi. The following example demonstrates normalizing
# DNs from certificates and principals from Kerberos into a common identity string:
#
# nifi.security.identity.mapping.pattern.dn=^CN=(.*?), OU=(.*?), O=(.*?), L=(.*?), ST=(.*?), C=(.*?)$
# nifi.security.identity.mapping.value.dn=$1@$2
# nifi.security.identity.mapping.pattern.kerb=^(.*?)/instance@(.*?)$
# nifi.security.identity.mapping.value.kerb=$1@$2
# cluster common properties (all nodes must have same values) #
nifi.cluster.protocol.heartbeat.interval=5 sec
nifi.cluster.protocol.is.secure=false
# cluster node properties (only configure for cluster nodes) #
nifi.cluster.is.node=false
nifi.cluster.node.address=
nifi.cluster.node.protocol.port=
nifi.cluster.node.protocol.threads=10
nifi.cluster.node.event.history.size=25
nifi.cluster.node.connection.timeout=5 sec
nifi.cluster.node.read.timeout=5 sec
nifi.cluster.firewall.file=
# zookeeper properties, used for cluster management #
nifi.zookeeper.connect.string=
nifi.zookeeper.connect.timeout=3 secs
nifi.zookeeper.session.timeout=3 secs
nifi.zookeeper.root.node=/nifi
# kerberos #
nifi.kerberos.krb5.file=
nifi.kerberos.service.principal=
nifi.kerberos.keytab.location=
nifi.kerberos.authentication.expiration=12 hours

173
nifi-framework-bundle/nifi-framework/nifi-runtime/src/test/resources/NiFiProperties/conf/nifi.properties
View File

@ -1,173 +0,0 @@
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# Core Properties #
nifi.flow.configuration.file=./target/conf/flow.json.gz
nifi.flow.configuration.archive.enabled=true
nifi.flow.configuration.archive.dir=./target/conf/archive/
nifi.flow.configuration.archive.max.time=30 days
nifi.flow.configuration.archive.max.storage=500 MB
nifi.flowcontroller.autoResumeState=true
nifi.flowcontroller.graceful.shutdown.period=10 sec
nifi.flowservice.writedelay.interval=500 ms
nifi.administrative.yield.duration=30 sec
# If a component has no work to do (is "bored"), how long should we wait before checking again for work?
nifi.bored.yield.duration=10 millis
nifi.authorizer.configuration.file=./target/conf/authorizers.xml
nifi.login.identity.provider.configuration.file=./target/conf/login-identity-providers.xml
nifi.nar.library.directory=./target/lib
nifi.nar.working.directory=./target/work/nar/
nifi.documentation.working.directory=./target/work/docs/components
####################
# State Management #
####################
nifi.state.management.configuration.file=./target/conf/state-management.xml
# The ID of the local state provider
nifi.state.management.provider.local=local-provider
# The ID of the cluster-wide state provider. This will be ignored if NiFi is not clustered but must be populated if running in a cluster.
nifi.state.management.provider.cluster=zk-provider
# Specifies whether or not this instance of NiFi should run an embedded ZooKeeper server
nifi.state.management.embedded.zookeeper.start=false
# Properties file that provides the ZooKeeper properties to use if <nifi.state.management.embedded.zookeeper.start> is set to true
nifi.state.management.embedded.zookeeper.properties=./target/conf/zookeeper.properties
# Database Settings
nifi.database.directory=./target/database_repository
# FlowFile Repository
nifi.flowfile.repository.implementation=org.apache.nifi.controller.repository.WriteAheadFlowFileRepository
nifi.flowfile.repository.directory=./target/flowfile_repository
nifi.flowfile.repository.partitions=256
nifi.flowfile.repository.checkpoint.interval=2 mins
nifi.flowfile.repository.always.sync=false
nifi.swap.manager.implementation=org.apache.nifi.controller.FileSystemSwapManager
nifi.queue.swap.threshold=20000
nifi.swap.in.period=5 sec
nifi.swap.in.threads=1
nifi.swap.out.period=5 sec
nifi.swap.out.threads=4
# Content Repository
nifi.content.repository.implementation=org.apache.nifi.controller.repository.FileSystemRepository
nifi.content.claim.max.appendable.size=10 MB
nifi.content.claim.max.flow.files=100
nifi.content.repository.directory.default=./target/content_repository
nifi.content.repository.archive.max.retention.period=12 hours
nifi.content.repository.archive.max.usage.percentage=50%
nifi.content.repository.archive.enabled=true
nifi.content.repository.always.sync=false
nifi.content.viewer.url=/nifi-content-viewer/
# Provenance Repository Properties
nifi.provenance.repository.implementation=org.apache.nifi.provenance.WriteAheadProvenanceRepository
# Persistent Provenance Repository Properties
nifi.provenance.repository.directory.default=./target/provenance_repository
nifi.provenance.repository.max.storage.time=24 hours
nifi.provenance.repository.max.storage.size=1 GB
nifi.provenance.repository.rollover.time=30 secs
nifi.provenance.repository.rollover.size=100 MB
nifi.provenance.repository.query.threads=2
nifi.provenance.repository.index.threads=1
nifi.provenance.repository.compress.on.rollover=true
nifi.provenance.repository.always.sync=false
nifi.provenance.repository.journal.count=16
# Comma-separated list of fields. Fields that are not indexed will not be searchable. Valid fields are:
# EventType, FlowFileUUID, Filename, TransitURI, ProcessorID, AlternateIdentifierURI, Relationship, Details
nifi.provenance.repository.indexed.fields=EventType, FlowFileUUID, Filename, ProcessorID, Relationship
# FlowFile Attributes that should be indexed and made searchable. Some examples to consider are filename, uuid, mime.type
nifi.provenance.repository.indexed.attributes=
# Large values for the shard size will result in more Java heap usage when searching the Provenance Repository
# but should provide better performance
nifi.provenance.repository.index.shard.size=500 MB
# Indicates the maximum length that a FlowFile attribute can be when retrieving a Provenance Event from
# the repository. If the length of any attribute exceeds this value, it will be truncated when the event is retrieved.
nifi.provenance.repository.max.attribute.length=65536
# Volatile Provenance Respository Properties
nifi.provenance.repository.buffer.size=100000
# Component Status Repository
nifi.components.status.repository.implementation=org.apache.nifi.controller.status.history.VolatileComponentStatusRepository
nifi.components.status.repository.buffer.size=1440
nifi.components.status.snapshot.frequency=1 min
# Site to Site properties
nifi.remote.input.host=
nifi.remote.input.secure=false
nifi.remote.input.socket.port=
nifi.remote.input.http.enabled=true
nifi.remote.input.http.transaction.ttl=30 sec
# web properties #
nifi.web.war.directory=./target/lib
nifi.web.http.host=
nifi.web.http.port=8080
nifi.web.https.host=
nifi.web.https.port=
nifi.web.jetty.working.directory=./target/work/jetty
nifi.web.jetty.threads=200
nifi.security.keystore=
nifi.security.keystoreType=
nifi.security.keystorePasswd=
nifi.security.keyPasswd=
nifi.security.truststore=
nifi.security.truststoreType=
nifi.security.truststorePasswd=
nifi.security.user.authorizer=file-provider
nifi.security.user.login.identity.provider=
nifi.security.ocsp.responder.url=
nifi.security.ocsp.responder.certificate=
# Identity Mapping Properties #
# These properties allow normalizing user identities such that identities coming from different identity providers
# (certificates, LDAP, Kerberos) can be treated the same internally in NiFi. The following example demonstrates normalizing
# DNs from certificates and principals from Kerberos into a common identity string:
#
# nifi.security.identity.mapping.pattern.dn=^CN=(.*?), OU=(.*?), O=(.*?), L=(.*?), ST=(.*?), C=(.*?)$
# nifi.security.identity.mapping.value.dn=$1@$2
# nifi.security.identity.mapping.pattern.kerb=^(.*?)/instance@(.*?)$
# nifi.security.identity.mapping.value.kerb=$1@$2
# cluster common properties (all nodes must have same values) #
nifi.cluster.protocol.heartbeat.interval=5 sec
nifi.cluster.protocol.is.secure=false
# cluster node properties (only configure for cluster nodes) #
nifi.cluster.is.node=false
nifi.cluster.node.address=
nifi.cluster.node.protocol.port=
nifi.cluster.node.protocol.threads=10
nifi.cluster.node.event.history.size=25
nifi.cluster.node.connection.timeout=5 sec
nifi.cluster.node.read.timeout=5 sec
nifi.cluster.firewall.file=
# zookeeper properties, used for cluster management #
nifi.zookeeper.connect.string=
nifi.zookeeper.connect.timeout=3 secs
nifi.zookeeper.session.timeout=3 secs
nifi.zookeeper.root.node=/nifi
# kerberos #
nifi.kerberos.krb5.file=
nifi.kerberos.service.principal=
nifi.kerberos.keytab.location=
nifi.kerberos.authentication.expiration=12 hours

28
nifi-framework-bundle/nifi-framework/nifi-runtime/src/test/resources/logback-test.xml
View File

@ -1,28 +0,0 @@
<?xml version="1.0" encoding="UTF-8"?>
<!--
Licensed to the Apache Software Foundation (ASF) under one or more
contributor license agreements. See the NOTICE file distributed with
this work for additional information regarding copyright ownership.
The ASF licenses this file to You under the Apache License, Version 2.0
(the "License"); you may not use this file except in compliance with
the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
-->
<configuration>
<appender name="TEST" class="org.apache.nifi.ListAppender">
<encoder class="ch.qos.logback.classic.encoder.PatternLayoutEncoder">
<pattern>%-4r [%t] %-5p %c - %m%n</pattern>
</encoder>
</appender>
<logger name="org.apache.nifi.processor" level="DEBUG"/>
<root level="DEBUG">
<appender-ref ref="TEST"/>
</root>
</configuration>