mirror of https://github.com/apache/nifi.git
NIFI-13592 Removed unused nifi-runtime classes and methods (#9120)
- Removed unused key processing methods from NiFi runtime class - Removed unused stateless and embedded NiFi classes along with tests
This commit is contained in:
David Handermann
GitHub
parent
60a03b90cb
commit
25299f8106
|
|
@ -1,56 +0,0 @@
|
||||||
/*
|
|
||||||
* Licensed to the Apache Software Foundation (ASF) under one or more
|
|
||||||
* contributor license agreements. See the NOTICE file distributed with
|
|
||||||
* this work for additional information regarding copyright ownership.
|
|
||||||
* The ASF licenses this file to You under the Apache License, Version 2.0
|
|
||||||
* (the "License"); you may not use this file except in compliance with
|
|
||||||
* the License. You may obtain a copy of the License at
|
|
||||||
*
|
|
||||||
* http://www.apache.org/licenses/LICENSE-2.0
|
|
||||||
*
|
|
||||||
* Unless required by applicable law or agreed to in writing, software
|
|
||||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
|
||||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
||||||
* See the License for the specific language governing permissions and
|
|
||||||
* limitations under the License.
|
|
||||||
*/
|
|
||||||
package org.apache.nifi.bootstrap;
|
|
||||||
|
|
||||||
import java.io.File;
|
|
||||||
import java.lang.reflect.Method;
|
|
||||||
import java.net.URL;
|
|
||||||
import java.net.URLClassLoader;
|
|
||||||
import java.util.ArrayList;
|
|
||||||
import java.util.List;
|
|
||||||
import java.util.Objects;
|
|
||||||
|
|
||||||
public class RunStatelessNiFi {
|
|
||||||
|
|
||||||
public static void main(final String[] args) throws Exception {
|
|
||||||
|
|
||||||
String nifi_home = System.getenv("NIFI_HOME");
|
|
||||||
if (nifi_home == null || nifi_home.equals("")) {
|
|
||||||
nifi_home = ".";
|
|
||||||
}
|
|
||||||
|
|
||||||
final List<URL> cpURLs = new ArrayList<>();
|
|
||||||
final File libDir = new File(nifi_home + "/lib");
|
|
||||||
if (libDir.exists()) {
|
|
||||||
for (final File file : Objects.requireNonNull(libDir.listFiles((dir, filename) -> filename.toLowerCase().endsWith(".jar")))) {
|
|
||||||
cpURLs.add(file.toURI().toURL());
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
if (cpURLs.isEmpty()) {
|
|
||||||
throw new RuntimeException("Could not find lib directory at " + libDir.getAbsolutePath());
|
|
||||||
}
|
|
||||||
|
|
||||||
final URLClassLoader rootClassLoader = new URLClassLoader(cpURLs.toArray(new URL[0]));
|
|
||||||
Thread.currentThread().setContextClassLoader(rootClassLoader);
|
|
||||||
|
|
||||||
final Class<?> programClass = Class.forName("org.apache.nifi.StatelessNiFi", true, rootClassLoader);
|
|
||||||
final Method launchMethod = programClass.getMethod("main", String[].class);
|
|
||||||
launchMethod.setAccessible(true);
|
|
||||||
launchMethod.invoke(null, (Object) args);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
@ -34,26 +34,5 @@
|
||||||
<groupId>org.apache.nifi</groupId>
|
<groupId>org.apache.nifi</groupId>
|
||||||
<artifactId>nifi-nar-utils</artifactId>
|
<artifactId>nifi-nar-utils</artifactId>
|
||||||
</dependency>
|
</dependency>
|
||||||
<dependency>
|
|
||||||
<groupId>ch.qos.logback</groupId>
|
|
||||||
<artifactId>logback-classic</artifactId>
|
|
||||||
<scope>test</scope>
|
|
||||||
</dependency>
|
|
||||||
<dependency>
|
|
||||||
<groupId>ch.qos.logback</groupId>
|
|
||||||
<artifactId>logback-core</artifactId>
|
|
||||||
<scope>test</scope>
|
|
||||||
</dependency>
|
|
||||||
<dependency>
|
|
||||||
<groupId>org.apache.nifi</groupId>
|
|
||||||
<artifactId>nifi-properties-loader</artifactId>
|
|
||||||
<version>2.0.0-SNAPSHOT</version>
|
|
||||||
<scope>test</scope>
|
|
||||||
</dependency>
|
|
||||||
<dependency>
|
|
||||||
<groupId>org.apache.commons</groupId>
|
|
||||||
<artifactId>commons-lang3</artifactId>
|
|
||||||
<scope>test</scope>
|
|
||||||
</dependency>
|
|
||||||
</dependencies>
|
</dependencies>
|
||||||
</project>
|
</project>
|
||||||
|
|
|
||||||
|
|
@ -1,64 +0,0 @@
|
||||||
/*
|
|
||||||
* Licensed to the Apache Software Foundation (ASF) under one or more
|
|
||||||
* contributor license agreements. See the NOTICE file distributed with
|
|
||||||
* this work for additional information regarding copyright ownership.
|
|
||||||
* The ASF licenses this file to You under the Apache License, Version 2.0
|
|
||||||
* (the "License"); you may not use this file except in compliance with
|
|
||||||
* the License. You may obtain a copy of the License at
|
|
||||||
*
|
|
||||||
* http://www.apache.org/licenses/LICENSE-2.0
|
|
||||||
*
|
|
||||||
* Unless required by applicable law or agreed to in writing, software
|
|
||||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
|
||||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
||||||
* See the License for the specific language governing permissions and
|
|
||||||
* limitations under the License.
|
|
||||||
*/
|
|
||||||
|
|
||||||
package org.apache.nifi;
|
|
||||||
|
|
||||||
import java.io.IOException;
|
|
||||||
import java.lang.reflect.InvocationTargetException;
|
|
||||||
|
|
||||||
/**
|
|
||||||
* <p>
|
|
||||||
* Starts an instance of NiFi within the <b>same JVM</b>, which can later properly be shut down.
|
|
||||||
* Intended to be used for testing purposes.</p>
|
|
||||||
*
|
|
||||||
*/
|
|
||||||
public class EmbeddedNiFi extends NiFi {
|
|
||||||
|
|
||||||
public EmbeddedNiFi(String[] args, ClassLoader rootClassLoader)
|
|
||||||
throws ClassNotFoundException, IOException, NoSuchMethodException,
|
|
||||||
InstantiationException, IllegalAccessException, IllegalArgumentException, InvocationTargetException {
|
|
||||||
|
|
||||||
super(convertArgumentsToValidatedNiFiProperties(args), rootClassLoader);
|
|
||||||
}
|
|
||||||
|
|
||||||
public EmbeddedNiFi(String[] args, ClassLoader rootClassLoader, ClassLoader bootstrapClassLoader)
|
|
||||||
throws ClassNotFoundException, IOException, NoSuchMethodException,
|
|
||||||
InstantiationException, IllegalAccessException, IllegalArgumentException, InvocationTargetException {
|
|
||||||
|
|
||||||
super(convertArgumentsToValidatedNiFiProperties(args, bootstrapClassLoader), rootClassLoader);
|
|
||||||
}
|
|
||||||
|
|
||||||
@Override
|
|
||||||
protected void initLogging() {
|
|
||||||
// do nothing when running in embedded mode
|
|
||||||
}
|
|
||||||
|
|
||||||
@Override
|
|
||||||
protected void setDefaultUncaughtExceptionHandler() {
|
|
||||||
// do nothing when running in embedded mode
|
|
||||||
}
|
|
||||||
|
|
||||||
@Override
|
|
||||||
protected void addShutdownHook() {
|
|
||||||
// do nothing when running in embedded mode
|
|
||||||
}
|
|
||||||
|
|
||||||
@Override
|
|
||||||
public void shutdown() {
|
|
||||||
super.shutdown();
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
@ -34,7 +34,6 @@ import org.slf4j.bridge.SLF4JBridgeHandler;
|
||||||
|
|
||||||
import java.io.File;
|
import java.io.File;
|
||||||
import java.io.FileOutputStream;
|
import java.io.FileOutputStream;
|
||||||
import java.io.FileWriter;
|
|
||||||
import java.io.IOException;
|
import java.io.IOException;
|
||||||
import java.io.OutputStream;
|
import java.io.OutputStream;
|
||||||
import java.lang.reflect.InvocationTargetException;
|
import java.lang.reflect.InvocationTargetException;
|
||||||
|
|
@ -42,26 +41,15 @@ import java.lang.reflect.Method;
|
||||||
import java.net.MalformedURLException;
|
import java.net.MalformedURLException;
|
||||||
import java.net.URL;
|
import java.net.URL;
|
||||||
import java.net.URLClassLoader;
|
import java.net.URLClassLoader;
|
||||||
import java.nio.charset.StandardCharsets;
|
|
||||||
import java.nio.file.Files;
|
import java.nio.file.Files;
|
||||||
import java.nio.file.Path;
|
import java.nio.file.Path;
|
||||||
import java.nio.file.Paths;
|
import java.nio.file.Paths;
|
||||||
import java.time.LocalDateTime;
|
import java.time.LocalDateTime;
|
||||||
import java.time.format.DateTimeFormatter;
|
import java.time.format.DateTimeFormatter;
|
||||||
import java.util.ArrayList;
|
import java.util.ArrayList;
|
||||||
import java.util.Arrays;
|
|
||||||
import java.util.List;
|
import java.util.List;
|
||||||
import java.util.Random;
|
|
||||||
import java.util.Set;
|
import java.util.Set;
|
||||||
import java.util.Timer;
|
|
||||||
import java.util.TimerTask;
|
|
||||||
import java.util.concurrent.Executors;
|
|
||||||
import java.util.concurrent.ScheduledExecutorService;
|
|
||||||
import java.util.concurrent.ScheduledFuture;
|
|
||||||
import java.util.concurrent.ThreadFactory;
|
|
||||||
import java.util.concurrent.TimeUnit;
|
import java.util.concurrent.TimeUnit;
|
||||||
import java.util.concurrent.atomic.AtomicInteger;
|
|
||||||
import java.util.concurrent.atomic.AtomicLong;
|
|
||||||
import java.util.stream.Stream;
|
import java.util.stream.Stream;
|
||||||
|
|
||||||
public class NiFi implements NiFiEntryPoint {
|
public class NiFi implements NiFiEntryPoint {
|
||||||
|
|
@ -70,7 +58,6 @@ public class NiFi implements NiFiEntryPoint {
|
||||||
public static final DateTimeFormatter DATE_TIME_FORMATTER = DateTimeFormatter.ofPattern("yyyy-MM-dd_HH-mm-ss");
|
public static final DateTimeFormatter DATE_TIME_FORMATTER = DateTimeFormatter.ofPattern("yyyy-MM-dd_HH-mm-ss");
|
||||||
|
|
||||||
private static final Logger LOGGER = LoggerFactory.getLogger(NiFi.class);
|
private static final Logger LOGGER = LoggerFactory.getLogger(NiFi.class);
|
||||||
private static final String KEY_FILE_FLAG = "-K";
|
|
||||||
|
|
||||||
private final NiFiServer nifiServer;
|
private final NiFiServer nifiServer;
|
||||||
private final BootstrapListener bootstrapListener;
|
private final BootstrapListener bootstrapListener;
|
||||||
|
|
@ -130,8 +117,6 @@ public class NiFi implements NiFiEntryPoint {
|
||||||
FileUtils.deleteFilesInDirectory(webWorkingDir, null, LOGGER, true, true);
|
FileUtils.deleteFilesInDirectory(webWorkingDir, null, LOGGER, true, true);
|
||||||
FileUtils.deleteFile(webWorkingDir, LOGGER, 3);
|
FileUtils.deleteFile(webWorkingDir, LOGGER, 3);
|
||||||
|
|
||||||
detectTimingIssues();
|
|
||||||
|
|
||||||
// redirect JUL log events
|
// redirect JUL log events
|
||||||
initLogging();
|
initLogging();
|
||||||
|
|
||||||
|
|
@ -187,9 +172,7 @@ public class NiFi implements NiFiEntryPoint {
|
||||||
}
|
}
|
||||||
|
|
||||||
protected void setDefaultUncaughtExceptionHandler() {
|
protected void setDefaultUncaughtExceptionHandler() {
|
||||||
Thread.setDefaultUncaughtExceptionHandler((thread, exception) -> {
|
Thread.setDefaultUncaughtExceptionHandler((thread, exception) -> LOGGER.error("An Unknown Error Occurred in Thread {}", thread, exception));
|
||||||
LOGGER.error("An Unknown Error Occurred in Thread {}", thread, exception);
|
|
||||||
});
|
|
||||||
}
|
}
|
||||||
|
|
||||||
protected void addShutdownHook() {
|
protected void addShutdownHook() {
|
||||||
|
|
@ -269,57 +252,6 @@ public class NiFi implements NiFiEntryPoint {
|
||||||
LOGGER.info("Application Server shutdown completed");
|
LOGGER.info("Application Server shutdown completed");
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
|
||||||
* Determine if the machine we're running on has timing issues.
|
|
||||||
*/
|
|
||||||
private void detectTimingIssues() {
|
|
||||||
final int minRequiredOccurrences = 25;
|
|
||||||
final int maxOccurrencesOutOfRange = 15;
|
|
||||||
final AtomicLong lastTriggerMillis = new AtomicLong(System.currentTimeMillis());
|
|
||||||
|
|
||||||
final ScheduledExecutorService service = Executors.newScheduledThreadPool(1, new ThreadFactory() {
|
|
||||||
private final ThreadFactory defaultFactory = Executors.defaultThreadFactory();
|
|
||||||
|
|
||||||
@Override
|
|
||||||
public Thread newThread(final Runnable runnable) {
|
|
||||||
final Thread t = defaultFactory.newThread(runnable);
|
|
||||||
t.setDaemon(true);
|
|
||||||
t.setName("Detect Timing Issues");
|
|
||||||
return t;
|
|
||||||
}
|
|
||||||
});
|
|
||||||
|
|
||||||
final AtomicInteger occurrencesOutOfRange = new AtomicInteger(0);
|
|
||||||
final AtomicInteger occurrences = new AtomicInteger(0);
|
|
||||||
final Runnable command = () -> {
|
|
||||||
final long curMillis = System.currentTimeMillis();
|
|
||||||
final long difference = curMillis - lastTriggerMillis.get();
|
|
||||||
final long millisOff = Math.abs(difference - 2000L);
|
|
||||||
occurrences.incrementAndGet();
|
|
||||||
if (millisOff > 500L) {
|
|
||||||
occurrencesOutOfRange.incrementAndGet();
|
|
||||||
}
|
|
||||||
lastTriggerMillis.set(curMillis);
|
|
||||||
};
|
|
||||||
|
|
||||||
final ScheduledFuture<?> future = service.scheduleWithFixedDelay(command, 2000L, 2000L, TimeUnit.MILLISECONDS);
|
|
||||||
|
|
||||||
final TimerTask timerTask = new TimerTask() {
|
|
||||||
@Override
|
|
||||||
public void run() {
|
|
||||||
future.cancel(true);
|
|
||||||
service.shutdownNow();
|
|
||||||
|
|
||||||
if (occurrences.get() < minRequiredOccurrences || occurrencesOutOfRange.get() > maxOccurrencesOutOfRange) {
|
|
||||||
LOGGER.warn("NiFi has detected that this box is not responding within the expected timing interval, which may cause "
|
|
||||||
+ "Processors to be scheduled erratically. Please see the NiFi documentation for more information.");
|
|
||||||
}
|
|
||||||
}
|
|
||||||
};
|
|
||||||
final Timer timer = new Timer(true);
|
|
||||||
timer.schedule(timerTask, 60000L);
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Main entry point of the application.
|
* Main entry point of the application.
|
||||||
*
|
*
|
||||||
|
|
@ -328,24 +260,24 @@ public class NiFi implements NiFiEntryPoint {
|
||||||
public static void main(String[] args) {
|
public static void main(String[] args) {
|
||||||
LOGGER.info("Launching NiFi...");
|
LOGGER.info("Launching NiFi...");
|
||||||
try {
|
try {
|
||||||
NiFiProperties properties = convertArgumentsToValidatedNiFiProperties(args);
|
NiFiProperties properties = loadProperties();
|
||||||
new NiFi(properties);
|
new NiFi(properties);
|
||||||
} catch (final Throwable t) {
|
} catch (final Throwable t) {
|
||||||
LOGGER.error("Failure to launch NiFi", t);
|
LOGGER.error("Failure to launch NiFi", t);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
protected static NiFiProperties convertArgumentsToValidatedNiFiProperties(String[] args) {
|
protected static NiFiProperties loadProperties() {
|
||||||
return convertArgumentsToValidatedNiFiProperties(args, createBootstrapClassLoader());
|
return loadProperties(createBootstrapClassLoader());
|
||||||
}
|
}
|
||||||
|
|
||||||
protected static NiFiProperties convertArgumentsToValidatedNiFiProperties(String[] args, final ClassLoader bootstrapClassLoader) {
|
protected static NiFiProperties loadProperties(final ClassLoader bootstrapClassLoader) {
|
||||||
NiFiProperties properties = initializeProperties(args, bootstrapClassLoader);
|
NiFiProperties properties = initializeProperties(bootstrapClassLoader);
|
||||||
properties.validate();
|
properties.validate();
|
||||||
return properties;
|
return properties;
|
||||||
}
|
}
|
||||||
|
|
||||||
private static NiFiProperties initializeProperties(final String[] args, final ClassLoader boostrapLoader) {
|
private static NiFiProperties initializeProperties(final ClassLoader boostrapLoader) {
|
||||||
final ClassLoader contextClassLoader = Thread.currentThread().getContextClassLoader();
|
final ClassLoader contextClassLoader = Thread.currentThread().getContextClassLoader();
|
||||||
Thread.currentThread().setContextClassLoader(boostrapLoader);
|
Thread.currentThread().setContextClassLoader(boostrapLoader);
|
||||||
|
|
||||||
|
|
@ -369,97 +301,4 @@ public class NiFi implements NiFiEntryPoint {
|
||||||
Thread.currentThread().setContextClassLoader(contextClassLoader);
|
Thread.currentThread().setContextClassLoader(contextClassLoader);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
private static String loadFormattedKey(String[] args) {
|
|
||||||
String key = null;
|
|
||||||
List<String> parsedArgs = parseArgs(args);
|
|
||||||
// Check if args contain protection key
|
|
||||||
if (parsedArgs.contains(KEY_FILE_FLAG)) {
|
|
||||||
key = getKeyFromKeyFileAndPrune(parsedArgs);
|
|
||||||
// Format the key (check hex validity and remove spaces)
|
|
||||||
key = formatHexKey(key);
|
|
||||||
|
|
||||||
}
|
|
||||||
|
|
||||||
if (null == key) {
|
|
||||||
return "";
|
|
||||||
} else if (!isHexKeyValid(key)) {
|
|
||||||
throw new IllegalArgumentException("The key was not provided in valid hex format and of the correct length");
|
|
||||||
} else {
|
|
||||||
return key;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
private static String getKeyFromKeyFileAndPrune(List<String> parsedArgs) {
|
|
||||||
String key = null;
|
|
||||||
LOGGER.debug("The bootstrap process provided the {} flag", KEY_FILE_FLAG);
|
|
||||||
int i = parsedArgs.indexOf(KEY_FILE_FLAG);
|
|
||||||
if (parsedArgs.size() <= i + 1) {
|
|
||||||
LOGGER.error("The bootstrap process passed the {} flag without a filename", KEY_FILE_FLAG);
|
|
||||||
throw new IllegalArgumentException("The bootstrap process provided the " + KEY_FILE_FLAG + " flag but no key");
|
|
||||||
}
|
|
||||||
try {
|
|
||||||
String passwordfilePath = parsedArgs.get(i + 1);
|
|
||||||
// Slurp in the contents of the file:
|
|
||||||
byte[] encoded = Files.readAllBytes(Paths.get(passwordfilePath));
|
|
||||||
key = new String(encoded, StandardCharsets.UTF_8);
|
|
||||||
if (0 == key.length())
|
|
||||||
throw new IllegalArgumentException("Key in keyfile " + passwordfilePath + " yielded an empty key");
|
|
||||||
|
|
||||||
LOGGER.debug("Overwriting temporary bootstrap key file [{}]", passwordfilePath);
|
|
||||||
|
|
||||||
// Overwrite the contents of the file (to avoid littering file system
|
|
||||||
// unlinked with key material):
|
|
||||||
File passwordFile = new File(passwordfilePath);
|
|
||||||
FileWriter overwriter = new FileWriter(passwordFile, false);
|
|
||||||
|
|
||||||
// Construe a random pad:
|
|
||||||
Random random = new Random();
|
|
||||||
StringBuffer sb = new StringBuffer();
|
|
||||||
// Note on correctness: this pad is longer, but equally sufficient.
|
|
||||||
while (sb.length() < encoded.length) {
|
|
||||||
sb.append(Integer.toHexString(random.nextInt()));
|
|
||||||
}
|
|
||||||
String pad = sb.toString();
|
|
||||||
overwriter.write(pad);
|
|
||||||
overwriter.close();
|
|
||||||
|
|
||||||
LOGGER.debug("Removing temporary bootstrap key file [{}]", passwordfilePath);
|
|
||||||
passwordFile.delete();
|
|
||||||
|
|
||||||
} catch (IOException e) {
|
|
||||||
LOGGER.error("Caught IOException while retrieving the {} -passed keyfile; aborting", KEY_FILE_FLAG, e);
|
|
||||||
System.exit(1);
|
|
||||||
}
|
|
||||||
|
|
||||||
return key;
|
|
||||||
}
|
|
||||||
|
|
||||||
private static List<String> parseArgs(String[] args) {
|
|
||||||
List<String> parsedArgs = new ArrayList<>(Arrays.asList(args));
|
|
||||||
for (int i = 0; i < parsedArgs.size(); i++) {
|
|
||||||
if (parsedArgs.get(i).startsWith(KEY_FILE_FLAG + " ")) {
|
|
||||||
String[] split = parsedArgs.get(i).split(" ", 2);
|
|
||||||
parsedArgs.set(i, split[0]);
|
|
||||||
parsedArgs.add(i + 1, split[1]);
|
|
||||||
break;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return parsedArgs;
|
|
||||||
}
|
|
||||||
|
|
||||||
private static String formatHexKey(String input) {
|
|
||||||
if (input == null || input.trim().isEmpty()) {
|
|
||||||
return "";
|
|
||||||
}
|
|
||||||
return input.replaceAll("[^0-9a-fA-F]", "").toLowerCase();
|
|
||||||
}
|
|
||||||
|
|
||||||
private static boolean isHexKeyValid(String key) {
|
|
||||||
if (key == null || key.trim().isEmpty()) {
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
// Key length is in "nibbles" (i.e. one hex char = 4 bits)
|
|
||||||
return Arrays.asList(128, 196, 256).contains(key.length() * 4) && key.matches("^[0-9a-fA-F]*$");
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -1,115 +0,0 @@
|
||||||
/*
|
|
||||||
* Licensed to the Apache Software Foundation (ASF) under one or more
|
|
||||||
* contributor license agreements. See the NOTICE file distributed with
|
|
||||||
* this work for additional information regarding copyright ownership.
|
|
||||||
* The ASF licenses this file to You under the Apache License, Version 2.0
|
|
||||||
* (the "License"); you may not use this file except in compliance with
|
|
||||||
* the License. You may obtain a copy of the License at
|
|
||||||
*
|
|
||||||
* http://www.apache.org/licenses/LICENSE-2.0
|
|
||||||
*
|
|
||||||
* Unless required by applicable law or agreed to in writing, software
|
|
||||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
|
||||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
||||||
* See the License for the specific language governing permissions and
|
|
||||||
* limitations under the License.
|
|
||||||
*/
|
|
||||||
package org.apache.nifi;
|
|
||||||
|
|
||||||
import org.apache.nifi.nar.NarUnpacker;
|
|
||||||
import org.apache.nifi.nar.NarUnpackMode;
|
|
||||||
import org.slf4j.Logger;
|
|
||||||
import org.slf4j.LoggerFactory;
|
|
||||||
|
|
||||||
import java.io.File;
|
|
||||||
import java.io.FileNotFoundException;
|
|
||||||
import java.io.IOException;
|
|
||||||
import java.lang.reflect.InvocationTargetException;
|
|
||||||
import java.lang.reflect.Method;
|
|
||||||
import java.net.MalformedURLException;
|
|
||||||
import java.net.URL;
|
|
||||||
import java.net.URLClassLoader;
|
|
||||||
import java.util.ArrayList;
|
|
||||||
import java.util.List;
|
|
||||||
import java.util.Objects;
|
|
||||||
import java.util.concurrent.TimeUnit;
|
|
||||||
|
|
||||||
public class StatelessNiFi {
|
|
||||||
private static final Logger logger = LoggerFactory.getLogger(StatelessNiFi.class);
|
|
||||||
|
|
||||||
public static final String PROGRAM_CLASS_NAME = "org.apache.nifi.stateless.runtimes.Program";
|
|
||||||
|
|
||||||
public static final String EXTRACT_NARS = "ExtractNars";
|
|
||||||
|
|
||||||
public static void main(final String[] args) throws IOException, ClassNotFoundException, NoSuchMethodException, InvocationTargetException, IllegalAccessException {
|
|
||||||
|
|
||||||
String nifi_home = System.getenv("NIFI_HOME");
|
|
||||||
if (nifi_home == null || nifi_home.equals("")) {
|
|
||||||
nifi_home = ".";
|
|
||||||
}
|
|
||||||
|
|
||||||
final File libDir = new File(nifi_home + "/lib");
|
|
||||||
final File narWorkingDirectory = new File(nifi_home + "/work/stateless-nars");
|
|
||||||
|
|
||||||
if (args.length >= 1 && args[0].equals(EXTRACT_NARS)) {
|
|
||||||
if (!libDir.exists()) {
|
|
||||||
System.out.println("Specified lib directory <" + libDir + "> does not exist");
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
|
|
||||||
final File[] narFiles = libDir.listFiles(file -> file.getName().endsWith(".nar"));
|
|
||||||
if (narFiles == null) {
|
|
||||||
System.out.println("Could not obtain listing of lib directory <" + libDir + ">");
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
|
|
||||||
if (!narWorkingDirectory.exists() && !narWorkingDirectory.mkdirs()) {
|
|
||||||
throw new IOException("Could not create NAR working directory <" + narWorkingDirectory + ">");
|
|
||||||
}
|
|
||||||
|
|
||||||
logger.info("Unpacking {} NARs", narFiles.length);
|
|
||||||
final long startUnpack = System.nanoTime();
|
|
||||||
for (final File narFile : narFiles) {
|
|
||||||
NarUnpacker.unpackNar(narFile, narWorkingDirectory, false, NarUnpackMode.UNPACK_TO_UBER_JAR);
|
|
||||||
}
|
|
||||||
|
|
||||||
final long millis = TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - startUnpack);
|
|
||||||
logger.info("Finished unpacking {} NARs in {} millis", narFiles.length, millis);
|
|
||||||
|
|
||||||
System.exit(0);
|
|
||||||
}
|
|
||||||
|
|
||||||
File frameworkWorkingDirectory;
|
|
||||||
try {
|
|
||||||
frameworkWorkingDirectory = Objects.requireNonNull(narWorkingDirectory.listFiles(file -> file.getName().startsWith("nifi-framework")))[0];
|
|
||||||
} catch (Exception ex) {
|
|
||||||
throw new FileNotFoundException("Could not find core stateless dependencies in the working directory <" + narWorkingDirectory + ">");
|
|
||||||
}
|
|
||||||
|
|
||||||
final File bundledDependenciesDir = new File(frameworkWorkingDirectory, NarUnpacker.BUNDLED_DEPENDENCIES_DIRECTORY);
|
|
||||||
final File[] jarFiles = bundledDependenciesDir.listFiles();
|
|
||||||
if (jarFiles == null) {
|
|
||||||
throw new IOException("Could not obtain listing of NiFi-Framework NAR's bundled dependencies in working directory <" + bundledDependenciesDir + ">");
|
|
||||||
}
|
|
||||||
final URL[] jarUrls = toURLs(jarFiles);
|
|
||||||
|
|
||||||
|
|
||||||
final ClassLoader rootClassLoader = Thread.currentThread().getContextClassLoader();
|
|
||||||
final URLClassLoader frameworkClassLoader = new URLClassLoader(jarUrls, rootClassLoader);
|
|
||||||
Thread.currentThread().setContextClassLoader(frameworkClassLoader);
|
|
||||||
|
|
||||||
final Class<?> programClass = Class.forName(PROGRAM_CLASS_NAME, true, frameworkClassLoader);
|
|
||||||
final Method launchMethod = programClass.getMethod("launch", String[].class, ClassLoader.class, File.class);
|
|
||||||
launchMethod.setAccessible(true);
|
|
||||||
launchMethod.invoke(null, args, rootClassLoader, narWorkingDirectory);
|
|
||||||
}
|
|
||||||
|
|
||||||
private static URL[] toURLs(final File[] files) throws MalformedURLException {
|
|
||||||
final List<URL> urls = new ArrayList<>();
|
|
||||||
for (final File file : files) {
|
|
||||||
urls.add(file.toURI().toURL());
|
|
||||||
}
|
|
||||||
|
|
||||||
return urls.toArray(new URL[0]);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
@ -1,39 +0,0 @@
|
||||||
/**
|
|
||||||
* Licensed to the Apache Software Foundation (ASF) under one or more
|
|
||||||
* contributor license agreements. See the NOTICE file distributed with
|
|
||||||
* this work for additional information regarding copyright ownership.
|
|
||||||
* The ASF licenses this file to You under the Apache License, Version 2.0
|
|
||||||
* (the "License"); you may not use this file except in compliance with
|
|
||||||
* the License. You may obtain a copy of the License at
|
|
||||||
* <p>
|
|
||||||
* http://www.apache.org/licenses/LICENSE-2.0
|
|
||||||
* <p>
|
|
||||||
* Unless required by applicable law or agreed to in writing, software
|
|
||||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
|
||||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
||||||
* See the License for the specific language governing permissions and
|
|
||||||
* limitations under the License.
|
|
||||||
*/
|
|
||||||
package org.apache.nifi.headless;
|
|
||||||
|
|
||||||
public class FlowEnrichmentException extends Exception {
|
|
||||||
public FlowEnrichmentException() {
|
|
||||||
super();
|
|
||||||
}
|
|
||||||
|
|
||||||
public FlowEnrichmentException(String message) {
|
|
||||||
super(message);
|
|
||||||
}
|
|
||||||
|
|
||||||
public FlowEnrichmentException(String message, Throwable cause) {
|
|
||||||
super(message, cause);
|
|
||||||
}
|
|
||||||
|
|
||||||
public FlowEnrichmentException(Throwable cause) {
|
|
||||||
super(cause);
|
|
||||||
}
|
|
||||||
|
|
||||||
protected FlowEnrichmentException(String message, Throwable cause, boolean enableSuppression, boolean writableStackTrace) {
|
|
||||||
super(message, cause, enableSuppression, writableStackTrace);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
@ -1,41 +0,0 @@
|
||||||
/*
|
|
||||||
* Licensed to the Apache Software Foundation (ASF) under one or more
|
|
||||||
* contributor license agreements. See the NOTICE file distributed with
|
|
||||||
* this work for additional information regarding copyright ownership.
|
|
||||||
* The ASF licenses this file to You under the Apache License, Version 2.0
|
|
||||||
* (the "License"); you may not use this file except in compliance with
|
|
||||||
* the License. You may obtain a copy of the License at
|
|
||||||
*
|
|
||||||
* http://www.apache.org/licenses/LICENSE-2.0
|
|
||||||
*
|
|
||||||
* Unless required by applicable law or agreed to in writing, software
|
|
||||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
|
||||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
||||||
* See the License for the specific language governing permissions and
|
|
||||||
* limitations under the License.
|
|
||||||
*/
|
|
||||||
package org.apache.nifi;
|
|
||||||
|
|
||||||
import ch.qos.logback.classic.spi.LoggingEvent;
|
|
||||||
import ch.qos.logback.core.AppenderBase;
|
|
||||||
|
|
||||||
import java.util.ArrayList;
|
|
||||||
import java.util.Collections;
|
|
||||||
import java.util.List;
|
|
||||||
|
|
||||||
public class ListAppender extends AppenderBase<LoggingEvent> {
|
|
||||||
private static final List<LoggingEvent> LOGGING_EVENTS = Collections.synchronizedList(new ArrayList<>());
|
|
||||||
|
|
||||||
public static List<LoggingEvent> getLoggingEvents() {
|
|
||||||
return LOGGING_EVENTS;
|
|
||||||
}
|
|
||||||
|
|
||||||
public static void clear() {
|
|
||||||
LOGGING_EVENTS.clear();
|
|
||||||
}
|
|
||||||
|
|
||||||
@Override
|
|
||||||
protected void append(final LoggingEvent loggingEvent) {
|
|
||||||
LOGGING_EVENTS.add(loggingEvent);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
@ -1,116 +0,0 @@
|
||||||
/*
|
|
||||||
* Licensed to the Apache Software Foundation (ASF) under one or more
|
|
||||||
* contributor license agreements. See the NOTICE file distributed with
|
|
||||||
* this work for additional information regarding copyright ownership.
|
|
||||||
* The ASF licenses this file to You under the Apache License, Version 2.0
|
|
||||||
* (the "License"); you may not use this file except in compliance with
|
|
||||||
* the License. You may obtain a copy of the License at
|
|
||||||
*
|
|
||||||
* http://www.apache.org/licenses/LICENSE-2.0
|
|
||||||
*
|
|
||||||
* Unless required by applicable law or agreed to in writing, software
|
|
||||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
|
||||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
||||||
* See the License for the specific language governing permissions and
|
|
||||||
* limitations under the License.
|
|
||||||
*/
|
|
||||||
package org.apache.nifi;
|
|
||||||
|
|
||||||
import ch.qos.logback.classic.spi.LoggingEvent;
|
|
||||||
import org.apache.commons.lang3.StringUtils;
|
|
||||||
import org.apache.nifi.util.NiFiProperties;
|
|
||||||
import org.junit.jupiter.api.AfterEach;
|
|
||||||
import org.junit.jupiter.api.BeforeEach;
|
|
||||||
import org.junit.jupiter.api.Test;
|
|
||||||
|
|
||||||
import java.io.File;
|
|
||||||
import java.io.FileWriter;
|
|
||||||
import java.io.IOException;
|
|
||||||
import java.io.PrintWriter;
|
|
||||||
import java.util.Optional;
|
|
||||||
|
|
||||||
import static org.junit.jupiter.api.Assertions.assertTrue;
|
|
||||||
|
|
||||||
public class NiFiTest {
|
|
||||||
private static final String[] ARGUMENTS = new String[]{};
|
|
||||||
|
|
||||||
private static final String KEY_ARGUMENT = "-K";
|
|
||||||
|
|
||||||
private static final String[] FILE_NOT_SPECIFIED_ARGUMENTS = new String[]{KEY_ARGUMENT};
|
|
||||||
|
|
||||||
private static final String FAILURE_TO_LAUNCH = "Failure to launch NiFi";
|
|
||||||
|
|
||||||
private static final String PROPERTIES_LOADED = "Application Properties loaded";
|
|
||||||
|
|
||||||
private static final String PROPERTIES_PATH = "/NiFiProperties/conf/nifi.properties";
|
|
||||||
|
|
||||||
private static final String ENCRYPTED_PROPERTIES_PATH = "/NiFiProperties/conf/encrypted.nifi.properties";
|
|
||||||
|
|
||||||
private static final String ROOT_KEY = StringUtils.repeat("0", 64);
|
|
||||||
|
|
||||||
@BeforeEach
|
|
||||||
public void setAppender() {
|
|
||||||
ListAppender.clear();
|
|
||||||
}
|
|
||||||
|
|
||||||
@AfterEach
|
|
||||||
public void clearPropertiesFilePath() {
|
|
||||||
System.setProperty(NiFiProperties.PROPERTIES_FILE_PATH, StringUtils.EMPTY);
|
|
||||||
}
|
|
||||||
|
|
||||||
@Test
|
|
||||||
public void testMainBootstrapKeyFileNotSpecified() {
|
|
||||||
setPropertiesFilePath(PROPERTIES_PATH);
|
|
||||||
|
|
||||||
NiFi.main(FILE_NOT_SPECIFIED_ARGUMENTS);
|
|
||||||
|
|
||||||
assertFailureToLaunch();
|
|
||||||
}
|
|
||||||
|
|
||||||
@Test
|
|
||||||
public void testMainBootstrapKeyNotSpecified() {
|
|
||||||
setPropertiesFilePath(PROPERTIES_PATH);
|
|
||||||
|
|
||||||
NiFi.main(ARGUMENTS);
|
|
||||||
|
|
||||||
assertFailureToLaunch();
|
|
||||||
}
|
|
||||||
|
|
||||||
@Test
|
|
||||||
public void testMainEncryptedNiFiProperties() throws IOException {
|
|
||||||
final File rootKeyFile = File.createTempFile(getClass().getSimpleName(), ".root.key");
|
|
||||||
rootKeyFile.deleteOnExit();
|
|
||||||
try (final PrintWriter writer = new PrintWriter(new FileWriter(rootKeyFile))) {
|
|
||||||
writer.println(ROOT_KEY);
|
|
||||||
}
|
|
||||||
|
|
||||||
setPropertiesFilePath(ENCRYPTED_PROPERTIES_PATH);
|
|
||||||
|
|
||||||
NiFi.main(new String[]{KEY_ARGUMENT, rootKeyFile.getAbsolutePath()});
|
|
||||||
|
|
||||||
assertApplicationPropertiesLoaded();
|
|
||||||
assertFailureToLaunch();
|
|
||||||
}
|
|
||||||
|
|
||||||
private void assertApplicationPropertiesLoaded() {
|
|
||||||
final Optional<LoggingEvent> event = ListAppender.getLoggingEvents().stream().filter(
|
|
||||||
loggingEvent -> loggingEvent.getMessage().startsWith(PROPERTIES_LOADED)
|
|
||||||
).findFirst();
|
|
||||||
assertTrue(event.isPresent(), "Properties loaded log not found");
|
|
||||||
}
|
|
||||||
|
|
||||||
private void assertFailureToLaunch() {
|
|
||||||
final Optional<LoggingEvent> event = ListAppender.getLoggingEvents().stream().filter(
|
|
||||||
loggingEvent -> loggingEvent.getMessage().startsWith(FAILURE_TO_LAUNCH)
|
|
||||||
).findFirst();
|
|
||||||
assertTrue(event.isPresent(), "Failure log not found");
|
|
||||||
}
|
|
||||||
|
|
||||||
private void setPropertiesFilePath(final String relativePath) {
|
|
||||||
System.setProperty(NiFiProperties.PROPERTIES_FILE_PATH, getResourcesPath(relativePath));
|
|
||||||
}
|
|
||||||
|
|
||||||
private String getResourcesPath(final String relativePath) {
|
|
||||||
return getClass().getResource(relativePath).getPath();
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
@ -1,16 +0,0 @@
|
||||||
# Licensed to the Apache Software Foundation (ASF) under one or more
|
|
||||||
# contributor license agreements. See the NOTICE file distributed with
|
|
||||||
# this work for additional information regarding copyright ownership.
|
|
||||||
# The ASF licenses this file to You under the Apache License, Version 2.0
|
|
||||||
# (the "License"); you may not use this file except in compliance with
|
|
||||||
# the License. You may obtain a copy of the License at
|
|
||||||
#
|
|
||||||
# http://www.apache.org/licenses/LICENSE-2.0
|
|
||||||
#
|
|
||||||
# Unless required by applicable law or agreed to in writing, software
|
|
||||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
||||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
||||||
# See the License for the specific language governing permissions and
|
|
||||||
# limitations under the License.
|
|
||||||
|
|
||||||
|
|
||||||
|
|
@ -1,183 +0,0 @@
|
||||||
# Licensed to the Apache Software Foundation (ASF) under one or more
|
|
||||||
# contributor license agreements. See the NOTICE file distributed with
|
|
||||||
# this work for additional information regarding copyright ownership.
|
|
||||||
# The ASF licenses this file to You under the Apache License, Version 2.0
|
|
||||||
# (the "License"); you may not use this file except in compliance with
|
|
||||||
# the License. You may obtain a copy of the License at
|
|
||||||
#
|
|
||||||
# http://www.apache.org/licenses/LICENSE-2.0
|
|
||||||
#
|
|
||||||
# Unless required by applicable law or agreed to in writing, software
|
|
||||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
||||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
||||||
# See the License for the specific language governing permissions and
|
|
||||||
# limitations under the License.
|
|
||||||
|
|
||||||
# Core Properties #
|
|
||||||
nifi.flow.configuration.file=./target/conf/flow.json.gz
|
|
||||||
nifi.flow.configuration.archive.enabled=true
|
|
||||||
nifi.flow.configuration.archive.dir=./target/conf/archive/
|
|
||||||
nifi.flow.configuration.archive.max.time=30 days
|
|
||||||
nifi.flow.configuration.archive.max.storage=500 MB
|
|
||||||
nifi.flowcontroller.autoResumeState=true
|
|
||||||
nifi.flowcontroller.graceful.shutdown.period=10 sec
|
|
||||||
nifi.flowservice.writedelay.interval=500 ms
|
|
||||||
nifi.administrative.yield.duration=30 sec
|
|
||||||
# If a component has no work to do (is "bored"), how long should we wait before checking again for work?
|
|
||||||
nifi.bored.yield.duration=10 millis
|
|
||||||
|
|
||||||
nifi.authorizer.configuration.file=./target/conf/authorizers.xml
|
|
||||||
nifi.login.identity.provider.configuration.file=./target/conf/login-identity-providers.xml
|
|
||||||
nifi.ui.banner.text=dXwnu9mLyPETJrq1||n9e5dk5+HSTBCGOA/Sy6VYzwPw3baeRNvglalA1Pr1PcToyc4/qT6md24YOP4xVz14jd
|
|
||||||
nifi.ui.banner.text.protected=aes/gcm/256
|
|
||||||
nifi.nar.library.directory=./target/lib
|
|
||||||
nifi.nar.working.directory=./target/work/nar/
|
|
||||||
nifi.documentation.working.directory=./target/work/docs/components
|
|
||||||
|
|
||||||
####################
|
|
||||||
# State Management #
|
|
||||||
####################
|
|
||||||
nifi.state.management.configuration.file=./target/conf/state-management.xml
|
|
||||||
# The ID of the local state provider
|
|
||||||
nifi.state.management.provider.local=local-provider
|
|
||||||
# The ID of the cluster-wide state provider. This will be ignored if NiFi is not clustered but must be populated if running in a cluster.
|
|
||||||
nifi.state.management.provider.cluster=zk-provider
|
|
||||||
# Specifies whether or not this instance of NiFi should run an embedded ZooKeeper server
|
|
||||||
nifi.state.management.embedded.zookeeper.start=false
|
|
||||||
# Properties file that provides the ZooKeeper properties to use if <nifi.state.management.embedded.zookeeper.start> is set to true
|
|
||||||
nifi.state.management.embedded.zookeeper.properties=./target/conf/zookeeper.properties
|
|
||||||
|
|
||||||
|
|
||||||
# Database Settings
|
|
||||||
nifi.database.directory=./target/database_repository
|
|
||||||
|
|
||||||
# FlowFile Repository
|
|
||||||
nifi.flowfile.repository.implementation=org.apache.nifi.controller.repository.WriteAheadFlowFileRepository
|
|
||||||
nifi.flowfile.repository.directory=./target/flowfile_repository
|
|
||||||
nifi.flowfile.repository.partitions=256
|
|
||||||
nifi.flowfile.repository.checkpoint.interval=2 mins
|
|
||||||
nifi.flowfile.repository.always.sync=false
|
|
||||||
|
|
||||||
nifi.swap.manager.implementation=org.apache.nifi.controller.FileSystemSwapManager
|
|
||||||
nifi.queue.swap.threshold=20000
|
|
||||||
nifi.swap.in.period=5 sec
|
|
||||||
nifi.swap.in.threads=1
|
|
||||||
nifi.swap.out.period=5 sec
|
|
||||||
nifi.swap.out.threads=4
|
|
||||||
|
|
||||||
# Content Repository
|
|
||||||
nifi.content.repository.implementation=org.apache.nifi.controller.repository.FileSystemRepository
|
|
||||||
nifi.content.claim.max.appendable.size=10 MB
|
|
||||||
nifi.content.claim.max.flow.files=100
|
|
||||||
nifi.content.repository.directory.default=./target/content_repository
|
|
||||||
nifi.content.repository.archive.max.retention.period=12 hours
|
|
||||||
nifi.content.repository.archive.max.usage.percentage=50%
|
|
||||||
nifi.content.repository.archive.enabled=true
|
|
||||||
nifi.content.repository.always.sync=false
|
|
||||||
nifi.content.viewer.url=/nifi-content-viewer/
|
|
||||||
|
|
||||||
# Provenance Repository Properties
|
|
||||||
nifi.provenance.repository.implementation=org.apache.nifi.provenance.WriteAheadProvenanceRepository
|
|
||||||
|
|
||||||
# Persistent Provenance Repository Properties
|
|
||||||
nifi.provenance.repository.directory.default=./target/provenance_repository
|
|
||||||
nifi.provenance.repository.max.storage.time=24 hours
|
|
||||||
nifi.provenance.repository.max.storage.size=1 GB
|
|
||||||
nifi.provenance.repository.rollover.time=30 secs
|
|
||||||
nifi.provenance.repository.rollover.size=100 MB
|
|
||||||
nifi.provenance.repository.query.threads=2
|
|
||||||
nifi.provenance.repository.index.threads=1
|
|
||||||
nifi.provenance.repository.compress.on.rollover=true
|
|
||||||
nifi.provenance.repository.always.sync=false
|
|
||||||
nifi.provenance.repository.journal.count=16
|
|
||||||
# Comma-separated list of fields. Fields that are not indexed will not be searchable. Valid fields are:
|
|
||||||
# EventType, FlowFileUUID, Filename, TransitURI, ProcessorID, AlternateIdentifierURI, Relationship, Details
|
|
||||||
nifi.provenance.repository.indexed.fields=EventType, FlowFileUUID, Filename, ProcessorID, Relationship
|
|
||||||
# FlowFile Attributes that should be indexed and made searchable. Some examples to consider are filename, uuid, mime.type
|
|
||||||
nifi.provenance.repository.indexed.attributes=
|
|
||||||
# Large values for the shard size will result in more Java heap usage when searching the Provenance Repository
|
|
||||||
# but should provide better performance
|
|
||||||
nifi.provenance.repository.index.shard.size=500 MB
|
|
||||||
# Indicates the maximum length that a FlowFile attribute can be when retrieving a Provenance Event from
|
|
||||||
# the repository. If the length of any attribute exceeds this value, it will be truncated when the event is retrieved.
|
|
||||||
nifi.provenance.repository.max.attribute.length=65536
|
|
||||||
|
|
||||||
# Volatile Provenance Respository Properties
|
|
||||||
nifi.provenance.repository.buffer.size=100000
|
|
||||||
|
|
||||||
# Component Status Repository
|
|
||||||
nifi.components.status.repository.implementation=org.apache.nifi.controller.status.history.VolatileComponentStatusRepository
|
|
||||||
nifi.components.status.repository.buffer.size=1440
|
|
||||||
nifi.components.status.snapshot.frequency=1 min
|
|
||||||
|
|
||||||
# Site to Site properties
|
|
||||||
nifi.remote.input.host=
|
|
||||||
nifi.remote.input.secure=false
|
|
||||||
nifi.remote.input.socket.port=
|
|
||||||
nifi.remote.input.http.enabled=true
|
|
||||||
nifi.remote.input.http.transaction.ttl=30 sec
|
|
||||||
|
|
||||||
# web properties #
|
|
||||||
nifi.web.war.directory=./target/lib
|
|
||||||
nifi.web.http.host=
|
|
||||||
nifi.web.http.port=8080
|
|
||||||
nifi.web.https.host=
|
|
||||||
nifi.web.https.port=
|
|
||||||
nifi.web.jetty.working.directory=./target/work/jetty
|
|
||||||
nifi.web.jetty.threads=200
|
|
||||||
|
|
||||||
# security properties #
|
|
||||||
nifi.sensitive.props.key=dQU402Mz4J+t+e18||6+ictR0Nssq3/rR/d8fq5CFAKmpakr9jCyPIJYxG7n6D86gxsu2TRp4M48ugUw==
|
|
||||||
nifi.sensitive.props.key.protected=aes/gcm/256
|
|
||||||
nifi.sensitive.props.algorithm=NIFI_PBKDF2_AES_GCM_256
|
|
||||||
nifi.sensitive.props.additional.keys=nifi.ui.banner.text
|
|
||||||
|
|
||||||
nifi.security.keystore=/path/to/keystore.jks
|
|
||||||
nifi.security.keystoreType=JKS
|
|
||||||
nifi.security.keystorePasswd=Q8T3wv+Xl2ie98GV||qsuY9wa/Rt27cqFXs8ebX25E1iSbFAEFcD0cjCwrl3Tw6HghQjBIaCzQ
|
|
||||||
nifi.security.keystorePasswd.protected=aes/gcm/256
|
|
||||||
nifi.security.keyPasswd=1S0XmoiAr379B8rg||PPZzjdw9BAJSon9g4xm9uscFhCCyk734FTjXtRnBXUy819zsoQ==
|
|
||||||
nifi.security.keyPasswd.protected=aes/gcm/256
|
|
||||||
nifi.security.truststore=
|
|
||||||
nifi.security.truststoreType=
|
|
||||||
nifi.security.truststorePasswd=
|
|
||||||
nifi.security.user.authorizer=file-provider
|
|
||||||
nifi.security.user.login.identity.provider=
|
|
||||||
nifi.security.ocsp.responder.url=
|
|
||||||
nifi.security.ocsp.responder.certificate=
|
|
||||||
|
|
||||||
# Identity Mapping Properties #
|
|
||||||
# These properties allow normalizing user identities such that identities coming from different identity providers
|
|
||||||
# (certificates, LDAP, Kerberos) can be treated the same internally in NiFi. The following example demonstrates normalizing
|
|
||||||
# DNs from certificates and principals from Kerberos into a common identity string:
|
|
||||||
#
|
|
||||||
# nifi.security.identity.mapping.pattern.dn=^CN=(.*?), OU=(.*?), O=(.*?), L=(.*?), ST=(.*?), C=(.*?)$
|
|
||||||
# nifi.security.identity.mapping.value.dn=$1@$2
|
|
||||||
# nifi.security.identity.mapping.pattern.kerb=^(.*?)/instance@(.*?)$
|
|
||||||
# nifi.security.identity.mapping.value.kerb=$1@$2
|
|
||||||
|
|
||||||
# cluster common properties (all nodes must have same values) #
|
|
||||||
nifi.cluster.protocol.heartbeat.interval=5 sec
|
|
||||||
nifi.cluster.protocol.is.secure=false
|
|
||||||
|
|
||||||
# cluster node properties (only configure for cluster nodes) #
|
|
||||||
nifi.cluster.is.node=false
|
|
||||||
nifi.cluster.node.address=
|
|
||||||
nifi.cluster.node.protocol.port=
|
|
||||||
nifi.cluster.node.protocol.threads=10
|
|
||||||
nifi.cluster.node.event.history.size=25
|
|
||||||
nifi.cluster.node.connection.timeout=5 sec
|
|
||||||
nifi.cluster.node.read.timeout=5 sec
|
|
||||||
nifi.cluster.firewall.file=
|
|
||||||
|
|
||||||
# zookeeper properties, used for cluster management #
|
|
||||||
nifi.zookeeper.connect.string=
|
|
||||||
nifi.zookeeper.connect.timeout=3 secs
|
|
||||||
nifi.zookeeper.session.timeout=3 secs
|
|
||||||
nifi.zookeeper.root.node=/nifi
|
|
||||||
|
|
||||||
# kerberos #
|
|
||||||
nifi.kerberos.krb5.file=
|
|
||||||
nifi.kerberos.service.principal=
|
|
||||||
nifi.kerberos.keytab.location=
|
|
||||||
nifi.kerberos.authentication.expiration=12 hours
|
|
||||||
|
|
@ -1,173 +0,0 @@
|
||||||
# Licensed to the Apache Software Foundation (ASF) under one or more
|
|
||||||
# contributor license agreements. See the NOTICE file distributed with
|
|
||||||
# this work for additional information regarding copyright ownership.
|
|
||||||
# The ASF licenses this file to You under the Apache License, Version 2.0
|
|
||||||
# (the "License"); you may not use this file except in compliance with
|
|
||||||
# the License. You may obtain a copy of the License at
|
|
||||||
#
|
|
||||||
# http://www.apache.org/licenses/LICENSE-2.0
|
|
||||||
#
|
|
||||||
# Unless required by applicable law or agreed to in writing, software
|
|
||||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
||||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
||||||
# See the License for the specific language governing permissions and
|
|
||||||
# limitations under the License.
|
|
||||||
|
|
||||||
# Core Properties #
|
|
||||||
nifi.flow.configuration.file=./target/conf/flow.json.gz
|
|
||||||
nifi.flow.configuration.archive.enabled=true
|
|
||||||
nifi.flow.configuration.archive.dir=./target/conf/archive/
|
|
||||||
nifi.flow.configuration.archive.max.time=30 days
|
|
||||||
nifi.flow.configuration.archive.max.storage=500 MB
|
|
||||||
nifi.flowcontroller.autoResumeState=true
|
|
||||||
nifi.flowcontroller.graceful.shutdown.period=10 sec
|
|
||||||
nifi.flowservice.writedelay.interval=500 ms
|
|
||||||
nifi.administrative.yield.duration=30 sec
|
|
||||||
# If a component has no work to do (is "bored"), how long should we wait before checking again for work?
|
|
||||||
nifi.bored.yield.duration=10 millis
|
|
||||||
|
|
||||||
nifi.authorizer.configuration.file=./target/conf/authorizers.xml
|
|
||||||
nifi.login.identity.provider.configuration.file=./target/conf/login-identity-providers.xml
|
|
||||||
nifi.nar.library.directory=./target/lib
|
|
||||||
nifi.nar.working.directory=./target/work/nar/
|
|
||||||
nifi.documentation.working.directory=./target/work/docs/components
|
|
||||||
|
|
||||||
####################
|
|
||||||
# State Management #
|
|
||||||
####################
|
|
||||||
nifi.state.management.configuration.file=./target/conf/state-management.xml
|
|
||||||
# The ID of the local state provider
|
|
||||||
nifi.state.management.provider.local=local-provider
|
|
||||||
# The ID of the cluster-wide state provider. This will be ignored if NiFi is not clustered but must be populated if running in a cluster.
|
|
||||||
nifi.state.management.provider.cluster=zk-provider
|
|
||||||
# Specifies whether or not this instance of NiFi should run an embedded ZooKeeper server
|
|
||||||
nifi.state.management.embedded.zookeeper.start=false
|
|
||||||
# Properties file that provides the ZooKeeper properties to use if <nifi.state.management.embedded.zookeeper.start> is set to true
|
|
||||||
nifi.state.management.embedded.zookeeper.properties=./target/conf/zookeeper.properties
|
|
||||||
|
|
||||||
|
|
||||||
# Database Settings
|
|
||||||
nifi.database.directory=./target/database_repository
|
|
||||||
|
|
||||||
# FlowFile Repository
|
|
||||||
nifi.flowfile.repository.implementation=org.apache.nifi.controller.repository.WriteAheadFlowFileRepository
|
|
||||||
nifi.flowfile.repository.directory=./target/flowfile_repository
|
|
||||||
nifi.flowfile.repository.partitions=256
|
|
||||||
nifi.flowfile.repository.checkpoint.interval=2 mins
|
|
||||||
nifi.flowfile.repository.always.sync=false
|
|
||||||
|
|
||||||
nifi.swap.manager.implementation=org.apache.nifi.controller.FileSystemSwapManager
|
|
||||||
nifi.queue.swap.threshold=20000
|
|
||||||
nifi.swap.in.period=5 sec
|
|
||||||
nifi.swap.in.threads=1
|
|
||||||
nifi.swap.out.period=5 sec
|
|
||||||
nifi.swap.out.threads=4
|
|
||||||
|
|
||||||
# Content Repository
|
|
||||||
nifi.content.repository.implementation=org.apache.nifi.controller.repository.FileSystemRepository
|
|
||||||
nifi.content.claim.max.appendable.size=10 MB
|
|
||||||
nifi.content.claim.max.flow.files=100
|
|
||||||
nifi.content.repository.directory.default=./target/content_repository
|
|
||||||
nifi.content.repository.archive.max.retention.period=12 hours
|
|
||||||
nifi.content.repository.archive.max.usage.percentage=50%
|
|
||||||
nifi.content.repository.archive.enabled=true
|
|
||||||
nifi.content.repository.always.sync=false
|
|
||||||
nifi.content.viewer.url=/nifi-content-viewer/
|
|
||||||
|
|
||||||
# Provenance Repository Properties
|
|
||||||
nifi.provenance.repository.implementation=org.apache.nifi.provenance.WriteAheadProvenanceRepository
|
|
||||||
|
|
||||||
# Persistent Provenance Repository Properties
|
|
||||||
nifi.provenance.repository.directory.default=./target/provenance_repository
|
|
||||||
nifi.provenance.repository.max.storage.time=24 hours
|
|
||||||
nifi.provenance.repository.max.storage.size=1 GB
|
|
||||||
nifi.provenance.repository.rollover.time=30 secs
|
|
||||||
nifi.provenance.repository.rollover.size=100 MB
|
|
||||||
nifi.provenance.repository.query.threads=2
|
|
||||||
nifi.provenance.repository.index.threads=1
|
|
||||||
nifi.provenance.repository.compress.on.rollover=true
|
|
||||||
nifi.provenance.repository.always.sync=false
|
|
||||||
nifi.provenance.repository.journal.count=16
|
|
||||||
# Comma-separated list of fields. Fields that are not indexed will not be searchable. Valid fields are:
|
|
||||||
# EventType, FlowFileUUID, Filename, TransitURI, ProcessorID, AlternateIdentifierURI, Relationship, Details
|
|
||||||
nifi.provenance.repository.indexed.fields=EventType, FlowFileUUID, Filename, ProcessorID, Relationship
|
|
||||||
# FlowFile Attributes that should be indexed and made searchable. Some examples to consider are filename, uuid, mime.type
|
|
||||||
nifi.provenance.repository.indexed.attributes=
|
|
||||||
# Large values for the shard size will result in more Java heap usage when searching the Provenance Repository
|
|
||||||
# but should provide better performance
|
|
||||||
nifi.provenance.repository.index.shard.size=500 MB
|
|
||||||
# Indicates the maximum length that a FlowFile attribute can be when retrieving a Provenance Event from
|
|
||||||
# the repository. If the length of any attribute exceeds this value, it will be truncated when the event is retrieved.
|
|
||||||
nifi.provenance.repository.max.attribute.length=65536
|
|
||||||
|
|
||||||
# Volatile Provenance Respository Properties
|
|
||||||
nifi.provenance.repository.buffer.size=100000
|
|
||||||
|
|
||||||
# Component Status Repository
|
|
||||||
nifi.components.status.repository.implementation=org.apache.nifi.controller.status.history.VolatileComponentStatusRepository
|
|
||||||
nifi.components.status.repository.buffer.size=1440
|
|
||||||
nifi.components.status.snapshot.frequency=1 min
|
|
||||||
|
|
||||||
# Site to Site properties
|
|
||||||
nifi.remote.input.host=
|
|
||||||
nifi.remote.input.secure=false
|
|
||||||
nifi.remote.input.socket.port=
|
|
||||||
nifi.remote.input.http.enabled=true
|
|
||||||
nifi.remote.input.http.transaction.ttl=30 sec
|
|
||||||
|
|
||||||
# web properties #
|
|
||||||
nifi.web.war.directory=./target/lib
|
|
||||||
nifi.web.http.host=
|
|
||||||
nifi.web.http.port=8080
|
|
||||||
nifi.web.https.host=
|
|
||||||
nifi.web.https.port=
|
|
||||||
nifi.web.jetty.working.directory=./target/work/jetty
|
|
||||||
nifi.web.jetty.threads=200
|
|
||||||
|
|
||||||
nifi.security.keystore=
|
|
||||||
nifi.security.keystoreType=
|
|
||||||
nifi.security.keystorePasswd=
|
|
||||||
nifi.security.keyPasswd=
|
|
||||||
nifi.security.truststore=
|
|
||||||
nifi.security.truststoreType=
|
|
||||||
nifi.security.truststorePasswd=
|
|
||||||
nifi.security.user.authorizer=file-provider
|
|
||||||
nifi.security.user.login.identity.provider=
|
|
||||||
nifi.security.ocsp.responder.url=
|
|
||||||
nifi.security.ocsp.responder.certificate=
|
|
||||||
|
|
||||||
# Identity Mapping Properties #
|
|
||||||
# These properties allow normalizing user identities such that identities coming from different identity providers
|
|
||||||
# (certificates, LDAP, Kerberos) can be treated the same internally in NiFi. The following example demonstrates normalizing
|
|
||||||
# DNs from certificates and principals from Kerberos into a common identity string:
|
|
||||||
#
|
|
||||||
# nifi.security.identity.mapping.pattern.dn=^CN=(.*?), OU=(.*?), O=(.*?), L=(.*?), ST=(.*?), C=(.*?)$
|
|
||||||
# nifi.security.identity.mapping.value.dn=$1@$2
|
|
||||||
# nifi.security.identity.mapping.pattern.kerb=^(.*?)/instance@(.*?)$
|
|
||||||
# nifi.security.identity.mapping.value.kerb=$1@$2
|
|
||||||
|
|
||||||
# cluster common properties (all nodes must have same values) #
|
|
||||||
nifi.cluster.protocol.heartbeat.interval=5 sec
|
|
||||||
nifi.cluster.protocol.is.secure=false
|
|
||||||
|
|
||||||
# cluster node properties (only configure for cluster nodes) #
|
|
||||||
nifi.cluster.is.node=false
|
|
||||||
nifi.cluster.node.address=
|
|
||||||
nifi.cluster.node.protocol.port=
|
|
||||||
nifi.cluster.node.protocol.threads=10
|
|
||||||
nifi.cluster.node.event.history.size=25
|
|
||||||
nifi.cluster.node.connection.timeout=5 sec
|
|
||||||
nifi.cluster.node.read.timeout=5 sec
|
|
||||||
nifi.cluster.firewall.file=
|
|
||||||
|
|
||||||
# zookeeper properties, used for cluster management #
|
|
||||||
nifi.zookeeper.connect.string=
|
|
||||||
nifi.zookeeper.connect.timeout=3 secs
|
|
||||||
nifi.zookeeper.session.timeout=3 secs
|
|
||||||
nifi.zookeeper.root.node=/nifi
|
|
||||||
|
|
||||||
# kerberos #
|
|
||||||
nifi.kerberos.krb5.file=
|
|
||||||
nifi.kerberos.service.principal=
|
|
||||||
nifi.kerberos.keytab.location=
|
|
||||||
nifi.kerberos.authentication.expiration=12 hours
|
|
||||||
|
|
@ -1,28 +0,0 @@
|
||||||
<?xml version="1.0" encoding="UTF-8"?>
|
|
||||||
<!--
|
|
||||||
Licensed to the Apache Software Foundation (ASF) under one or more
|
|
||||||
contributor license agreements. See the NOTICE file distributed with
|
|
||||||
this work for additional information regarding copyright ownership.
|
|
||||||
The ASF licenses this file to You under the Apache License, Version 2.0
|
|
||||||
(the "License"); you may not use this file except in compliance with
|
|
||||||
the License. You may obtain a copy of the License at
|
|
||||||
http://www.apache.org/licenses/LICENSE-2.0
|
|
||||||
Unless required by applicable law or agreed to in writing, software
|
|
||||||
distributed under the License is distributed on an "AS IS" BASIS,
|
|
||||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
||||||
See the License for the specific language governing permissions and
|
|
||||||
limitations under the License.
|
|
||||||
-->
|
|
||||||
|
|
||||||
<configuration>
|
|
||||||
<appender name="TEST" class="org.apache.nifi.ListAppender">
|
|
||||||
<encoder class="ch.qos.logback.classic.encoder.PatternLayoutEncoder">
|
|
||||||
<pattern>%-4r [%t] %-5p %c - %m%n</pattern>
|
|
||||||
</encoder>
|
|
||||||
</appender>
|
|
||||||
|
|
||||||
<logger name="org.apache.nifi.processor" level="DEBUG"/>
|
|
||||||
<root level="DEBUG">
|
|
||||||
<appender-ref ref="TEST"/>
|
|
||||||
</root>
|
|
||||||
</configuration>
|
|
||||||
Loading…
Reference in New Issue