Apache
/
nifi
mirror of https://github.com/apache/nifi.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

NIFI-9861: Removed BlockListClassLoader in favor of AllowListClassLoader 

NIFI-9861: Fixed stateless-processor-tests assembly to ensure that all necessary libraries were included; removed BlockListClassLoader
NIFI-9861: Fixed issue in which we would list .class files as files that we allow through the AllowListClassLoader but didn't allow them.

This closes #5925.

Signed-off-by: Peter Turcsanyi <turcsanyi@apache.org>
This commit is contained in:
Mark Payne 2022-04-01 12:59:01 -04:00 committed by Peter Turcsanyi
parent c88c404dd8
commit aeaec170d5
5 changed files with 124 additions and 94 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
nifi-nar-bundles/nifi-stateless-processor-bundle/nifi-stateless-processor-tests/pom.xml
View File

@ -74,6 +74,11 @@
<artifactId>nifi-registry-data-model</artifactId>
<version>1.17.0-SNAPSHOT</version>
</dependency>
<dependency>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-api</artifactId>
<scope>runtime</scope>
</dependency>
<dependency>
<groupId>org.apache.nifi</groupId>

2
nifi-nar-bundles/nifi-stateless-processor-bundle/nifi-stateless-processor-tests/src/test/assembly/dependencies.xml
View File

@ -39,6 +39,8 @@
<include>nifi-server-api</include>
<include>nifi-runtime</include>
<include>nifi-nar-utils</include>
<include>nifi-stateless-api</include>
<include>slf4j-api</include>
</includes>
</dependencySet>

33
nifi-stateless/nifi-stateless-bootstrap/src/main/java/org/apache/nifi/stateless/bootstrap/BlockListClassLoader.java → nifi-stateless/nifi-stateless-bootstrap/src/main/java/org/apache/nifi/stateless/bootstrap/AllowListClassLoader.java
View File

@ -22,7 +22,7 @@ import java.util.Set;
/**
* <p>
* A ClassLoader that blocks a specific set of selected classes from being loaded by its parent. This ClassLoader does not load any classes itself
* A ClassLoader that allows only a specific set of selected classes to be loaded by its parent. This ClassLoader does not load any classes itself
* but serves as a mechanism for preventing unwanted classes from a parent from being used.
* </p>
* <p>
@ -32,40 +32,41 @@ import java.util.Set;
* </p>
* <p>
* Because we cannot control what is loaded by the System ClassLoader (that's up to the embedding application), the best that we can do is to block NiFi's extensions'
* ClassLoaders from accessing those classes. This ClassLoader allows us to do just that, blocking specific classes that have been loaded by the parent ClassLoader
* from being accessible by child ClassLoaders.
* ClassLoaders from accessing those classes. This ClassLoader allows us to do just that, allowing only specific classes that have been loaded by the parent ClassLoader
* to be visible/accessible by child ClassLoaders.
* </p>
*/
public class BlockListClassLoader extends ClassLoader {
private final Set<String> blockList;
public class AllowListClassLoader extends ClassLoader {
private final Set<String> allowed;
public BlockListClassLoader(final ClassLoader parent, final Set<String> blockList) {
public AllowListClassLoader(final ClassLoader parent, final Set<String> allowed) {
super(parent);
this.blockList = blockList;
this.allowed = allowed;
}
/**
* @return the set of all Class names that will be blocked from loading by the parent
* @return the set of all Class names that will not be blocked from loading by the parent
*/
public Set<String> getClassesBlocked() {
return Collections.unmodifiableSet(blockList);
public Set<String> getClassesAllowed() {
return Collections.unmodifiableSet(allowed);
}
@Override
protected Class<?> loadClass(final String name, final boolean resolve) throws ClassNotFoundException {
if (blockList.contains(name)) {
throw new ClassNotFoundException(name + " was blocked by BlockListClassLoader");
if (allowed.contains(name)) {
return super.loadClass(name, resolve);
}
return super.loadClass(name, resolve);
throw new ClassNotFoundException(name + " was blocked by AllowListClassLoader");
}
@Override
protected Class<?> findClass(final String name) throws ClassNotFoundException {
if (blockList.contains(name)) {
throw new ClassNotFoundException(name + " was blocked by BlockListClassLoader");
if (allowed.contains(name)) {
return super.findClass(name);
}
return super.findClass(name);
throw new ClassNotFoundException(name + " was blocked by AllowListClassLoader");
}
}

167
nifi-stateless/nifi-stateless-bootstrap/src/main/java/org/apache/nifi/stateless/bootstrap/StatelessBootstrap.java
View File

@ -36,13 +36,8 @@ import org.slf4j.LoggerFactory;
import java.io.File;
import java.io.IOException;
import java.net.URISyntaxException;
import java.net.URL;
import java.net.URLClassLoader;
import java.nio.file.Path;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
import java.util.Enumeration;
import java.util.HashSet;
@ -54,6 +49,7 @@ import java.util.function.Predicate;
import java.util.jar.JarFile;
import java.util.regex.Pattern;
import java.util.zip.ZipEntry;
import java.util.zip.ZipFile;
public class StatelessBootstrap {
private static final Logger logger = LoggerFactory.getLogger(StatelessBootstrap.class);
@ -120,7 +116,7 @@ public class StatelessBootstrap {
final long unpackMillis = System.currentTimeMillis() - unpackStart;
logger.info("Unpacked NAR files in {} millis", unpackMillis);
final BlockListClassLoader statelessClassLoader = createExtensionRootClassLoader(narDirectory, rootClassLoader);
final AllowListClassLoader statelessClassLoader = createExtensionRootClassLoader(narDirectory, rootClassLoader);
final File statelessNarWorkingDir = locateStatelessNarWorkingDirectory(extensionsWorkingDir);
final NarClassLoader engineClassLoader;
@ -136,7 +132,7 @@ public class StatelessBootstrap {
/**
* Creates a ClassLoader that is to be used as the 'root'/parent for all NiFi Extensions' ClassLoaders. The ClassLoader will inherit from its parent
* any classes that exist in JAR files that can be found in the given NAR Directory. However, it will not allow any other classes to be loaded from the parent.
* any classes that exist in JAR files that can be found in the given NAR Directory or within the Java home directory. However, it will not allow any other classes to be loaded from the parent.
* This approach is important because we need to ensure that the ClassLoader that is provided to extensions when run from NiFi Stateless is the same as the ClassLoader
* that will be provided to it in traditional NiFi. Whereas in traditional NiFi, we have the ability to control the System ClassLoader, Stateless NiFi is designed to be
* embedded, so we cannot control the System ClassLoader of the embedding application. This gives us a way to ensure that we control what is available to Extensions and
@ -145,13 +141,9 @@ public class StatelessBootstrap {
*
* @param narDirectory the NAR directory whose .jar files should be made available via the parent.
* @param parent the parent class loader that the given BlockListClassLoader should delegate to for classes that it does not block
* @return a BlockListClassLoader that allows only the appropriate classes to be loaded from the given parent
* @return an AllowListClassLoader that allows only the appropriate classes to be loaded from the given parent
*/
private static BlockListClassLoader createExtensionRootClassLoader(final File narDirectory, final ClassLoader parent) throws IOException {
if (!(parent instanceof URLClassLoader)) {
return new BlockListClassLoader(parent, Collections.emptySet());
}
private static AllowListClassLoader createExtensionRootClassLoader(final File narDirectory, final ClassLoader parent) throws IOException {
final File[] narDirectoryFiles = narDirectory.listFiles();
if (narDirectoryFiles == null) {
throw new IOException("Could not get a listing of the NAR directory");
@ -159,13 +151,6 @@ public class StatelessBootstrap {
logger.debug("NAR directory used to find files to allow being loaded by Stateless Extension Classloaders from parent {}: {}", parent, narDirectory);
final Set<URL> urls = new HashSet<>();
findClassLoaderUrls(parent, urls);
final Set<String> classesBlocked = new HashSet<>();
final Set<String> filesBlocked = new HashSet<>();
findClassNamesInJars(urls, classesBlocked, filesBlocked);
final Set<String> classesAllowed = new HashSet<>();
final Set<String> filesAllowed = new HashSet<>();
for (final File file : narDirectoryFiles) {
@ -181,58 +166,75 @@ public class StatelessBootstrap {
filesAllowed.add(file.getName());
}
}
logger.debug("The following JAR files are proposed to be blocked from being loaded by Stateless Extensions ClassLoaders from parent {}: {}", parent, filesBlocked);
logger.debug("Of the full list above, the following JAR files will be explicitly allowed to be loaded by Stateless Extensions ClassLoaders from parent {}: {}", parent, filesAllowed);
classesBlocked.removeAll(classesAllowed);
filesBlocked.removeAll(filesAllowed);
logger.debug("The final list of JAR files blocked from being loaded by Stateless Extensions ClassLoaders from parent {}: {}", parent, filesBlocked);
logger.debug("The final list of classes blocked from being loaded by Stateless Extension ClassLoaders from parent {}: {}", parent, classesBlocked);
final BlockListClassLoader blockingClassLoader = new BlockListClassLoader(parent, classesBlocked);
return blockingClassLoader;
}
private static void findClassNamesInJars(final Collection<URL> jarUrls, final Set<String> classesFound, final Set<String> jarFilesFound) throws IOException {
final String javaHome = System.getProperty("java.home");
final File javaHomeDir = new File(javaHome);
final File javaLib = new File(javaHomeDir, "lib");
final String javaLibPath = javaLib.getAbsolutePath();
for (final URL url : jarUrls) {
final File file;
try {
file = new File(url.toURI());
} catch (URISyntaxException e) {
logger.warn("Could not find file for {} in classpath", url);
continue;
}
final String absolutePath = file.getAbsolutePath();
if (absolutePath.startsWith(javaLibPath)) {
continue;
}
findClassNamesInJar(file, classesFound);
jarFilesFound.add(file.getName());
final Set<File> javaHomeFiles = findJavaHomeFiles();
final Set<String> javaHomeFilenames = new HashSet<>();
for (final File file : javaHomeFiles) {
findLoadableClasses(file, classesAllowed);
javaHomeFilenames.add(file.getName());
}
logger.debug("The following JAR files will be explicitly allowed to be loaded by Stateless Extensions ClassLoaders from parent {}: {}", parent, filesAllowed);
logger.debug("The following JAR/JMOD files from ${JAVA_HOME} will be explicitly allowed to be loaded by Stateless Extensions ClassLoaders from parent {}: {}", parent, javaHomeFilenames);
logger.debug("The final list of classes allowed to be loaded by Stateless Extension ClassLoaders from parent {}: {}", parent, classesAllowed);
final AllowListClassLoader allowListClassLoader = new AllowListClassLoader(parent, classesAllowed);
return allowListClassLoader;
}
private static void findClassLoaderUrls(final ClassLoader classLoader, final Set<URL> urls) {
if (classLoader == null) {
private static Set<File> findJavaHomeFiles() {
final String javaHomeValue = System.getProperty("java.home");
if (javaHomeValue == null) {
logger.warn("Could not find java.home system property so will not allow any classes explicitly from java.home in AllowListClassLoader");
return Collections.emptySet();
}
final File javaHome = new File(javaHomeValue);
if (!javaHome.exists()) {
logger.warn("System property for java.home is {} but that directory does not exist so will not allow any classes explicitly from java.home in AllowListClassLoader", javaHomeValue);
return Collections.emptySet();
}
final File[] javaHomeFiles = javaHome.listFiles();
if (javaHomeFiles == null) {
logger.warn("System property for java.home is {} but that directory is not readable so will not allow any classes explicitly from java.home in AllowListClassLoader", javaHomeValue);
return Collections.emptySet();
}
final Set<File> loadableFiles = new HashSet<>();
for (final File file : javaHomeFiles) {
findLoadableFiles(file, loadableFiles);
}
return loadableFiles;
}
private static void findLoadableFiles(final File file, final Set<File> loadable) {
if (file.isDirectory()) {
final File[] children = file.listFiles();
if (children == null) {
return;
}
for (final File child : children) {
findLoadableFiles(child, loadable);
}
return;
}
if (classLoader instanceof URLClassLoader) {
final URLClassLoader urlClassLoader = (URLClassLoader) classLoader;
urls.addAll(Arrays.asList(urlClassLoader.getURLs()));
final String filename = file.getName();
if (filename.endsWith(".jar") || filename.endsWith(".jmod")) {
loadable.add(file);
}
}
// If the classLoader is the system class loader, we are done. We don't want to process the parent of
// the system class loader (which would be the Launcher$ExtClassLoader that contains the JDK/JRE classes, etc)
if (classLoader != ClassLoader.getSystemClassLoader()) {
findClassLoaderUrls(classLoader.getParent(), urls);
private static void findLoadableClasses(final File file, final Set<String> classNames) throws IOException {
final String filename = file.getName();
if (filename.endsWith(".jar")) {
findClassNamesInJar(file, classNames);
} else if (filename.endsWith(".jmod")) {
findClassesInJmod(file, classNames);
}
}
@ -241,16 +243,37 @@ public class StatelessBootstrap {
return;
}
final JarFile jarFile = new JarFile(file);
final Enumeration<? extends ZipEntry> enumeration = jarFile.entries();
while (enumeration.hasMoreElements()) {
final ZipEntry zipEntry = enumeration.nextElement();
final String entryName = zipEntry.getName();
try (final JarFile jarFile = new JarFile(file)) {
final Enumeration<? extends ZipEntry> enumeration = jarFile.entries();
while (enumeration.hasMoreElements()) {
final ZipEntry zipEntry = enumeration.nextElement();
final String entryName = zipEntry.getName();
if (entryName.endsWith(".class")) {
final int lastIndex = entryName.lastIndexOf(".class");
final String className = entryName.substring(0, lastIndex).replace("/", ".");
classNames.add(className);
if (entryName.endsWith(".class")) {
final int lastIndex = entryName.lastIndexOf(".class");
final String className = entryName.substring(0, lastIndex).replace("/", ".");
classNames.add(className);
}
}
}
}
private static void findClassesInJmod(final File file, final Set<String> classNames) throws IOException {
if (!file.getName().endsWith(".jmod") || !file.isFile() || !file.exists()) {
return;
}
try (final ZipFile zipFile = new ZipFile(file)) {
final Enumeration<? extends ZipEntry> enumeration = zipFile.entries();
while (enumeration.hasMoreElements()) {
final ZipEntry zipEntry = enumeration.nextElement();
final String entryName = zipEntry.getName();
if (entryName.startsWith("classes/") && entryName.endsWith(".class")) {
final int lastIndex = entryName.lastIndexOf(".class");
final String className = entryName.substring(8, lastIndex).replace("/", ".");
classNames.add(className);
}
}
}
}

11
nifi-system-tests/nifi-stateless-system-test-suite/pom.xml
View File

@ -73,20 +73,19 @@
<groupId>org.apache.nifi</groupId>
<artifactId>nifi-api</artifactId>
<version>1.17.0-SNAPSHOT</version>
<scope>test</scope>
<scope>runtime</scope>
</dependency>
<dependency>
<groupId>org.apache.nifi</groupId>
<artifactId>nifi-server-api</artifactId>
<version>1.17.0-SNAPSHOT</version>
<scope>test</scope>
<scope>runtime</scope>
</dependency>
<dependency>
<groupId>org.apache.nifi</groupId>
<artifactId>nifi-framework-api</artifactId>
<version>1.17.0-SNAPSHOT</version>
<scope>compile</scope>
<optional>true</optional>
<scope>runtime</scope>
</dependency>
<dependency>
<groupId>org.apache.nifi</groupId>
@ -98,7 +97,7 @@
<groupId>org.apache.nifi</groupId>
<artifactId>nifi-properties</artifactId>
<version>1.17.0-SNAPSHOT</version>
<scope>test</scope>
<scope>runtime</scope>
</dependency>
<dependency>
<groupId>org.slf4j</groupId>
@ -113,7 +112,7 @@
<groupId>javax.servlet</groupId>
<artifactId>javax.servlet-api</artifactId>
<version>3.1.0</version>
<scope>test</scope>
<scope>runtime</scope>
</dependency>