diff --git a/nifi-nar-bundles/nifi-extension-utils/nifi-hadoop-utils/src/main/java/org/apache/nifi/processors/hadoop/AbstractHadoopProcessor.java b/nifi-nar-bundles/nifi-extension-utils/nifi-hadoop-utils/src/main/java/org/apache/nifi/processors/hadoop/AbstractHadoopProcessor.java index 222effd25f..7c574a72f8 100644 --- a/nifi-nar-bundles/nifi-extension-utils/nifi-hadoop-utils/src/main/java/org/apache/nifi/processors/hadoop/AbstractHadoopProcessor.java +++ b/nifi-nar-bundles/nifi-extension-utils/nifi-hadoop-utils/src/main/java/org/apache/nifi/processors/hadoop/AbstractHadoopProcessor.java @@ -205,7 +205,7 @@ public abstract class AbstractHadoopProcessor extends AbstractProcessor implemen @Override public String getClassloaderIsolationKey(final PropertyContext context) { - final String explicitKerberosPrincipal = context.getProperty(kerberosProperties.getKerberosPrincipal()).getValue(); + final String explicitKerberosPrincipal = context.getProperty(kerberosProperties.getKerberosPrincipal()).evaluateAttributeExpressions().getValue(); if (explicitKerberosPrincipal != null) { return explicitKerberosPrincipal; } diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/main/java/org/apache/nifi/controller/service/StandardControllerServiceNode.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/main/java/org/apache/nifi/controller/service/StandardControllerServiceNode.java index bdae2b11b0..8593201423 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/main/java/org/apache/nifi/controller/service/StandardControllerServiceNode.java +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/main/java/org/apache/nifi/controller/service/StandardControllerServiceNode.java @@ -338,8 +338,9 @@ public class StandardControllerServiceNode extends AbstractComponentNode impleme @Override public void verifyCanEnable() { - if (getState() != ControllerServiceState.DISABLED) { - throw new IllegalStateException(getControllerServiceImplementation().getIdentifier() + " cannot be enabled because it is not disabled"); + final ControllerServiceState state = getState(); + if (state != ControllerServiceState.DISABLED) { + throw new IllegalStateException(getControllerServiceImplementation().getIdentifier() + " cannot be enabled because it is not disabled - it has a state of " + state); } } diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-nar-utils/src/main/java/org/apache/nifi/nar/InstanceClassLoader.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-nar-utils/src/main/java/org/apache/nifi/nar/InstanceClassLoader.java index e7153d7438..b024bb523b 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-nar-utils/src/main/java/org/apache/nifi/nar/InstanceClassLoader.java +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-nar-utils/src/main/java/org/apache/nifi/nar/InstanceClassLoader.java @@ -69,10 +69,6 @@ public class InstanceClassLoader extends AbstractNativeLibHandlingClassLoader { this.instanceType = type; this.instanceUrls = instanceUrls == null ? Collections.emptySet() : Collections.unmodifiableSet(new HashSet<>(instanceUrls)); this.additionalResourceUrls = additionalResourceUrls == null ? Collections.emptySet() : Collections.unmodifiableSet(new HashSet<>(additionalResourceUrls)); - - if (parent instanceof SharedInstanceClassLoader) { - ((SharedInstanceClassLoader) parent).incrementReferenceCount(); - } } @Override diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-nar-utils/src/main/java/org/apache/nifi/nar/NarThreadContextClassLoader.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-nar-utils/src/main/java/org/apache/nifi/nar/NarThreadContextClassLoader.java index 949d207342..915afa5288 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-nar-utils/src/main/java/org/apache/nifi/nar/NarThreadContextClassLoader.java +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-nar-utils/src/main/java/org/apache/nifi/nar/NarThreadContextClassLoader.java @@ -266,6 +266,7 @@ public class NarThreadContextClassLoader extends URLClassLoader { private static ClassLoader createClassLoader(final String implementationClassName, final String instanceId, final Bundle bundle, final ExtensionManager extensionManager) throws ClassNotFoundException { + final ClassLoader bundleClassLoader = bundle.getClassLoader(); final Class<?> rawClass = Class.forName(implementationClassName, true, bundleClassLoader); diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-nar-utils/src/main/java/org/apache/nifi/nar/SharedInstanceClassLoader.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-nar-utils/src/main/java/org/apache/nifi/nar/SharedInstanceClassLoader.java index 5aabef6ac2..5ec1327e93 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-nar-utils/src/main/java/org/apache/nifi/nar/SharedInstanceClassLoader.java +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-nar-utils/src/main/java/org/apache/nifi/nar/SharedInstanceClassLoader.java @@ -24,6 +24,7 @@ import java.util.Set; public class SharedInstanceClassLoader extends InstanceClassLoader { private long referenceCount = 0L; + private boolean closed = false; public SharedInstanceClassLoader(final String identifier, final String type, final Set<URL> instanceUrls, final Set<URL> additionalResourceUrls, final Set<File> narNativeLibDirs, final ClassLoader parent) { @@ -35,11 +36,17 @@ public class SharedInstanceClassLoader extends InstanceClassLoader { referenceCount--; if (referenceCount <= 0) { + closed = true; super.close(); } } - public synchronized void incrementReferenceCount() { + public synchronized boolean incrementReferenceCount() { + if (closed) { + return false; + } + referenceCount++; + return true; } } diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-nar-utils/src/main/java/org/apache/nifi/nar/StandardExtensionDiscoveringManager.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-nar-utils/src/main/java/org/apache/nifi/nar/StandardExtensionDiscoveringManager.java index baa29b6411..f8e1806abf 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-nar-utils/src/main/java/org/apache/nifi/nar/StandardExtensionDiscoveringManager.java +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-nar-utils/src/main/java/org/apache/nifi/nar/StandardExtensionDiscoveringManager.java @@ -88,7 +88,7 @@ public class StandardExtensionDiscoveringManager implements ExtensionDiscovering private final Map<String, ConfigurableComponent> tempComponentLookup = new HashMap<>(); private final Map<String, InstanceClassLoader> instanceClassloaderLookup = new ConcurrentHashMap<>(); - private final ConcurrentMap<BaseClassLoaderKey, ClassLoader> sharedBaseClassloaders = new ConcurrentHashMap<>(); + private final ConcurrentMap<BaseClassLoaderKey, SharedInstanceClassLoader> sharedBaseClassloaders = new ConcurrentHashMap<>(); public StandardExtensionDiscoveringManager() { this(Collections.emptyList()); @@ -407,8 +407,8 @@ public class StandardExtensionDiscoveringManager implements ExtensionDiscovering if (requiresInstanceClassLoading.cloneAncestorResources()) { // Check to see if there's already a shared ClassLoader that can be used as the parent/base classloader if (baseClassLoaderKey != null) { - final ClassLoader sharedBaseClassloader = sharedBaseClassloaders.get(baseClassLoaderKey); - if (sharedBaseClassloader != null) { + final SharedInstanceClassLoader sharedBaseClassloader = sharedBaseClassloaders.get(baseClassLoaderKey); + if (sharedBaseClassloader != null && sharedBaseClassloader.incrementReferenceCount()) { resolvedSharedClassLoader = true; ancestorClassLoader = sharedBaseClassloader; logger.debug("Creating InstanceClassLoader for type {} using shared Base ClassLoader {} for component {}", type, sharedBaseClassloader, instanceIdentifier); @@ -444,6 +444,8 @@ public class StandardExtensionDiscoveringManager implements ExtensionDiscovering // Created a shared class loader that is everything we need except for the additional URLs, as the additional URLs are instance-specific. final SharedInstanceClassLoader sharedClassLoader = new SharedInstanceClassLoader(instanceIdentifier, classType, instanceUrls, Collections.emptySet(), narNativeLibDirs, ancestorClassLoader); + sharedClassLoader.incrementReferenceCount(); + instanceClassLoader = new InstanceClassLoader(instanceIdentifier, classType, Collections.emptySet(), additionalUrls, Collections.emptySet(), sharedClassLoader); logger.debug("Creating InstanceClassLoader for type {} using newly created shared Base ClassLoader {} for component {}", type, sharedClassLoader, instanceIdentifier); diff --git a/nifi-system-tests/nifi-system-test-extensions-bundle/nifi-system-test-extensions/src/main/java/org/apache/nifi/processors/tests/system/WriteFlowFileCountToFile.java b/nifi-system-tests/nifi-system-test-extensions-bundle/nifi-system-test-extensions/src/main/java/org/apache/nifi/processors/tests/system/WriteFlowFileCountToFile.java index 7e78f0a0e0..ef6eb75016 100644 --- a/nifi-system-tests/nifi-system-test-extensions-bundle/nifi-system-test-extensions/src/main/java/org/apache/nifi/processors/tests/system/WriteFlowFileCountToFile.java +++ b/nifi-system-tests/nifi-system-test-extensions-bundle/nifi-system-test-extensions/src/main/java/org/apache/nifi/processors/tests/system/WriteFlowFileCountToFile.java @@ -34,7 +34,7 @@ import java.io.FileOutputStream; import java.io.OutputStream; import java.nio.charset.StandardCharsets; import java.util.Arrays; -import java.util.Collections; +import java.util.HashSet; import java.util.List; import java.util.Set; import java.util.concurrent.atomic.AtomicLong; @@ -64,18 +64,31 @@ public class WriteFlowFileCountToFile extends AbstractProcessor implements Class .defaultValue("counts.txt") .build(); + static final PropertyDescriptor CLASS_TO_CREATE = new Builder() + .name("Class to Create") + .displayName("Class to Create") + .description("If specified, each iteration of #onTrigger will create an instance of this class in order to test ClassLoader behavior. If unable to create the object, the FlowFile will be " + + "routed to failure") + .required(false) + .addValidator(NON_EMPTY_VALIDATOR) + .build(); + private final Relationship REL_SUCCESS = new Relationship.Builder() .name("success") .build(); + private final Relationship REL_FAILURE = new Relationship.Builder() + .name("failure") + .autoTerminateDefault(true) + .build(); @Override protected List<PropertyDescriptor> getSupportedPropertyDescriptors() { - return Arrays.asList(ISOLATION_KEY, FILE_TO_WRITE); + return Arrays.asList(ISOLATION_KEY, FILE_TO_WRITE, CLASS_TO_CREATE); } @Override public Set<Relationship> getRelationships() { - return Collections.singleton(REL_SUCCESS); + return new HashSet<>(Arrays.asList(REL_SUCCESS, REL_FAILURE)); } @Override @@ -90,6 +103,17 @@ public class WriteFlowFileCountToFile extends AbstractProcessor implements Class return; } + final String className = context.getProperty(CLASS_TO_CREATE).getValue(); + if (className != null) { + try { + Class.forName(className, true, Thread.currentThread().getContextClassLoader()); + } catch (final ClassNotFoundException e) { + getLogger().error("Failed to load class {} for {}; routing to failure", className, flowFile); + session.transfer(flowFile, REL_FAILURE); + return; + } + } + final long counterValue = counter.incrementAndGet(); final byte[] fileContents = String.valueOf(counterValue).getBytes(StandardCharsets.UTF_8); diff --git a/nifi-system-tests/nifi-system-test-suite/src/test/java/org/apache/nifi/tests/system/classloaders/ClassloaderIsolationKeyIT.java b/nifi-system-tests/nifi-system-test-suite/src/test/java/org/apache/nifi/tests/system/classloaders/ClassloaderIsolationKeyIT.java index 0665ae85f1..4c5fb822e5 100644 --- a/nifi-system-tests/nifi-system-test-suite/src/test/java/org/apache/nifi/tests/system/classloaders/ClassloaderIsolationKeyIT.java +++ b/nifi-system-tests/nifi-system-test-suite/src/test/java/org/apache/nifi/tests/system/classloaders/ClassloaderIsolationKeyIT.java @@ -19,6 +19,7 @@ package org.apache.nifi.tests.system.classloaders; import org.apache.nifi.tests.system.NiFiSystemIT; import org.apache.nifi.toolkit.cli.impl.client.nifi.NiFiClientException; +import org.apache.nifi.web.api.entity.ConnectionEntity; import org.apache.nifi.web.api.entity.ProcessorEntity; import org.junit.Test; @@ -28,8 +29,62 @@ import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.util.Collections; +import static org.junit.Assert.assertEquals; + public class ClassloaderIsolationKeyIT extends NiFiSystemIT { + /** + * After creating 1+ processors with the same ClassLoader Isolation Key, and then removing them, + * the SharedInstanceClassLoader will be closed. If we then create a new processor with the same + * ClassLoader Isolation Key, we need to ensure that we are then able to load classes from the ClassLoader + * that were not loaded previously. + */ + @Test + public void testRemoveAllInstancesThenCreateForSameIsolationKeyAllowsClassLoading() throws NiFiClientException, IOException, InterruptedException { + final ProcessorEntity generate = getClientUtil().createProcessor("GenerateFlowFile"); + final ProcessorEntity counter = getClientUtil().createProcessor("WriteFlowFileCountToFile"); + final ProcessorEntity terminate = getClientUtil().createProcessor("TerminateFlowFile"); + + getClientUtil().updateProcessorProperties(counter, Collections.singletonMap("File to Write", "count1.txt")); + getClientUtil().updateProcessorProperties(counter, Collections.singletonMap("Isolation Key", "abc123")); + + getClientUtil().createConnection(generate, counter, "success"); + final ConnectionEntity counterToTerminate = getClientUtil().createConnection(counter, terminate, "success"); + + getClientUtil().waitForValidProcessor(counter.getId()); + + getClientUtil().startProcessor(generate); + getClientUtil().startProcessor(counter); + + waitForQueueCount(counterToTerminate.getId(), 1); + + // Stop components, purge FlowFiles, delete all components + destroyFlow(); + + final ProcessorEntity newGenerate = getClientUtil().createProcessor("GenerateFlowFile"); + final ProcessorEntity newCounter = getClientUtil().createProcessor("WriteFlowFileCountToFile"); + final ProcessorEntity terminateSuccess = getClientUtil().createProcessor("TerminateFlowFile"); + final ProcessorEntity terminateFailure = getClientUtil().createProcessor("TerminateFlowFile"); + + final ConnectionEntity generateToCounter = getClientUtil().createConnection(newGenerate, newCounter, "success"); + final ConnectionEntity counterSuccess = getClientUtil().createConnection(newCounter, terminateSuccess, "success"); + final ConnectionEntity counterFailure = getClientUtil().createConnection(newCounter, terminateFailure, "failure"); + + getClientUtil().updateProcessorProperties(newCounter, Collections.singletonMap("Class to Create", "org.apache.nifi.processors.tests.system.CountEvents")); + getClientUtil().updateProcessorProperties(newCounter, Collections.singletonMap("File to Write", "count1.txt")); + getClientUtil().updateProcessorProperties(newCounter, Collections.singletonMap("Isolation Key", "abc123")); + + getClientUtil().waitForValidProcessor(newCounter.getId()); + + getClientUtil().startProcessor(newGenerate); + getClientUtil().startProcessor(newCounter); + + waitForQueueCount(generateToCounter.getId(), 0); + assertEquals(0, getConnectionQueueSize(counterFailure.getId())); + assertEquals(1, getConnectionQueueSize(counterSuccess.getId())); + } + + @Test public void testClassloaderChanges() throws NiFiClientException, IOException, InterruptedException { final ProcessorEntity generate = getClientUtil().createProcessor("GenerateFlowFile"); diff --git a/nifi-system-tests/nifi-system-test-suite/src/test/resources/conf/clustered/node1/logback.xml b/nifi-system-tests/nifi-system-test-suite/src/test/resources/conf/clustered/node1/logback.xml index 6c3f0bbb1e..c9bc0330f5 100644 --- a/nifi-system-tests/nifi-system-test-suite/src/test/resources/conf/clustered/node1/logback.xml +++ b/nifi-system-tests/nifi-system-test-suite/src/test/resources/conf/clustered/node1/logback.xml @@ -57,6 +57,17 @@ </encoder> </appender> + <appender name="REQUEST_FILE" class="ch.qos.logback.core.rolling.RollingFileAppender"> + <file>${org.apache.nifi.bootstrap.config.log.dir}/nifi-request.log</file> + <rollingPolicy class="ch.qos.logback.core.rolling.TimeBasedRollingPolicy"> + <fileNamePattern>${org.apache.nifi.bootstrap.config.log.dir}/nifi-request_%d.log</fileNamePattern> + <maxHistory>30</maxHistory> + </rollingPolicy> + <encoder class="ch.qos.logback.classic.encoder.PatternLayoutEncoder"> + <pattern>%msg%n</pattern> + </encoder> + </appender> + <appender name="BOOTSTRAP_FILE" class="ch.qos.logback.core.rolling.RollingFileAppender"> <file>${org.apache.nifi.bootstrap.config.log.dir}/nifi-bootstrap.log</file> <rollingPolicy class="ch.qos.logback.core.rolling.TimeBasedRollingPolicy"> @@ -82,21 +93,22 @@ </appender> <!-- valid logging levels: TRACE, DEBUG, INFO, WARN, ERROR --> + <logger name="org.apache.nifi" level="INFO"/> <logger name="org.apache.nifi.processors" level="INFO"/> <logger name="org.apache.nifi.processors.standard.LogAttribute" level="INFO"/> <logger name="org.apache.nifi.processors.standard.LogMessage" level="INFO"/> <logger name="org.apache.nifi.controller.repository.StandardProcessSession" level="WARN" /> - <logger name="org.apache.nifi.connectable.LocalPort" level="DEBUG"/> - <logger name="org.apache.nifi.web.util.ClusterReplicationComponentLifecycle" level="DEBUG" /> <logger name="org.apache.zookeeper.ClientCnxn" level="ERROR" /> <logger name="org.apache.zookeeper.server.NIOServerCnxn" level="ERROR" /> <logger name="org.apache.zookeeper.server.NIOServerCnxnFactory" level="ERROR" /> + <logger name="org.apache.zookeeper.server.NettyServerCnxnFactory" level="ERROR" /> <logger name="org.apache.zookeeper.server.quorum" level="ERROR" /> <logger name="org.apache.zookeeper.ZooKeeper" level="ERROR" /> <logger name="org.apache.zookeeper.server.PrepRequestProcessor" level="ERROR" /> + <logger name="org.apache.nifi.controller.reporting.LogComponentStatuses" level="ERROR" /> <logger name="org.apache.calcite.runtime.CalciteException" level="OFF" /> @@ -128,6 +140,17 @@ <!-- Suppress non-error messages from SMBJ which was emitting large amounts of INFO logs by default --> <logger name="com.hierynomus.smbj" level="WARN" /> + <!-- Suppress non-error messages from AWS KCL which was emitting large amounts of INFO logs by default --> + <logger name="com.amazonaws.services.kinesis" level="WARN" /> + + <!-- Suppress non-error messages from Apache Atlas which was emitting large amounts of INFO logs by default --> + <logger name="org.apache.atlas" level="WARN"/> + + <!-- These log messages would normally go to the USER_FILE log, but they belong in the APP_FILE --> + <logger name="org.apache.nifi.web.security.requests" level="INFO" additivity="false"> + <appender-ref ref="APP_FILE"/> + </logger> + <!-- Logger for capturing user events. We do not want to propagate these log events to the root logger. These messages are only sent to the @@ -148,10 +171,17 @@ <logger name="org.apache.nifi.web.api.AccessResource" level="INFO" additivity="false"> <appender-ref ref="USER_FILE"/> </logger> - <logger name="org.apache.nifi.web.api" level="DEBUG" additivity="false"> + <logger name="org.springframework.security.saml.log" level="WARN" additivity="false"> + <appender-ref ref="USER_FILE"/> + </logger> + <logger name="org.opensaml" level="WARN" additivity="false"> <appender-ref ref="USER_FILE"/> </logger> + <!-- Web Server Request Log --> + <logger name="org.apache.nifi.web.server.RequestLog" level="INFO" additivity="false"> + <appender-ref ref="REQUEST_FILE"/> + </logger> <!-- Logger for capturing Bootstrap logs and NiFi's standard error and standard out. @@ -174,9 +204,8 @@ <appender-ref ref="BOOTSTRAP_FILE" /> </logger> - <root level="INFO"> - <appender-ref ref="APP_FILE"/> + <appender-ref ref="APP_FILE" /> </root> -</configuration> \ No newline at end of file +</configuration> diff --git a/nifi-system-tests/nifi-system-test-suite/src/test/resources/conf/clustered/node2/logback.xml b/nifi-system-tests/nifi-system-test-suite/src/test/resources/conf/clustered/node2/logback.xml index a9cdb2623a..c9bc0330f5 100644 --- a/nifi-system-tests/nifi-system-test-suite/src/test/resources/conf/clustered/node2/logback.xml +++ b/nifi-system-tests/nifi-system-test-suite/src/test/resources/conf/clustered/node2/logback.xml @@ -57,6 +57,17 @@ </encoder> </appender> + <appender name="REQUEST_FILE" class="ch.qos.logback.core.rolling.RollingFileAppender"> + <file>${org.apache.nifi.bootstrap.config.log.dir}/nifi-request.log</file> + <rollingPolicy class="ch.qos.logback.core.rolling.TimeBasedRollingPolicy"> + <fileNamePattern>${org.apache.nifi.bootstrap.config.log.dir}/nifi-request_%d.log</fileNamePattern> + <maxHistory>30</maxHistory> + </rollingPolicy> + <encoder class="ch.qos.logback.classic.encoder.PatternLayoutEncoder"> + <pattern>%msg%n</pattern> + </encoder> + </appender> + <appender name="BOOTSTRAP_FILE" class="ch.qos.logback.core.rolling.RollingFileAppender"> <file>${org.apache.nifi.bootstrap.config.log.dir}/nifi-bootstrap.log</file> <rollingPolicy class="ch.qos.logback.core.rolling.TimeBasedRollingPolicy"> @@ -82,25 +93,22 @@ </appender> <!-- valid logging levels: TRACE, DEBUG, INFO, WARN, ERROR --> + <logger name="org.apache.nifi" level="INFO"/> <logger name="org.apache.nifi.processors" level="INFO"/> <logger name="org.apache.nifi.processors.standard.LogAttribute" level="INFO"/> <logger name="org.apache.nifi.processors.standard.LogMessage" level="INFO"/> <logger name="org.apache.nifi.controller.repository.StandardProcessSession" level="WARN" /> - <logger name="org.apache.nifi.connectable.LocalPort" level="DEBUG"/> - <logger name="org.apache.nifi.web.util.ClusterReplicationComponentLifecycle" level="DEBUG" /> - <logger name="org.apache.nifi.groups.AbstractComponentScheduler" level="DEBUG" /> - <logger name="org.apache.nifi.controller.XmlFlowSynchronizer" level="DEBUG" /> - <logger name="org.apache.nifi.controller.inheritance" level="DEBUG" /> - <logger name="org.apache.nifi.controller.serialization.VersionedFlowSynchronizer" level="DEBUG" /> <logger name="org.apache.zookeeper.ClientCnxn" level="ERROR" /> <logger name="org.apache.zookeeper.server.NIOServerCnxn" level="ERROR" /> <logger name="org.apache.zookeeper.server.NIOServerCnxnFactory" level="ERROR" /> + <logger name="org.apache.zookeeper.server.NettyServerCnxnFactory" level="ERROR" /> <logger name="org.apache.zookeeper.server.quorum" level="ERROR" /> <logger name="org.apache.zookeeper.ZooKeeper" level="ERROR" /> <logger name="org.apache.zookeeper.server.PrepRequestProcessor" level="ERROR" /> + <logger name="org.apache.nifi.controller.reporting.LogComponentStatuses" level="ERROR" /> <logger name="org.apache.calcite.runtime.CalciteException" level="OFF" /> @@ -132,6 +140,17 @@ <!-- Suppress non-error messages from SMBJ which was emitting large amounts of INFO logs by default --> <logger name="com.hierynomus.smbj" level="WARN" /> + <!-- Suppress non-error messages from AWS KCL which was emitting large amounts of INFO logs by default --> + <logger name="com.amazonaws.services.kinesis" level="WARN" /> + + <!-- Suppress non-error messages from Apache Atlas which was emitting large amounts of INFO logs by default --> + <logger name="org.apache.atlas" level="WARN"/> + + <!-- These log messages would normally go to the USER_FILE log, but they belong in the APP_FILE --> + <logger name="org.apache.nifi.web.security.requests" level="INFO" additivity="false"> + <appender-ref ref="APP_FILE"/> + </logger> + <!-- Logger for capturing user events. We do not want to propagate these log events to the root logger. These messages are only sent to the @@ -152,10 +171,17 @@ <logger name="org.apache.nifi.web.api.AccessResource" level="INFO" additivity="false"> <appender-ref ref="USER_FILE"/> </logger> - <logger name="org.apache.nifi.web.api" level="DEBUG" additivity="false"> + <logger name="org.springframework.security.saml.log" level="WARN" additivity="false"> + <appender-ref ref="USER_FILE"/> + </logger> + <logger name="org.opensaml" level="WARN" additivity="false"> <appender-ref ref="USER_FILE"/> </logger> + <!-- Web Server Request Log --> + <logger name="org.apache.nifi.web.server.RequestLog" level="INFO" additivity="false"> + <appender-ref ref="REQUEST_FILE"/> + </logger> <!-- Logger for capturing Bootstrap logs and NiFi's standard error and standard out. @@ -178,9 +204,8 @@ <appender-ref ref="BOOTSTRAP_FILE" /> </logger> - <root level="INFO"> - <appender-ref ref="APP_FILE"/> + <appender-ref ref="APP_FILE" /> </root> -</configuration> \ No newline at end of file +</configuration> diff --git a/nifi-system-tests/nifi-system-test-suite/src/test/resources/conf/default/logback.xml b/nifi-system-tests/nifi-system-test-suite/src/test/resources/conf/default/logback.xml index c42b3be1e0..c9bc0330f5 100644 --- a/nifi-system-tests/nifi-system-test-suite/src/test/resources/conf/default/logback.xml +++ b/nifi-system-tests/nifi-system-test-suite/src/test/resources/conf/default/logback.xml @@ -57,6 +57,17 @@ </encoder> </appender> + <appender name="REQUEST_FILE" class="ch.qos.logback.core.rolling.RollingFileAppender"> + <file>${org.apache.nifi.bootstrap.config.log.dir}/nifi-request.log</file> + <rollingPolicy class="ch.qos.logback.core.rolling.TimeBasedRollingPolicy"> + <fileNamePattern>${org.apache.nifi.bootstrap.config.log.dir}/nifi-request_%d.log</fileNamePattern> + <maxHistory>30</maxHistory> + </rollingPolicy> + <encoder class="ch.qos.logback.classic.encoder.PatternLayoutEncoder"> + <pattern>%msg%n</pattern> + </encoder> + </appender> + <appender name="BOOTSTRAP_FILE" class="ch.qos.logback.core.rolling.RollingFileAppender"> <file>${org.apache.nifi.bootstrap.config.log.dir}/nifi-bootstrap.log</file> <rollingPolicy class="ch.qos.logback.core.rolling.TimeBasedRollingPolicy"> @@ -82,20 +93,22 @@ </appender> <!-- valid logging levels: TRACE, DEBUG, INFO, WARN, ERROR --> - <logger name="org.apache.nifi.web.util.LocalComponentLifecycle" level="DEBUG" /> + <logger name="org.apache.nifi" level="INFO"/> <logger name="org.apache.nifi.processors" level="INFO"/> <logger name="org.apache.nifi.processors.standard.LogAttribute" level="INFO"/> <logger name="org.apache.nifi.processors.standard.LogMessage" level="INFO"/> <logger name="org.apache.nifi.controller.repository.StandardProcessSession" level="WARN" /> - <logger name="org.apache.nifi.connectable.LocalPort" level="DEBUG"/> + <logger name="org.apache.zookeeper.ClientCnxn" level="ERROR" /> <logger name="org.apache.zookeeper.server.NIOServerCnxn" level="ERROR" /> <logger name="org.apache.zookeeper.server.NIOServerCnxnFactory" level="ERROR" /> + <logger name="org.apache.zookeeper.server.NettyServerCnxnFactory" level="ERROR" /> <logger name="org.apache.zookeeper.server.quorum" level="ERROR" /> <logger name="org.apache.zookeeper.ZooKeeper" level="ERROR" /> <logger name="org.apache.zookeeper.server.PrepRequestProcessor" level="ERROR" /> + <logger name="org.apache.nifi.controller.reporting.LogComponentStatuses" level="ERROR" /> <logger name="org.apache.calcite.runtime.CalciteException" level="OFF" /> @@ -127,6 +140,17 @@ <!-- Suppress non-error messages from SMBJ which was emitting large amounts of INFO logs by default --> <logger name="com.hierynomus.smbj" level="WARN" /> + <!-- Suppress non-error messages from AWS KCL which was emitting large amounts of INFO logs by default --> + <logger name="com.amazonaws.services.kinesis" level="WARN" /> + + <!-- Suppress non-error messages from Apache Atlas which was emitting large amounts of INFO logs by default --> + <logger name="org.apache.atlas" level="WARN"/> + + <!-- These log messages would normally go to the USER_FILE log, but they belong in the APP_FILE --> + <logger name="org.apache.nifi.web.security.requests" level="INFO" additivity="false"> + <appender-ref ref="APP_FILE"/> + </logger> + <!-- Logger for capturing user events. We do not want to propagate these log events to the root logger. These messages are only sent to the @@ -147,7 +171,17 @@ <logger name="org.apache.nifi.web.api.AccessResource" level="INFO" additivity="false"> <appender-ref ref="USER_FILE"/> </logger> + <logger name="org.springframework.security.saml.log" level="WARN" additivity="false"> + <appender-ref ref="USER_FILE"/> + </logger> + <logger name="org.opensaml" level="WARN" additivity="false"> + <appender-ref ref="USER_FILE"/> + </logger> + <!-- Web Server Request Log --> + <logger name="org.apache.nifi.web.server.RequestLog" level="INFO" additivity="false"> + <appender-ref ref="REQUEST_FILE"/> + </logger> <!-- Logger for capturing Bootstrap logs and NiFi's standard error and standard out. @@ -170,9 +204,8 @@ <appender-ref ref="BOOTSTRAP_FILE" /> </logger> - <root level="INFO"> - <appender-ref ref="APP_FILE"/> + <appender-ref ref="APP_FILE" /> </root> -</configuration> \ No newline at end of file +</configuration>