mirror of
https://github.com/apache/nifi.git
synced 2025-02-06 01:58:32 +00:00
NIFI-12114 Create separate test instance for python extensions
Signed-off-by: Arpad Boda <aboda@apache.org> This closes #7780
This commit is contained in:
Nandor Soma Abonyi
Arpad Boda
parent
16c527271b
commit
4b0c8bf6af
1
.github/workflows/system-tests.yml
vendored
1
.github/workflows/system-tests.yml
vendored
@ -56,6 +56,7 @@ env:
|
||||
package
|
||||
verify
|
||||
-P integration-tests
|
||||
-D include-python-integration-tests=true
|
||||
MAVEN_PROJECTS: >-
|
||||
-pl :nifi-python-framework
|
||||
-pl :nifi-python-extension-api
|
||||
|
||||
@ -24,6 +24,30 @@
|
||||
<artifactId>nifi-system-test-suite</artifactId>
|
||||
<packaging>jar</packaging>
|
||||
|
||||
<profiles>
|
||||
<profile>
|
||||
<id>include-python-integration-tests</id>
|
||||
<activation>
|
||||
<property>
|
||||
<name>include-python-integration-tests</name>
|
||||
<value>!true</value>
|
||||
</property>
|
||||
</activation>
|
||||
<build>
|
||||
<plugins>
|
||||
<plugin>
|
||||
<artifactId>maven-failsafe-plugin</artifactId>
|
||||
<configuration>
|
||||
<excludes>
|
||||
<exclude>PythonProcessorIT.java</exclude>
|
||||
</excludes>
|
||||
</configuration>
|
||||
</plugin>
|
||||
</plugins>
|
||||
</build>
|
||||
</profile>
|
||||
</profiles>
|
||||
|
||||
<build>
|
||||
<plugins>
|
||||
<plugin>
|
||||
|
||||
@ -201,7 +201,7 @@ public abstract class NiFiSystemIT implements NiFiInstanceProvider {
|
||||
.bootstrapConfig("src/test/resources/conf/default/bootstrap.conf")
|
||||
.instanceDirectory("target/standalone-instance")
|
||||
.overrideNifiProperties(getNifiPropertiesOverrides())
|
||||
.unpackPythonExtensions(isUnpackPythonExtensions())
|
||||
.unpackPythonExtensions(false)
|
||||
.build());
|
||||
}
|
||||
|
||||
@ -211,6 +211,16 @@ public abstract class NiFiSystemIT implements NiFiInstanceProvider {
|
||||
"src/test/resources/conf/clustered/node2/bootstrap.conf");
|
||||
}
|
||||
|
||||
public NiFiInstanceFactory createPythonicInstanceFactory() {
|
||||
return new SpawnedStandaloneNiFiInstanceFactory(
|
||||
new InstanceConfiguration.Builder()
|
||||
.bootstrapConfig("src/test/resources/conf/pythonic/bootstrap.conf")
|
||||
.instanceDirectory("target/pythonic-instance")
|
||||
.overrideNifiProperties(getNifiPropertiesOverrides())
|
||||
.unpackPythonExtensions(true)
|
||||
.build());
|
||||
}
|
||||
|
||||
protected String getTestName() {
|
||||
return testInfo.getDisplayName();
|
||||
}
|
||||
@ -545,10 +555,6 @@ public abstract class NiFiSystemIT implements NiFiInstanceProvider {
|
||||
return node2Dto;
|
||||
}
|
||||
|
||||
protected boolean isUnpackPythonExtensions() {
|
||||
return false;
|
||||
}
|
||||
|
||||
/**
|
||||
* Disconnects a node from the cluster
|
||||
* @param nodeIndex the 1-based index of the node
|
||||
|
||||
@ -17,6 +17,7 @@
|
||||
|
||||
package org.apache.nifi.tests.system.python;
|
||||
|
||||
import org.apache.nifi.tests.system.NiFiInstanceFactory;
|
||||
import org.apache.nifi.tests.system.NiFiSystemIT;
|
||||
import org.apache.nifi.toolkit.cli.impl.client.nifi.NiFiClientException;
|
||||
import org.apache.nifi.web.api.dto.ProcessorConfigDTO;
|
||||
@ -41,8 +42,8 @@ import static org.junit.jupiter.api.Assertions.assertTrue;
|
||||
public class PythonProcessorIT extends NiFiSystemIT {
|
||||
|
||||
@Override
|
||||
protected boolean isUnpackPythonExtensions() {
|
||||
return true;
|
||||
public NiFiInstanceFactory getInstanceFactory() {
|
||||
return createPythonicInstanceFactory();
|
||||
}
|
||||
|
||||
@Test
|
||||
|
||||
@ -42,7 +42,7 @@ nifi.documentation.working.directory=./work/docs/components
|
||||
# Python Extensions #
|
||||
#####################
|
||||
# Uncomment in order to enable Python Extensions.
|
||||
nifi.python.command=python
|
||||
#nifi.python.command=python
|
||||
nifi.python.framework.source.directory=./python/framework
|
||||
nifi.python.extensions.source.directory.default=./python/extensions
|
||||
nifi.python.working.directory=./work/python
|
||||
|
||||
@ -42,7 +42,7 @@ nifi.documentation.working.directory=./work/docs/components
|
||||
# Python Extensions #
|
||||
#####################
|
||||
# Uncomment in order to enable Python Extensions.
|
||||
nifi.python.command=python
|
||||
#nifi.python.command=python
|
||||
nifi.python.framework.source.directory=./python/framework
|
||||
nifi.python.extensions.source.directory.default=./python/extensions
|
||||
nifi.python.working.directory=./work/python
|
||||
|
||||
@ -42,7 +42,7 @@ nifi.documentation.working.directory=./work/docs/components
|
||||
# Python Extensions #
|
||||
#####################
|
||||
# Uncomment in order to enable Python Extensions.
|
||||
nifi.python.command=python
|
||||
#nifi.python.command=python
|
||||
nifi.python.framework.source.directory=./python/framework
|
||||
nifi.python.extensions.source.directory.default=./python/extensions
|
||||
nifi.python.working.directory=./work/python
|
||||
|
||||
@ -0,0 +1,33 @@
|
||||
# Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
# contributor license agreements. See the NOTICE file distributed with
|
||||
# this work for additional information regarding copyright ownership.
|
||||
# The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
# (the "License"); you may not use this file except in compliance with
|
||||
# the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
# Configure where NiFi's lib and conf directories live
|
||||
lib.dir=../nifi-lib-assembly/lib
|
||||
conf.dir=./conf
|
||||
working.dir=./target/pythonic-instance
|
||||
|
||||
# How long to wait after telling NiFi to shutdown before explicitly killing the Process
|
||||
graceful.shutdown.seconds=20
|
||||
|
||||
# JVM memory settings
|
||||
java.arg.2=-Xms512m
|
||||
java.arg.3=-Xmx512m
|
||||
|
||||
java.arg.14=-Djava.awt.headless=true
|
||||
|
||||
#java.arg.debug=-agentlib:jdwp=transport=dt_socket,server=y,suspend=n,address=8002
|
||||
|
||||
# Disable Logback web shutdown hook using System property
|
||||
java.arg.logbackShutdown=-DlogbackDisableServletContainerInitializer=true
|
||||
@ -0,0 +1,214 @@
|
||||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<!--
|
||||
Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
contributor license agreements. See the NOTICE file distributed with
|
||||
this work for additional information regarding copyright ownership.
|
||||
The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
(the "License"); you may not use this file except in compliance with
|
||||
the License. You may obtain a copy of the License at
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
-->
|
||||
|
||||
<configuration scan="true" scanPeriod="30 seconds">
|
||||
<shutdownHook class="ch.qos.logback.core.hook.DefaultShutdownHook" />
|
||||
|
||||
<contextListener class="ch.qos.logback.classic.jul.LevelChangePropagator">
|
||||
<resetJUL>true</resetJUL>
|
||||
</contextListener>
|
||||
|
||||
<appender name="APP_FILE" class="ch.qos.logback.core.rolling.RollingFileAppender">
|
||||
<file>${org.apache.nifi.bootstrap.config.log.dir}/nifi-app.log</file>
|
||||
<rollingPolicy class="ch.qos.logback.core.rolling.SizeAndTimeBasedRollingPolicy">
|
||||
<!--
|
||||
For daily rollover, use 'app_%d.log'.
|
||||
For hourly rollover, use 'app_%d{yyyy-MM-dd_HH}.log'.
|
||||
To GZIP rolled files, replace '.log' with '.log.gz'.
|
||||
To ZIP rolled files, replace '.log' with '.log.zip'.
|
||||
-->
|
||||
<fileNamePattern>${org.apache.nifi.bootstrap.config.log.dir}/nifi-app_%d{yyyy-MM-dd_HH}.%i.log</fileNamePattern>
|
||||
<maxFileSize>100MB</maxFileSize>
|
||||
<!-- keep 30 log files worth of history -->
|
||||
<maxHistory>30</maxHistory>
|
||||
</rollingPolicy>
|
||||
<immediateFlush>true</immediateFlush>
|
||||
<encoder class="ch.qos.logback.classic.encoder.PatternLayoutEncoder">
|
||||
<pattern>%date %level [%thread] %logger{40} %msg%n</pattern>
|
||||
</encoder>
|
||||
</appender>
|
||||
|
||||
<appender name="USER_FILE" class="ch.qos.logback.core.rolling.RollingFileAppender">
|
||||
<file>${org.apache.nifi.bootstrap.config.log.dir}/nifi-user.log</file>
|
||||
<rollingPolicy class="ch.qos.logback.core.rolling.TimeBasedRollingPolicy">
|
||||
<!--
|
||||
For daily rollover, use 'user_%d.log'.
|
||||
For hourly rollover, use 'user_%d{yyyy-MM-dd_HH}.log'.
|
||||
To GZIP rolled files, replace '.log' with '.log.gz'.
|
||||
To ZIP rolled files, replace '.log' with '.log.zip'.
|
||||
-->
|
||||
<fileNamePattern>${org.apache.nifi.bootstrap.config.log.dir}/nifi-user_%d.log</fileNamePattern>
|
||||
<!-- keep 30 log files worth of history -->
|
||||
<maxHistory>30</maxHistory>
|
||||
</rollingPolicy>
|
||||
<encoder class="ch.qos.logback.classic.encoder.PatternLayoutEncoder">
|
||||
<pattern>%date %level [%thread] %logger{40} %msg%n</pattern>
|
||||
</encoder>
|
||||
</appender>
|
||||
|
||||
<appender name="REQUEST_FILE" class="ch.qos.logback.core.rolling.RollingFileAppender">
|
||||
<file>${org.apache.nifi.bootstrap.config.log.dir}/nifi-request.log</file>
|
||||
<rollingPolicy class="ch.qos.logback.core.rolling.TimeBasedRollingPolicy">
|
||||
<fileNamePattern>${org.apache.nifi.bootstrap.config.log.dir}/nifi-request_%d.log</fileNamePattern>
|
||||
<maxHistory>30</maxHistory>
|
||||
</rollingPolicy>
|
||||
<encoder class="ch.qos.logback.classic.encoder.PatternLayoutEncoder">
|
||||
<pattern>%msg%n</pattern>
|
||||
</encoder>
|
||||
</appender>
|
||||
|
||||
<appender name="BOOTSTRAP_FILE" class="ch.qos.logback.core.rolling.RollingFileAppender">
|
||||
<file>${org.apache.nifi.bootstrap.config.log.dir}/nifi-bootstrap.log</file>
|
||||
<rollingPolicy class="ch.qos.logback.core.rolling.TimeBasedRollingPolicy">
|
||||
<!--
|
||||
For daily rollover, use 'bootstrap_%d.log'.
|
||||
For hourly rollover, use 'bootstrap_%d{yyyy-MM-dd_HH}.log'.
|
||||
To GZIP rolled files, replace '.log' with '.log.gz'.
|
||||
To ZIP rolled files, replace '.log' with '.log.zip'.
|
||||
-->
|
||||
<fileNamePattern>${org.apache.nifi.bootstrap.config.log.dir}/nifi-bootstrap_%d.log</fileNamePattern>
|
||||
<!-- keep 5 log files worth of history -->
|
||||
<maxHistory>5</maxHistory>
|
||||
</rollingPolicy>
|
||||
<encoder class="ch.qos.logback.classic.encoder.PatternLayoutEncoder">
|
||||
<pattern>%date %level [%thread] %logger{40} %msg%n</pattern>
|
||||
</encoder>
|
||||
</appender>
|
||||
|
||||
<appender name="CONSOLE" class="ch.qos.logback.core.ConsoleAppender">
|
||||
<encoder class="ch.qos.logback.classic.encoder.PatternLayoutEncoder">
|
||||
<pattern>%date %level [%thread] %logger{40} %msg%n</pattern>
|
||||
</encoder>
|
||||
</appender>
|
||||
|
||||
<!-- valid logging levels: TRACE, DEBUG, INFO, WARN, ERROR -->
|
||||
|
||||
<logger name="org.apache.nifi" level="INFO"/>
|
||||
<logger name="org.apache.nifi.processors" level="INFO"/>
|
||||
<logger name="org.apache.nifi.processors.standard.LogAttribute" level="INFO"/>
|
||||
<logger name="org.apache.nifi.processors.standard.LogMessage" level="INFO"/>
|
||||
<logger name="org.apache.nifi.controller.repository.StandardProcessSession" level="WARN" />
|
||||
|
||||
|
||||
<logger name="org.apache.zookeeper.ClientCnxn" level="ERROR" />
|
||||
<logger name="org.apache.zookeeper.server.NIOServerCnxn" level="ERROR" />
|
||||
<logger name="org.apache.zookeeper.server.NIOServerCnxnFactory" level="ERROR" />
|
||||
<logger name="org.apache.zookeeper.server.NettyServerCnxnFactory" level="ERROR" />
|
||||
<logger name="org.apache.zookeeper.server.quorum" level="ERROR" />
|
||||
<logger name="org.apache.zookeeper.ZooKeeper" level="ERROR" />
|
||||
<logger name="org.apache.zookeeper.server.PrepRequestProcessor" level="ERROR" />
|
||||
<logger name="org.apache.nifi.controller.reporting.LogComponentStatuses" level="ERROR" />
|
||||
|
||||
<logger name="org.apache.calcite.runtime.CalciteException" level="OFF" />
|
||||
<logger name="deprecation" level="OFF" />
|
||||
|
||||
<logger name="org.apache.curator.framework.recipes.leader.LeaderSelector" level="OFF" />
|
||||
<logger name="org.apache.curator.ConnectionState" level="OFF" />
|
||||
|
||||
<!-- Logger for managing logging statements for nifi clusters. -->
|
||||
<logger name="org.apache.nifi.cluster" level="INFO"/>
|
||||
|
||||
<!-- Logger for logging HTTP requests received by the web server. -->
|
||||
<logger name="org.apache.nifi.server.JettyServer" level="INFO"/>
|
||||
|
||||
<!-- Logger for managing logging statements for jetty -->
|
||||
<logger name="org.eclipse.jetty" level="INFO"/>
|
||||
|
||||
<!-- Suppress non-error messages due to excessive logging by class or library -->
|
||||
<logger name="org.springframework" level="ERROR"/>
|
||||
|
||||
<!-- Suppress non-error messages due to known warning about redundant path annotation (NIFI-574) -->
|
||||
<logger name="org.glassfish.jersey.internal.Errors" level="ERROR"/>
|
||||
|
||||
<!-- Suppress non-error messages due to Jetty AnnotationParser emitting a large amount of WARNS. Issue described in NIFI-5479. -->
|
||||
<logger name="org.eclipse.jetty.annotations.AnnotationParser" level="ERROR"/>
|
||||
|
||||
<!-- Suppress non-error messages from SSHJ which was emitting large amounts of INFO logs by default -->
|
||||
<logger name="net.schmizz.sshj" level="WARN" />
|
||||
<logger name="com.hierynomus.sshj" level="WARN" />
|
||||
|
||||
<!-- Suppress non-error messages from SMBJ which was emitting large amounts of INFO logs by default -->
|
||||
<logger name="com.hierynomus.smbj" level="WARN" />
|
||||
|
||||
<!-- Suppress non-error messages from AWS KCL which was emitting large amounts of INFO logs by default -->
|
||||
<logger name="com.amazonaws.services.kinesis" level="WARN" />
|
||||
|
||||
<!-- Suppress non-error messages from Apache Atlas which was emitting large amounts of INFO logs by default -->
|
||||
<logger name="org.apache.atlas" level="WARN"/>
|
||||
|
||||
<!-- These log messages would normally go to the USER_FILE log, but they belong in the APP_FILE -->
|
||||
<logger name="org.apache.nifi.web.security.requests" level="INFO" additivity="false">
|
||||
<appender-ref ref="APP_FILE"/>
|
||||
</logger>
|
||||
|
||||
<!--
|
||||
Logger for capturing user events. We do not want to propagate these
|
||||
log events to the root logger. These messages are only sent to the
|
||||
user-log appender.
|
||||
-->
|
||||
<logger name="org.apache.nifi.web.security" level="INFO" additivity="false">
|
||||
<appender-ref ref="USER_FILE"/>
|
||||
</logger>
|
||||
<logger name="org.apache.nifi.web.api.config" level="INFO" additivity="false">
|
||||
<appender-ref ref="USER_FILE"/>
|
||||
</logger>
|
||||
<logger name="org.apache.nifi.authorization" level="INFO" additivity="false">
|
||||
<appender-ref ref="USER_FILE"/>
|
||||
</logger>
|
||||
<logger name="org.apache.nifi.cluster.authorization" level="INFO" additivity="false">
|
||||
<appender-ref ref="USER_FILE"/>
|
||||
</logger>
|
||||
<logger name="org.apache.nifi.web.api.AccessResource" level="INFO" additivity="false">
|
||||
<appender-ref ref="USER_FILE"/>
|
||||
</logger>
|
||||
<logger name="org.springframework.security.saml.log" level="WARN" additivity="false">
|
||||
<appender-ref ref="USER_FILE"/>
|
||||
</logger>
|
||||
<logger name="org.opensaml" level="WARN" additivity="false">
|
||||
<appender-ref ref="USER_FILE"/>
|
||||
</logger>
|
||||
|
||||
<!-- Web Server Request Log -->
|
||||
<logger name="org.apache.nifi.web.server.RequestLog" level="INFO" additivity="false">
|
||||
<appender-ref ref="REQUEST_FILE"/>
|
||||
</logger>
|
||||
|
||||
<!--
|
||||
Logger for capturing Bootstrap logs and NiFi's standard error and standard out.
|
||||
-->
|
||||
<logger name="org.apache.nifi.bootstrap" level="INFO" additivity="false">
|
||||
<appender-ref ref="BOOTSTRAP_FILE" />
|
||||
</logger>
|
||||
<logger name="org.apache.nifi.bootstrap.Command" level="INFO" additivity="false">
|
||||
<appender-ref ref="CONSOLE" />
|
||||
<appender-ref ref="BOOTSTRAP_FILE" />
|
||||
</logger>
|
||||
|
||||
<!-- Everything written to NiFi's Standard Out will be logged with the logger org.apache.nifi.StdOut at INFO level -->
|
||||
<logger name="org.apache.nifi.StdOut" level="INFO" additivity="false">
|
||||
<appender-ref ref="BOOTSTRAP_FILE" />
|
||||
</logger>
|
||||
|
||||
<!-- Everything written to NiFi's Standard Error will be logged with the logger org.apache.nifi.StdErr at ERROR level -->
|
||||
<logger name="org.apache.nifi.StdErr" level="ERROR" additivity="false">
|
||||
<appender-ref ref="BOOTSTRAP_FILE" />
|
||||
</logger>
|
||||
|
||||
<root level="INFO">
|
||||
<appender-ref ref="APP_FILE" />
|
||||
</root>
|
||||
|
||||
</configuration>
|
||||
@ -0,0 +1,266 @@
|
||||
#
|
||||
# /*
|
||||
# * Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
# * contributor license agreements. See the NOTICE file distributed with
|
||||
# * this work for additional information regarding copyright ownership.
|
||||
# * The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
# * (the "License"); you may not use this file except in compliance with
|
||||
# * the License. You may obtain a copy of the License at
|
||||
# *
|
||||
# * http://www.apache.org/licenses/LICENSE-2.0
|
||||
# *
|
||||
# * Unless required by applicable law or agreed to in writing, software
|
||||
# * distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# * See the License for the specific language governing permissions and
|
||||
# * limitations under the License.
|
||||
# */
|
||||
#
|
||||
|
||||
# Core Properties #
|
||||
nifi.flow.configuration.file=./conf/flow.xml.gz
|
||||
nifi.flow.configuration.archive.enabled=true
|
||||
nifi.flow.configuration.archive.dir=./conf/archive/
|
||||
nifi.flow.configuration.archive.max.time=30 days
|
||||
nifi.flow.configuration.archive.max.storage=500 MB
|
||||
nifi.flow.configuration.archive.max.count=
|
||||
nifi.flowcontroller.autoResumeState=true
|
||||
nifi.flowcontroller.graceful.shutdown.period=10 sec
|
||||
nifi.flowservice.writedelay.interval=500 ms
|
||||
nifi.administrative.yield.duration=100 millis
|
||||
# If a component has no work to do (is "bored"), how long should we wait before checking again for work?
|
||||
nifi.bored.yield.duration=10 millis
|
||||
nifi.queue.backpressure.count=10000
|
||||
nifi.queue.backpressure.size=1 GB
|
||||
|
||||
nifi.authorizer.configuration.file=./conf/authorizers.xml
|
||||
nifi.login.identity.provider.configuration.file=./conf/login-identity-providers.xml
|
||||
nifi.ui.banner.text=
|
||||
nifi.ui.autorefresh.interval=30 sec
|
||||
nifi.nar.library.directory=./lib
|
||||
nifi.nar.library.autoload.directory=./extensions
|
||||
nifi.nar.working.directory=./work/nar/
|
||||
nifi.documentation.working.directory=./work/docs/components
|
||||
|
||||
#####################
|
||||
# Python Extensions #
|
||||
#####################
|
||||
# Uncomment in order to enable Python Extensions.
|
||||
nifi.python.command=python3
|
||||
nifi.python.framework.source.directory=./python/framework
|
||||
nifi.python.extensions.source.directory.default=./python/extensions
|
||||
nifi.python.working.directory=./work/python
|
||||
nifi.python.max.processes=100
|
||||
nifi.python.max.processes.per.extension.type=10
|
||||
nifi.python.logs.directory=./logs
|
||||
|
||||
####################
|
||||
# State Management #
|
||||
####################
|
||||
nifi.state.management.configuration.file=conf/state-management.xml
|
||||
# The ID of the local state provider
|
||||
nifi.state.management.provider.local=local-provider
|
||||
# The ID of the cluster-wide state provider. This will be ignored if NiFi is not clustered but must be populated if running in a cluster.
|
||||
nifi.state.management.provider.cluster=zk-provider
|
||||
# Specifies whether or not this instance of NiFi should run an embedded ZooKeeper server
|
||||
nifi.state.management.embedded.zookeeper.start=false
|
||||
# Properties file that provides the ZooKeeper properties to use if <nifi.state.management.embedded.zookeeper.start> is set to true
|
||||
nifi.state.management.embedded.zookeeper.properties=./conf/zookeeper.properties
|
||||
|
||||
|
||||
# H2 Settings
|
||||
nifi.database.directory=./database_repository
|
||||
nifi.h2.url.append=;LOCK_TIMEOUT=25000;WRITE_DELAY=0;AUTO_SERVER=FALSE
|
||||
|
||||
# FlowFile Repository
|
||||
nifi.flowfile.repository.implementation=org.apache.nifi.controller.repository.WriteAheadFlowFileRepository
|
||||
nifi.flowfile.repository.wal.implementation=org.apache.nifi.wali.SequentialAccessWriteAheadLog
|
||||
nifi.flowfile.repository.directory=./flowfile_repository
|
||||
nifi.flowfile.repository.partitions=256
|
||||
nifi.flowfile.repository.checkpoint.interval=5 mins
|
||||
nifi.flowfile.repository.always.sync=false
|
||||
|
||||
nifi.swap.manager.implementation=org.apache.nifi.controller.FileSystemSwapManager
|
||||
nifi.queue.swap.threshold=20000
|
||||
nifi.swap.in.period=5 sec
|
||||
nifi.swap.in.threads=1
|
||||
nifi.swap.out.period=5 sec
|
||||
nifi.swap.out.threads=4
|
||||
|
||||
# Content Repository
|
||||
nifi.content.repository.implementation=org.apache.nifi.controller.repository.FileSystemRepository
|
||||
nifi.content.claim.max.appendable.size=50 KB
|
||||
nifi.content.repository.directory.default=./content_repository
|
||||
nifi.content.repository.archive.max.retention.period=12 hours
|
||||
nifi.content.repository.archive.max.usage.percentage=90%
|
||||
nifi.content.repository.archive.enabled=true
|
||||
nifi.content.repository.always.sync=false
|
||||
nifi.content.viewer.url=../nifi-content-viewer/
|
||||
|
||||
# Provenance Repository Properties
|
||||
nifi.provenance.repository.implementation=org.apache.nifi.provenance.WriteAheadProvenanceRepository
|
||||
nifi.provenance.repository.debug.frequency=1000000
|
||||
|
||||
# Persistent Provenance Repository Properties
|
||||
nifi.provenance.repository.directory.default=./provenance_repository
|
||||
nifi.provenance.repository.max.storage.time=24 hours
|
||||
nifi.provenance.repository.max.storage.size=1 GB
|
||||
nifi.provenance.repository.rollover.time=30 secs
|
||||
nifi.provenance.repository.rollover.events=1000
|
||||
nifi.provenance.repository.rollover.size=100 MB
|
||||
nifi.provenance.repository.query.threads=2
|
||||
nifi.provenance.repository.index.threads=2
|
||||
nifi.provenance.repository.compress.on.rollover=true
|
||||
nifi.provenance.repository.always.sync=false
|
||||
# Comma-separated list of fields. Fields that are not indexed will not be searchable. Valid fields are:
|
||||
# EventType, FlowFileUUID, Filename, TransitURI, ProcessorID, AlternateIdentifierURI, Relationship, Details
|
||||
nifi.provenance.repository.indexed.fields=EventType, FlowFileUUID, Filename, ProcessorID, Relationship
|
||||
# FlowFile Attributes that should be indexed and made searchable. Some examples to consider are filename, uuid, mime.type
|
||||
nifi.provenance.repository.indexed.attributes=
|
||||
# Large values for the shard size will result in more Java heap usage when searching the Provenance Repository
|
||||
# but should provide better performance
|
||||
nifi.provenance.repository.index.shard.size=500 MB
|
||||
# Indicates the maximum length that a FlowFile attribute can be when retrieving a Provenance Event from
|
||||
# the repository. If the length of any attribute exceeds this value, it will be truncated when the event is retrieved.
|
||||
nifi.provenance.repository.max.attribute.length=65536
|
||||
nifi.provenance.repository.concurrent.merge.threads=2
|
||||
|
||||
|
||||
# Volatile Provenance Respository Properties
|
||||
nifi.provenance.repository.buffer.size=100000
|
||||
|
||||
# Component Status Repository
|
||||
nifi.components.status.repository.implementation=org.apache.nifi.controller.status.history.VolatileComponentStatusRepository
|
||||
nifi.components.status.repository.buffer.size=1440
|
||||
nifi.components.status.snapshot.frequency=1 min
|
||||
|
||||
# Site to Site properties
|
||||
nifi.remote.input.host=
|
||||
nifi.remote.input.secure=false
|
||||
nifi.remote.input.socket.port=7780
|
||||
nifi.remote.input.http.enabled=true
|
||||
nifi.remote.input.http.transaction.ttl=30 sec
|
||||
nifi.remote.contents.cache.expiration=30 secs
|
||||
|
||||
# web properties #
|
||||
nifi.web.war.directory=./lib
|
||||
nifi.web.http.host=
|
||||
nifi.web.http.port=5670
|
||||
nifi.web.http.network.interface.default=
|
||||
nifi.web.https.host=
|
||||
nifi.web.https.port=
|
||||
nifi.web.https.network.interface.default=
|
||||
nifi.web.jetty.working.directory=./work/jetty
|
||||
nifi.web.jetty.threads=200
|
||||
nifi.web.max.header.size=16 KB
|
||||
nifi.web.proxy.context.path=
|
||||
nifi.web.proxy.host=
|
||||
|
||||
# security properties #
|
||||
nifi.sensitive.props.key=nifi-system-tests
|
||||
nifi.sensitive.props.key.protected=
|
||||
nifi.sensitive.props.algorithm=NIFI_PBKDF2_AES_GCM_256
|
||||
nifi.sensitive.props.additional.keys=
|
||||
|
||||
nifi.security.keystore=certs/keystore.p12
|
||||
nifi.security.keystoreType=PKCS12
|
||||
nifi.security.keystorePasswd=NiFiSystemKeyStoreProvider
|
||||
nifi.security.keyPasswd=NiFiSystemKeyStoreProvider
|
||||
nifi.security.truststore=certs/truststore.p12
|
||||
nifi.security.truststoreType=PKCS12
|
||||
nifi.security.truststorePasswd=NiFiSystemKeyStoreProvider
|
||||
nifi.security.user.authorizer=managed-authorizer
|
||||
nifi.security.user.login.identity.provider=
|
||||
nifi.security.ocsp.responder.url=
|
||||
nifi.security.ocsp.responder.certificate=
|
||||
|
||||
# OpenId Connect SSO Properties #
|
||||
nifi.security.user.oidc.discovery.url=
|
||||
nifi.security.user.oidc.connect.timeout=5 secs
|
||||
nifi.security.user.oidc.read.timeout=5 secs
|
||||
nifi.security.user.oidc.client.id=
|
||||
nifi.security.user.oidc.client.secret=
|
||||
nifi.security.user.oidc.preferred.jwsalgorithm=
|
||||
|
||||
# Apache Knox SSO Properties #
|
||||
nifi.security.user.knox.url=
|
||||
nifi.security.user.knox.publicKey=
|
||||
nifi.security.user.knox.cookieName=hadoop-jwt
|
||||
nifi.security.user.knox.audiences=
|
||||
|
||||
# Identity Mapping Properties #
|
||||
# These properties allow normalizing user identities such that identities coming from different identity providers
|
||||
# (certificates, LDAP, Kerberos) can be treated the same internally in NiFi. The following example demonstrates normalizing
|
||||
# DNs from certificates and principals from Kerberos into a common identity string:
|
||||
#
|
||||
# nifi.security.identity.mapping.pattern.dn=^CN=(.*?), OU=(.*?), O=(.*?), L=(.*?), ST=(.*?), C=(.*?)$
|
||||
# nifi.security.identity.mapping.value.dn=$1@$2
|
||||
# nifi.security.identity.mapping.transform.dn=NONE
|
||||
# nifi.security.identity.mapping.pattern.kerb=^(.*?)/instance@(.*?)$
|
||||
# nifi.security.identity.mapping.value.kerb=$1@$2
|
||||
# nifi.security.identity.mapping.transform.kerb=UPPER
|
||||
|
||||
# Group Mapping Properties #
|
||||
# These properties allow normalizing group names coming from external sources like LDAP. The following example
|
||||
# lowercases any group name.
|
||||
#
|
||||
# nifi.security.group.mapping.pattern.anygroup=^(.*)$
|
||||
# nifi.security.group.mapping.value.anygroup=$1
|
||||
# nifi.security.group.mapping.transform.anygroup=LOWER
|
||||
|
||||
# cluster common properties (all nodes must have same values) #
|
||||
nifi.cluster.protocol.heartbeat.interval=5 sec
|
||||
nifi.cluster.protocol.is.secure=false
|
||||
|
||||
# cluster node properties (only configure for cluster nodes) #
|
||||
nifi.cluster.is.node=false
|
||||
nifi.cluster.node.address=
|
||||
nifi.cluster.node.protocol.port=
|
||||
nifi.cluster.node.protocol.threads=10
|
||||
nifi.cluster.node.protocol.max.threads=50
|
||||
nifi.cluster.node.event.history.size=25
|
||||
nifi.cluster.node.connection.timeout=5 sec
|
||||
nifi.cluster.node.read.timeout=5 sec
|
||||
nifi.cluster.node.max.concurrent.requests=100
|
||||
nifi.cluster.firewall.file=
|
||||
nifi.cluster.flow.election.max.wait.time=5 mins
|
||||
nifi.cluster.flow.election.max.candidates=
|
||||
|
||||
# cluster load balancing properties #
|
||||
nifi.cluster.load.balance.host=
|
||||
nifi.cluster.load.balance.port=6342
|
||||
nifi.cluster.load.balance.connections.per.node=4
|
||||
nifi.cluster.load.balance.max.thread.count=8
|
||||
nifi.cluster.load.balance.comms.timeout=30 sec
|
||||
|
||||
# zookeeper properties, used for cluster management #
|
||||
nifi.zookeeper.connect.string=
|
||||
nifi.zookeeper.connect.timeout=3 secs
|
||||
nifi.zookeeper.session.timeout=3 secs
|
||||
nifi.zookeeper.root.node=/nifi
|
||||
|
||||
# Zookeeper properties for the authentication scheme used when creating acls on znodes used for cluster management
|
||||
# Values supported for nifi.zookeeper.auth.type are "default", which will apply world/anyone rights on znodes
|
||||
# and "sasl" which will give rights to the sasl/kerberos identity used to authenticate the nifi node
|
||||
# The identity is determined using the value in nifi.kerberos.service.principal and the removeHostFromPrincipal
|
||||
# and removeRealmFromPrincipal values (which should align with the kerberos.removeHostFromPrincipal and kerberos.removeRealmFromPrincipal
|
||||
# values configured on the zookeeper server).
|
||||
nifi.zookeeper.auth.type=
|
||||
nifi.zookeeper.kerberos.removeHostFromPrincipal=
|
||||
nifi.zookeeper.kerberos.removeRealmFromPrincipal=
|
||||
|
||||
# kerberos #
|
||||
nifi.kerberos.krb5.file=
|
||||
|
||||
# kerberos service principal #
|
||||
nifi.kerberos.service.principal=
|
||||
nifi.kerberos.service.keytab.location=
|
||||
|
||||
# kerberos spnego principal #
|
||||
nifi.kerberos.spnego.principal=
|
||||
nifi.kerberos.spnego.keytab.location=
|
||||
nifi.kerberos.spnego.authentication.expiration=12 hours
|
||||
|
||||
# external properties files for variable registry
|
||||
# supports a comma delimited list of file locations
|
||||
nifi.variable.registry.properties=
|
||||
@ -0,0 +1,32 @@
|
||||
<!--
|
||||
Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
contributor license agreements. See the NOTICE file distributed with
|
||||
this work for additional information regarding copyright ownership.
|
||||
The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
(the "License"); you may not use this file except in compliance with
|
||||
the License. You may obtain a copy of the License at
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
-->
|
||||
<stateManagement>
|
||||
<local-provider>
|
||||
<id>local-provider</id>
|
||||
<class>org.apache.nifi.controller.state.providers.local.WriteAheadLocalStateProvider</class>
|
||||
<property name="Directory">./state/local</property>
|
||||
<property name="Always Sync">false</property>
|
||||
<property name="Partitions">16</property>
|
||||
<property name="Checkpoint Interval">2 mins</property>
|
||||
</local-provider>
|
||||
<cluster-provider>
|
||||
<id>zk-provider</id>
|
||||
<class>org.apache.nifi.controller.state.providers.zookeeper.ZooKeeperStateProvider</class>
|
||||
<property name="Connect String">localhost:62181</property>
|
||||
<property name="Root Node">/nifi-integration-test</property>
|
||||
<property name="Session Timeout">30 seconds</property>
|
||||
<property name="Access Control">Open</property>
|
||||
</cluster-provider>
|
||||
</stateManagement>
|
||||
@ -0,0 +1,45 @@
|
||||
#
|
||||
#
|
||||
# Licensed to the Apache Software Foundation (ASF) under one
|
||||
# or more contributor license agreements. See the NOTICE file
|
||||
# distributed with this work for additional information
|
||||
# regarding copyright ownership. The ASF licenses this file
|
||||
# to you under the Apache License, Version 2.0 (the
|
||||
# "License"); you may not use this file except in compliance
|
||||
# with the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing,
|
||||
# software distributed under the License is distributed on an
|
||||
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
# KIND, either express or implied. See the License for the
|
||||
# specific language governing permissions and limitations
|
||||
# under the License.
|
||||
#
|
||||
#
|
||||
#
|
||||
|
||||
clientPort=62181
|
||||
initLimit=10
|
||||
autopurge.purgeInterval=24
|
||||
syncLimit=5
|
||||
tickTime=2000
|
||||
dataDir=target/state/zookeeper
|
||||
autopurge.snapRetainCount=30
|
||||
|
||||
#
|
||||
# Specifies the servers that are part of this zookeeper ensemble. For
|
||||
# every NiFi instance running an embedded zookeeper, there needs to be
|
||||
# a server entry below. For instance:
|
||||
#
|
||||
# server.1=nifi-node1-hostname:2888:3888
|
||||
# server.2=nifi-node2-hostname:2888:3888
|
||||
# server.3=nifi-node3-hostname:2888:3888
|
||||
#
|
||||
# The index of the server corresponds to the myid file that gets created
|
||||
# in the dataDir of each node running an embedded zookeeper. See the
|
||||
# administration guide for more details.
|
||||
#
|
||||
|
||||
server.1=localhost:5777:6777
|
||||
Loading…
x
Reference in New Issue
Block a user