Apache
/
nifi
mirror of https://github.com/apache/nifi.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

NIFI-11772 Removed flow.xml.gz support 

- Created NIFI-12203 to evaluate issues with flow comparison surfaced in JoinClusterWithDifferentFlow

This closes #7661

Signed-off-by: David Handermann <exceptionfactory@apache.org>
This commit is contained in:
Bence Simon 2023-08-30 13:55:40 +02:00 committed by exceptionfactory
parent 1b57980067
commit 3f13604c36
No known key found for this signature in database
GPG Key ID: 29B6A52D2AAE8DBA
211 changed files with 4325 additions and 12679 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
minifi/minifi-bootstrap/src/main/java/org/apache/nifi/minifi/bootstrap/service/MiNiFiConfigurationChangeListener.java
View File

@ -58,11 +58,6 @@ public class MiNiFiConfigurationChangeListener implements ConfigurationChangeLis
this.flowEnrichService = flowEnrichService;
}
@Override
public String getDescriptor() {
return "MiNiFiConfigurationChangeListener";
}
@Override
public void handleChange(InputStream flowConfigInputStream) throws ConfigurationChangeException {
logger.info("Received notification of a change");
@ -107,6 +102,11 @@ public class MiNiFiConfigurationChangeListener implements ConfigurationChangeLis
}
}
@Override
public String getDescriptor() {
return "MiNiFiConfigurationChangeListener";
}
private void setActiveFlowReference(ByteBuffer flowConfig) {
logger.debug("Setting active flow reference {} with content:\n{}", flowConfig, new String(flowConfig.array(), UTF_8));
runner.getConfigFileReference().set(flowConfig);

4
minifi/minifi-bootstrap/src/main/java/org/apache/nifi/minifi/bootstrap/service/MiNiFiPropertiesGenerator.java
View File

@ -137,11 +137,11 @@ public class MiNiFiPropertiesGenerator {
Triple.of(NiFiProperties.SECURITY_OCSP_RESPONDER_URL, EMPTY, EMPTY),
Triple.of(NiFiProperties.SECURITY_OCSP_RESPONDER_CERTIFICATE, EMPTY, EMPTY),
Triple.of(NiFiProperties.CLUSTER_IS_NODE, "false", EMPTY),
Triple.of(NiFiProperties.FLOW_CONFIGURATION_FILE, "./conf/flow.xml.gz", EMPTY)
Triple.of(NiFiProperties.FLOW_CONFIGURATION_FILE, "./conf/flow.json.gz", EMPTY)
);
static final Map<String, String> MINIFI_TO_NIFI_PROPERTY_MAPPING = Map.of(
MiNiFiProperties.NIFI_MINIFI_FLOW_CONFIG.getKey(), NiFiProperties.FLOW_CONFIGURATION_JSON_FILE,
MiNiFiProperties.NIFI_MINIFI_FLOW_CONFIG.getKey(), NiFiProperties.FLOW_CONFIGURATION_FILE,
MiNiFiProperties.NIFI_MINIFI_SECURITY_KEYSTORE.getKey(), NiFiProperties.SECURITY_KEYSTORE,
MiNiFiProperties.NIFI_MINIFI_SECURITY_KEYSTORE_TYPE.getKey(), NiFiProperties.SECURITY_KEYSTORE_TYPE,
MiNiFiProperties.NIFI_MINIFI_SECURITY_KEYSTORE_PASSWD.getKey(), NiFiProperties.SECURITY_KEYSTORE_PASSWD,

2
minifi/minifi-bootstrap/src/test/resources/MINIFI-216/nifi.properties.before
View File

@ -15,7 +15,7 @@
# Core Properties #
nifi.flow.configuration.file=./conf/flow.xml.gz
nifi.flow.configuration.file=./conf/flow.json.gz
nifi.flow.configuration.archive.enabled=false
nifi.flow.configuration.archive.dir=./conf/archive/
nifi.flowcontroller.autoResumeState=true

2
minifi/minifi-bootstrap/src/test/resources/MINIFI-245/nifi.properties.before
View File

@ -15,7 +15,7 @@
# Core Properties #
nifi.flow.configuration.file=./conf/flow.xml.gz
nifi.flow.configuration.file=./conf/flow.json.gz
nifi.flow.configuration.archive.enabled=false
nifi.flow.configuration.archive.dir=./conf/archive/
nifi.flowcontroller.autoResumeState=true

2
minifi/minifi-bootstrap/src/test/resources/MINIFI-277/nifi.properties
View File

@ -15,7 +15,7 @@
# Core Properties #
nifi.flow.configuration.file=./conf/flow.xml.gz
nifi.flow.configuration.file=./conf/flow.json.gz
nifi.flow.configuration.archive.enabled=false
nifi.flow.configuration.archive.dir=./conf/archive/
nifi.flowcontroller.autoResumeState=true

2
minifi/minifi-bootstrap/src/test/resources/NIFI-8753/nifi.properties.before
View File

@ -15,7 +15,7 @@
# Core Properties #
nifi.flow.configuration.file=./conf/flow.xml.gz
nifi.flow.configuration.file=./conf/flow.json.gz
nifi.flow.configuration.archive.enabled=false
nifi.flow.configuration.archive.dir=./conf/archive/
nifi.flowcontroller.autoResumeState=true

2
minifi/minifi-docs/src/main/markdown/System_Admin_Guide.md
View File

@ -434,7 +434,7 @@ The "Core Properties" section applies to the core framework as a whole.
*Property* | *Description*
------------------------------------------ | -----------
`flow controller graceful shutdown period` | Indicates the shutdown period. The default value is `10 sec`.
`flow service write delay interval` | When many changes are made to the *flow.xml*, this property specifies how long to wait before writing out the changes, so as to batch the changes into a single write. The default value is `500 ms`.
`flow service write delay interval` | When many changes are made to the *flow.json*, this property specifies how long to wait before writing out the changes, so as to batch the changes into a single write. The default value is `500 ms`.
`administrative yield duration` | If a component allows an unexpected exception to escape, it is considered a bug. As a result, the framework will pause (or administratively yield) the component for this amount of time. This is done so that the component does not use up massive amounts of system resources, since it is known to have problems in the existing state. The default value is `30 sec`.
`bored yield duration` | When a component has no work to do (i.e., is "bored"), this is the amount of time it will wait before checking to see if it has new data to work on. This way, it does not use up CPU resources by checking for new work too often. When setting this property, be aware that it could add extra latency for components that do not constantly have work to do, as once they go into this "bored" state, they will wait this amount of time before checking for more work. The default value is `10 millis`.
`max concurrent threads` | The maximum number of threads any processor can have running at one time.

2
minifi/minifi-integration-tests/src/test/resources/conf/nifi.properties
View File

@ -14,7 +14,7 @@
# limitations under the License.
# Core Properties #
nifi.flow.configuration.file=./target/flow.xml.gz
nifi.flow.configuration.file=./target/flow.json.gz
nifi.flow.configuration.archive.dir=./target/archive/
nifi.flowcontroller.autoResumeState=true
nifi.flowcontroller.graceful.shutdown.period=10 sec

4
minifi/minifi-nar-bundles/minifi-framework-bundle/minifi-framework/minifi-framework-core/src/main/java/org/apache/nifi/minifi/c2/C2NifiClientService.java
View File

@ -45,7 +45,7 @@ import static org.apache.nifi.minifi.commons.api.MiNiFiProperties.C2_SECURITY_KE
import static org.apache.nifi.minifi.commons.api.MiNiFiProperties.C2_SECURITY_TRUSTSTORE_LOCATION;
import static org.apache.nifi.minifi.commons.api.MiNiFiProperties.C2_SECURITY_TRUSTSTORE_PASSWORD;
import static org.apache.nifi.minifi.commons.api.MiNiFiProperties.C2_SECURITY_TRUSTSTORE_TYPE;
import static org.apache.nifi.util.NiFiProperties.FLOW_CONFIGURATION_JSON_FILE;
import static org.apache.nifi.util.NiFiProperties.FLOW_CONFIGURATION_FILE;
import com.fasterxml.jackson.databind.ObjectMapper;
import java.io.IOException;
@ -216,7 +216,7 @@ public class C2NifiClientService {
FlowEnrichService flowEnrichService = new FlowEnrichService(niFiProperties);
UpdateConfigurationStrategy updateConfigurationStrategy =
new DefaultUpdateConfigurationStrategy(flowController, flowService, flowEnrichService, niFiProperties.getProperty(FLOW_CONFIGURATION_JSON_FILE));
new DefaultUpdateConfigurationStrategy(flowController, flowService, flowEnrichService, niFiProperties.getProperty(FLOW_CONFIGURATION_FILE));
return new C2OperationHandlerProvider(List.of(
new UpdateConfigurationOperationHandler(client, flowIdHolder, updateConfigurationStrategy, emptyOperandPropertiesProvider),

6
minifi/minifi-nar-bundles/minifi-framework-bundle/minifi-framework/minifi-server/src/main/java/org/apache/nifi/minifi/StandardMiNiFiServer.java
View File

@ -24,7 +24,6 @@ import java.io.IOException;
import java.io.UncheckedIOException;
import java.util.Optional;
import org.apache.commons.lang3.StringUtils;
import org.apache.nifi.controller.FlowSerializationStrategy;
import org.apache.nifi.headless.HeadlessNiFiServer;
import org.apache.nifi.minifi.bootstrap.BootstrapListener;
import org.apache.nifi.minifi.c2.C2NifiClientService;
@ -87,11 +86,6 @@ public class StandardMiNiFiServer extends HeadlessNiFiServer implements MiNiFiSe
}
}
@Override
protected FlowSerializationStrategy getFlowSerializationStrategy() {
return FlowSerializationStrategy.WRITE_JSON_ONLY;
}
private void initC2() {
if (Boolean.parseBoolean(getNiFiProperties().getProperty(MiNiFiProperties.C2_ENABLE.getKey(), MiNiFiProperties.C2_ENABLE.getDefaultValue()))) {
NiFiProperties niFiProperties = getNiFiProperties();

2
nifi-bootstrap/src/test/resources/nifi.properties.dns-sans
View File

@ -14,7 +14,7 @@
# limitations under the License.
# Core Properties #
nifi.flow.configuration.file=./target/flow.xml.gz
nifi.flow.configuration.file=./target/flow.json.gz
# Removing most properties for testing...

2
nifi-bootstrap/src/test/resources/nifi.properties.ip-sans
View File

@ -14,7 +14,7 @@
# limitations under the License.
# Core Properties #
nifi.flow.configuration.file=./target/flow.xml.gz
nifi.flow.configuration.file=./target/flow.json.gz
# Removing most properties for testing...

2
nifi-bootstrap/src/test/resources/nifi.properties.keystore-password
View File

@ -14,7 +14,7 @@
# limitations under the License.
# Core Properties #
nifi.flow.configuration.file=./target/flow.xml.gz
nifi.flow.configuration.file=./target/flow.json.gz
# Removing most properties for testing...

2
nifi-bootstrap/src/test/resources/nifi.properties.no-https
View File

@ -14,7 +14,7 @@
# limitations under the License.
# Core Properties #
nifi.flow.configuration.file=./target/flow.xml.gz
nifi.flow.configuration.file=./target/flow.json.gz
# Removing most properties for testing...

2
nifi-bootstrap/src/test/resources/nifi.properties.no-keystore-types
View File

@ -14,7 +14,7 @@
# limitations under the License.
# Core Properties #
nifi.flow.configuration.file=./target/flow.xml.gz
nifi.flow.configuration.file=./target/flow.json.gz
# Removing most properties for testing...

2
nifi-bootstrap/src/test/resources/nifi.properties.no-keystores
View File

@ -14,7 +14,7 @@
# limitations under the License.
# Core Properties #
nifi.flow.configuration.file=./target/flow.xml.gz
nifi.flow.configuration.file=./target/flow.json.gz
# Removing most properties for testing...

2
nifi-bootstrap/src/test/resources/nifi.properties.only-keystore
View File

@ -14,7 +14,7 @@
# limitations under the License.
# Core Properties #
nifi.flow.configuration.file=./target/flow.xml.gz
nifi.flow.configuration.file=./target/flow.json.gz
# Removing most properties for testing...

2
nifi-bootstrap/src/test/resources/nifi.properties.only-truststore
View File

@ -14,7 +14,7 @@
# limitations under the License.
# Core Properties #
nifi.flow.configuration.file=./target/flow.xml.gz
nifi.flow.configuration.file=./target/flow.json.gz
# Removing most properties for testing...

2
nifi-bootstrap/src/test/resources/nifi.properties.stores-exist
View File

@ -14,7 +14,7 @@
# limitations under the License.
# Core Properties #
nifi.flow.configuration.file=./target/flow.xml.gz
nifi.flow.configuration.file=./target/flow.json.gz
# Removing most properties for testing...

2
nifi-bootstrap/src/test/resources/nifi.properties.success
View File

@ -14,7 +14,7 @@
# limitations under the License.
# Core Properties #
nifi.flow.configuration.file=./target/flow.xml.gz
nifi.flow.configuration.file=./target/flow.json.gz
# Removing most properties for testing...

2
nifi-bootstrap/src/test/resources/nifi.properties.truststore-password
View File

@ -14,7 +14,7 @@
# limitations under the License.
# Core Properties #
nifi.flow.configuration.file=./target/flow.xml.gz
nifi.flow.configuration.file=./target/flow.json.gz
# Removing most properties for testing...

55
nifi-commons/nifi-flow-encryptor/src/main/java/org/apache/nifi/flow/encryptor/StandardFlowEncryptor.java
View File

@ -1,55 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.nifi.flow.encryptor;
import org.apache.nifi.encrypt.PropertyEncryptor;
import java.io.BufferedInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.io.UncheckedIOException;
/**
* Standard Flow Encryptor handles reading Input Steam and writing Output Stream
*/
public class StandardFlowEncryptor implements FlowEncryptor {
private static final int XML_DECLARATION = '<';
/**
* Process Flow Configuration Stream replacing existing encrypted properties with new encrypted properties
*
* @param inputStream Flow Configuration Input Stream
* @param outputStream Flow Configuration Output Stream encrypted using new password
* @param inputEncryptor Property Encryptor for Input Configuration
* @param outputEncryptor Property Encryptor for Output Configuration
*/
@Override
public void processFlow(final InputStream inputStream, final OutputStream outputStream,
final PropertyEncryptor inputEncryptor, final PropertyEncryptor outputEncryptor) {
final BufferedInputStream bufferedInputStream = new BufferedInputStream(inputStream);
bufferedInputStream.mark(1);
try {
final int firstByte = bufferedInputStream.read();
bufferedInputStream.reset();
final FlowEncryptor flowEncryptor = (firstByte == XML_DECLARATION) ? new XmlFlowEncryptor() : new JsonFlowEncryptor();
flowEncryptor.processFlow(bufferedInputStream, outputStream, inputEncryptor, outputEncryptor);
} catch (final IOException e) {
throw new UncheckedIOException(e);
}
}
}

80
nifi-commons/nifi-flow-encryptor/src/main/java/org/apache/nifi/flow/encryptor/XmlFlowEncryptor.java
View File

@ -1,80 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.nifi.flow.encryptor;
import org.apache.nifi.encrypt.PropertyEncryptor;
import org.apache.nifi.xml.processing.stream.StandardXMLEventReaderProvider;
import org.apache.nifi.xml.processing.stream.XMLEventReaderProvider;
import javax.xml.stream.XMLEventFactory;
import javax.xml.stream.XMLEventReader;
import javax.xml.stream.XMLEventWriter;
import javax.xml.stream.XMLOutputFactory;
import javax.xml.stream.XMLStreamException;
import javax.xml.stream.events.Characters;
import javax.xml.stream.events.XMLEvent;
import javax.xml.transform.stream.StreamSource;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.io.UncheckedIOException;
import java.nio.charset.StandardCharsets;
import java.util.regex.Matcher;
public class XmlFlowEncryptor extends AbstractFlowEncryptor {
private static final XMLEventReaderProvider eventReaderProvider = new StandardXMLEventReaderProvider();
@Override
public void processFlow(final InputStream inputStream, final OutputStream outputStream,
final PropertyEncryptor inputEncryptor, final PropertyEncryptor outputEncryptor) {
final XMLOutputFactory xmlOutputFactory = XMLOutputFactory.newInstance();
final XMLEventFactory eventFactory = XMLEventFactory.newInstance();
try {
final XMLEventReader reader = eventReaderProvider.getEventReader(new StreamSource(inputStream));
final XMLEventWriter writer = xmlOutputFactory.createXMLEventWriter(outputStream, StandardCharsets.UTF_8.name());
while (reader.hasNext()) {
final XMLEvent event = reader.nextEvent();
if (event.getEventType() == XMLEvent.CHARACTERS) {
final Characters characters = event.asCharacters();
final String value = characters.getData();
final Matcher matcher = ENCRYPTED_PATTERN.matcher(value);
if (matcher.matches()) {
final String processedValue = getOutputEncrypted(matcher.group(FIRST_GROUP), inputEncryptor, outputEncryptor);
writer.add(eventFactory.createCharacters(processedValue));
} else {
writer.add(characters);
}
} else if (event.getEventType() == XMLEvent.START_DOCUMENT) {
writer.add(event);
writer.add(eventFactory.createSpace(System.lineSeparator()));
} else {
writer.add(event);
}
}
writer.flush();
writer.close();
reader.close();
outputStream.close();
inputStream.close();
} catch (final XMLStreamException e) {
throw new RuntimeException("Flow XML Processing Failed", e);
} catch (final IOException e) {
throw new UncheckedIOException("Failed Processing Flow Configuration", e);
}
}
}

17
nifi-commons/nifi-flow-encryptor/src/main/java/org/apache/nifi/flow/encryptor/command/FlowEncryptorCommand.java
View File

@ -16,12 +16,6 @@
*/
package org.apache.nifi.flow.encryptor.command;
import org.apache.nifi.encrypt.PropertyEncryptionMethod;
import org.apache.nifi.encrypt.PropertyEncryptor;
import org.apache.nifi.encrypt.PropertyEncryptorBuilder;
import org.apache.nifi.flow.encryptor.FlowEncryptor;
import org.apache.nifi.flow.encryptor.StandardFlowEncryptor;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileOutputStream;
@ -40,6 +34,11 @@ import java.util.Properties;
import java.util.stream.Collectors;
import java.util.zip.GZIPInputStream;
import java.util.zip.GZIPOutputStream;
import org.apache.nifi.encrypt.PropertyEncryptionMethod;
import org.apache.nifi.encrypt.PropertyEncryptor;
import org.apache.nifi.encrypt.PropertyEncryptorBuilder;
import org.apache.nifi.flow.encryptor.FlowEncryptor;
import org.apache.nifi.flow.encryptor.JsonFlowEncryptor;
/**
* Flow Encryptor Command capable of updating Sensitive Properties Key or Algorithm as well as Flow Configuration
@ -53,9 +52,7 @@ class FlowEncryptorCommand implements Runnable {
protected static final String CONFIGURATION_FILE = "nifi.flow.configuration.file";
protected static final String CONFIGURATION_JSON_FILE = "nifi.flow.configuration.json.file";
private static final List<String> CONFIGURATION_FILES = Arrays.asList(CONFIGURATION_FILE, CONFIGURATION_JSON_FILE);
private static final List<String> CONFIGURATION_FILES = Arrays.asList(CONFIGURATION_FILE);
private static final String FLOW_PREFIX = "nifi.flow.";
@ -131,7 +128,7 @@ class FlowEncryptorCommand implements Runnable {
final String inputPropertiesKey = getKey(properties);
final PropertyEncryptor inputEncryptor = getPropertyEncryptor(inputPropertiesKey, inputAlgorithm);
final FlowEncryptor flowEncryptor = new StandardFlowEncryptor();
final FlowEncryptor flowEncryptor = new JsonFlowEncryptor();
flowEncryptor.processFlow(flowInputStream, flowOutputStream, inputEncryptor, outputEncryptor);
}

89
nifi-commons/nifi-flow-encryptor/src/test/java/org/apache/nifi/flow/encryptor/StandardFlowEncryptorTest.java → nifi-commons/nifi-flow-encryptor/src/test/java/org/apache/nifi/flow/encryptor/JsonFlowEncryptorTest.java
View File

@ -16,11 +16,8 @@
*/
package org.apache.nifi.flow.encryptor;
import org.apache.nifi.encrypt.PropertyEncryptionMethod;
import org.apache.nifi.encrypt.PropertyEncryptor;
import org.apache.nifi.encrypt.PropertyEncryptorBuilder;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertTrue;
import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
@ -31,11 +28,13 @@ import java.util.Objects;
import java.util.UUID;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import org.apache.nifi.encrypt.PropertyEncryptionMethod;
import org.apache.nifi.encrypt.PropertyEncryptor;
import org.apache.nifi.encrypt.PropertyEncryptorBuilder;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertTrue;
public class StandardFlowEncryptorTest {
public class JsonFlowEncryptorTest {
private static final String INPUT_KEY = UUID.randomUUID().toString();
private static final String OUTPUT_KEY = UUID.randomUUID().toString();
@ -50,22 +49,23 @@ public class StandardFlowEncryptorTest {
private PropertyEncryptor outputEncryptor;
private StandardFlowEncryptor flowEncryptor;
private FlowEncryptor flowEncryptor;
@BeforeEach
public void setEncryptors() {
inputEncryptor = getPropertyEncryptor(INPUT_KEY, PropertyEncryptionMethod.NIFI_PBKDF2_AES_GCM_256.name());
outputEncryptor = getPropertyEncryptor(OUTPUT_KEY, PropertyEncryptionMethod.NIFI_ARGON2_AES_GCM_256.name());
flowEncryptor = new StandardFlowEncryptor();
flowEncryptor = new JsonFlowEncryptor();
}
@Test
public void testProcessEncrypted() {
final String property = StandardFlowEncryptorTest.class.getSimpleName();
final String property = JsonFlowEncryptorTest.class.getSimpleName();
final String encryptedProperty = String.format(ENCRYPTED_FORMAT, inputEncryptor.encrypt(property));
final String encryptedRow = String.format("<?xml version=\"1.0\" encoding=\"UTF-8\" standalone=\"no\"?>%n" +
"<test>%s</test>", encryptedProperty);
final String encryptedRow = String.format(
"{\"properties\":{\"username\":\"sample_username\",\"test\":\"%s\",\"position\":1.123456789123456789}}",
encryptedProperty);
final InputStream inputStream = new ByteArrayInputStream(encryptedRow.getBytes(StandardCharsets.UTF_8));
final ByteArrayOutputStream outputStream = new ByteArrayOutputStream();
@ -83,8 +83,7 @@ public class StandardFlowEncryptorTest {
@Test
public void testProcessNoEncrypted() {
final String property = String.format("<?xml version=\"1.0\" encoding=\"UTF-8\" standalone=\"no\"?>%n" +
"<test>%s</test>", StandardFlowEncryptorTest.class.getSimpleName());
final String property = String.format("{\"properties\":{\"username\":\"sample_username\",\"test\":\"%s\",\"position\":1.123456789123456789}}", JsonFlowEncryptorTest.class.getSimpleName());
final InputStream inputStream = new ByteArrayInputStream(property.getBytes(StandardCharsets.UTF_8));
final ByteArrayOutputStream outputStream = new ByteArrayOutputStream();
@ -92,12 +91,12 @@ public class StandardFlowEncryptorTest {
flowEncryptor.processFlow(inputStream, outputStream, inputEncryptor, outputEncryptor);
final String outputProperty = outputStream.toString();
assertEquals(removeXmlDeclaration(property).trim(), removeXmlDeclaration(outputProperty).trim());
assertEquals(property, outputProperty);
}
@Test
public void testProcessJson() throws IOException {
final String password = StandardFlowEncryptorTest.class.getSimpleName();
final String password = JsonFlowEncryptorTest.class.getSimpleName();
final String encryptedPassword = String.format(ENCRYPTED_FORMAT, inputEncryptor.encrypt(password));
final String sampleFlowJson = getSampleFlowJson(encryptedPassword);
@ -113,21 +112,6 @@ public class StandardFlowEncryptorTest {
}
}
@Test
public void testProcessXml() throws IOException {
final String password = StandardFlowEncryptorTest.class.getSimpleName();
final String encryptedPassword = String.format(ENCRYPTED_FORMAT, inputEncryptor.encrypt(password));
final String sampleFlowXml = getSampleFlowXml(encryptedPassword);
try (final InputStream inputStream = new ByteArrayInputStream(sampleFlowXml.getBytes(StandardCharsets.UTF_8))) {
try (final ByteArrayOutputStream outputStream = new ByteArrayOutputStream()) {
flowEncryptor.processFlow(inputStream, outputStream, inputEncryptor, outputEncryptor);
final String outputXml = outputStream.toString();
compareFlow(removeXmlDeclaration(sampleFlowXml).trim(), removeXmlDeclaration(outputXml).trim());
}
}
}
private PropertyEncryptor getPropertyEncryptor(final String propertiesKey, final String propertiesAlgorithm) {
return new PropertyEncryptorBuilder(propertiesKey).setAlgorithm(propertiesAlgorithm).build();
}
@ -145,43 +129,4 @@ public class StandardFlowEncryptorTest {
Objects.requireNonNull(password);
return String.format("{\"properties\":{\"username\":\"sample_username\",\"password\":\"%s\",\"position\":1.123456789123456789}}", password);
}
private String getSampleFlowXml(final String password) {
Objects.requireNonNull(password);
final String flowXml = String.format("<?xml version=\"1.0\" encoding=\"UTF-8\" standalone=\"no\"?>%n" +
"<processor>%n" +
"\t<property>%n" +
"\t\t<name>Username</name>%n" +
"\t\t<value>SAMPLE_USERNAME</value>%n" +
"\t</property>%n" +
"\t<property>%n" +
"\t\t<name>Password</name>%n" +
"\t\t<value>%s</value>%n" +
"\t</property>%n" +
"</processor>", password);
return getProcessedFlowXml(flowXml);
}
private String getProcessedFlowXml(final String flowXml) {
final PropertyEncryptor encryptor = new PropertyEncryptor() {
@Override
public String encrypt(String property) {
return property;
}
@Override
public String decrypt(String encryptedProperty) {
return encryptedProperty;
}
};
final InputStream inputStream = new ByteArrayInputStream(flowXml.getBytes(StandardCharsets.UTF_8));
final ByteArrayOutputStream outputStream = new ByteArrayOutputStream();
flowEncryptor.processFlow(inputStream, outputStream, encryptor, encryptor);
return outputStream.toString();
}
private String removeXmlDeclaration(final String xmlFlow) {
return xmlFlow.replaceAll("<\\?xml.+\\?>", "");
}
}

13
nifi-commons/nifi-flow-encryptor/src/test/java/org/apache/nifi/flow/encryptor/command/FlowEncryptorCommandTest.java
View File

@ -44,12 +44,8 @@ public class FlowEncryptorCommandTest {
private static final String FLOW_CONTENTS_JSON = "{\"property\":\"value\"}";
private static final String FLOW_CONTENTS_XML = "<property><value>PROPERTY</value></property>";
private static final String JSON_GZ = ".json.gz";
private static final String XML_GZ = ".xml.gz";
private static final String PROPERTIES_EXTENSION = ".properties";
private static final String BLANK_PROPERTIES = "/blank.nifi.properties";
@ -123,19 +119,16 @@ public class FlowEncryptorCommandTest {
}
protected static Path getBlankNiFiProperties() throws IOException, URISyntaxException {
final Path flowConfiguration = getFlowConfiguration(FLOW_CONTENTS_XML, XML_GZ);
final Path flowConfigurationJson = getFlowConfiguration(FLOW_CONTENTS_JSON, JSON_GZ);
return getNiFiProperties(flowConfiguration, flowConfigurationJson, BLANK_PROPERTIES);
return getNiFiProperties(flowConfigurationJson, BLANK_PROPERTIES);
}
protected static Path getPopulatedNiFiProperties() throws IOException, URISyntaxException {
final Path flowConfiguration = getFlowConfiguration(FLOW_CONTENTS_XML, XML_GZ);
final Path flowConfigurationJson = getFlowConfiguration(FLOW_CONTENTS_JSON, JSON_GZ);
return getNiFiProperties(flowConfiguration, flowConfigurationJson, POPULATED_PROPERTIES);
return getNiFiProperties(flowConfigurationJson, POPULATED_PROPERTIES);
}
private static Path getNiFiProperties(
final Path flowConfigurationPath,
final Path flowConfigurationJsonPath,
String propertiesResource
) throws IOException, URISyntaxException {
@ -143,8 +136,6 @@ public class FlowEncryptorCommandTest {
final List<String> sourceProperties = Files.readAllLines(sourcePropertiesPath);
final List<String> flowProperties = sourceProperties.stream().map(line -> {
if (line.startsWith(FlowEncryptorCommand.CONFIGURATION_FILE)) {
return line + flowConfigurationPath;
} else if (line.startsWith(FlowEncryptorCommand.CONFIGURATION_JSON_FILE)) {
return flowConfigurationJsonPath == null ? line : line + flowConfigurationJsonPath;
} else {
return line;

1
nifi-commons/nifi-flow-encryptor/src/test/resources/blank.nifi.properties
View File

@ -15,4 +15,3 @@
nifi.sensitive.props.key=
nifi.sensitive.props.algorithm=
nifi.flow.configuration.file=
nifi.flow.configuration.json.file=

1
nifi-commons/nifi-flow-encryptor/src/test/resources/populated.nifi.properties
View File

@ -15,4 +15,3 @@
nifi.sensitive.props.key=D5E41AC1-EEF8-4A54-930D-593F749AE95C
nifi.sensitive.props.algorithm=NIFI_ARGON2_AES_GCM_256
nifi.flow.configuration.file=
nifi.flow.configuration.json.file=

17
nifi-commons/nifi-properties/src/main/java/org/apache/nifi/util/NiFiProperties.java
View File

@ -54,7 +54,6 @@ public class NiFiProperties extends ApplicationProperties {
// core properties
public static final String PROPERTIES_FILE_PATH = "nifi.properties.file.path";
public static final String FLOW_CONFIGURATION_FILE = "nifi.flow.configuration.file";
public static final String FLOW_CONFIGURATION_JSON_FILE = "nifi.flow.configuration.json.file";
public static final String FLOW_CONFIGURATION_ARCHIVE_ENABLED = "nifi.flow.configuration.archive.enabled";
public static final String FLOW_CONFIGURATION_ARCHIVE_DIR = "nifi.flow.configuration.archive.dir";
public static final String FLOW_CONFIGURATION_ARCHIVE_MAX_TIME = "nifi.flow.configuration.archive.max.time";
@ -452,7 +451,6 @@ public class NiFiProperties extends ApplicationProperties {
super(props);
}
// getters for core properties //
public File getFlowConfigurationFile() {
try {
return new File(getProperty(FLOW_CONFIGURATION_FILE));
@ -461,21 +459,6 @@ public class NiFiProperties extends ApplicationProperties {
}
}
public File getFlowConfigurationJsonFile() {
final String jsonFilename = getProperty(FLOW_CONFIGURATION_JSON_FILE);
if (jsonFilename != null) {
return new File(jsonFilename);
}
final File xmlFile = getFlowConfigurationFile();
final String xmlFilename = xmlFile.getName();
if (xmlFilename.contains(".xml")) {
return new File(xmlFile.getParentFile(), xmlFilename.replace(".xml", ".json"));
}
return new File(xmlFile.getParentFile(), xmlFilename.replace(".gz", "") + ".json.gz");
}
public File getFlowConfigurationFileDir() {
try {
return getFlowConfigurationFile().getParentFile();

2
nifi-commons/nifi-properties/src/test/resources/NiFiProperties/conf/nifi.blank.properties
View File

@ -14,7 +14,7 @@
# limitations under the License.
# Core Properties #
nifi.flow.configuration.file=./target/flow.xml.gz
nifi.flow.configuration.file=./target/flow.json.gz
nifi.flow.configuration.archive.dir=./target/archive/
nifi.flowcontroller.autoResumeState=true
nifi.flowcontroller.graceful.shutdown.period=10 sec

2
nifi-commons/nifi-properties/src/test/resources/NiFiProperties/conf/nifi.missing.properties
View File

@ -14,7 +14,7 @@
# limitations under the License.
# Core Properties #
nifi.flow.configuration.file=./target/flow.xml.gz
nifi.flow.configuration.file=./target/flow.json.gz
nifi.flow.configuration.archive.dir=./target/archive/
nifi.flowcontroller.autoResumeState=true
nifi.flowcontroller.graceful.shutdown.period=10 sec

2
nifi-commons/nifi-properties/src/test/resources/NiFiProperties/conf/nifi.properties
View File

@ -14,7 +14,7 @@
# limitations under the License.
# Core Properties #
nifi.flow.configuration.file=./target/flow.xml.gz
nifi.flow.configuration.file=./target/flow.json.gz
nifi.flow.configuration.archive.dir=./target/archive/
nifi.flowcontroller.autoResumeState=true
nifi.flowcontroller.graceful.shutdown.period=10 sec

16
nifi-docs/src/main/asciidoc/administration-guide.adoc
View File

@ -3316,11 +3316,7 @@ The first section of the _nifi.properties_ file is for the Core Properties. Thes
|===
|*Property*|*Description*
|`nifi.flow.configuration.file`*|The location of the XML-based flow configuration file. The default value is `./conf/flow.xml.gz`. This is a legacy property. Older versions of NiFi used an
XML-formatted file to store the flow configuration. However, newer versions use a JSON representation. In order to maintain backward compatibility of flows and still load flows developed using
older versions of NiFi, upon startup, NiFi will use the `nifi.flow.configuration.json.file` first. If the file exists, it will be used. However, if it does not exist, NiFi will fall back to this
property to determine the XML version of the file and use it.
|`nifi.flow.configuration.json.file`*|The location of the flow configuration file (i.e., the file that contains what is currently displayed on the NiFi graph). The default value is `./conf/flow.json.gz`.
|`nifi.flow.configuration.file`*|The location of the JSON-based flow configuration file. The default value is `./conf/flow.json.gz`.
|`nifi.flow.configuration.archive.enabled`*|Specifies whether NiFi creates a backup copy of the flow automatically when the flow is updated. The default value is `true`.
|`nifi.flow.configuration.archive.dir`*|The location of the archive directory where backup copies of the _flow.json_ are saved. The default value is `./conf/archive`. NiFi removes old archive files to limit disk usage based on archived file lifespan, total size, and number of files, as specified with `nifi.flow.configuration.archive.max.time`, `max.storage` and `max.count` properties respectively. If none of these limitation for archiving is specified, NiFi uses default conditions, that is `30 days` for `max.time` and `500 MB` for `max.storage`. +
This cleanup mechanism takes into account only automatically created archived _flow.json_ files. If there are other files or directories in this archive directory, NiFi will ignore them. Automatically created archives have filename with ISO 8601 format timestamp prefix followed by `<original-filename>`. That is `<year><month><day>T<hour><minute><second>+<timezone offset>_<original filename>`. For example, `20160706T160719+0900_flow.json.gz`. NiFi checks filenames when it cleans archive directory. If you would like to keep a particular archive in this directory without worrying about NiFi deleting it, you can do so by copying it with a different filename pattern.
@ -4398,7 +4394,7 @@ If you are encrypting sensitive component properties in your dataflow via the se
If the below properties point to directories inside the NiFi base installation path, you must copy the target directories to the new NiFi. Stop your existing NiFi installation before you do this.
|`nifi.flow.configuration.json.file=`
|`nifi.flow.configuration.file=`
If you have retained the default value (`./conf/flow.json.gz`), copy _flow.json.gz_ from the existing to the new NiFi base install conf directory.
@ -4466,7 +4462,7 @@ When a value is set for `nifi.sensitive.props.key` in _nifi.properties_, the spe
2. Encrypts all the sensitive values with a specified new key.
3. Updates the _nifi.properties_ and _flow.json.gz_ files or creates new versions of them.
As an example, assume version 1.9.2 is the existing NiFi instance and the sensitive properties key is set to `password`. The goal is to move the 1.9.2 _flow.xml.gz_ to a 1.10.0 instance with a new sensitive properties key: `new_password`. Running the following Encrypt-Config command would read in the _flow.xml.gz_ and _nifi.properties_ files from 1.9.2 using the original sensitive properties key and write out new versions in 1.10.0 with the sensitive properties encrypted with the new password:
As an example, assume version 1.9.2 is the existing NiFi instance and the sensitive properties key is set to `password`. The goal is to move the 1.9.2 _flow.json.gz_ to a 1.10.0 instance with a new sensitive properties key: `new_password`. Running the following Encrypt-Config command would read in the _flow.json.gz_ and _nifi.properties_ files from 1.9.2 using the original sensitive properties key and write out new versions in 1.10.0 with the sensitive properties encrypted with the new password:
```
$ ./nifi-toolkit-1.10.0/bin/encrypt-config.sh -f /path/to/nifi/nifi-1.9.2/conf/flow.json.gz -g /path/to/nifi/nifi-1.10.0/conf/flow.json.gz -s new_password -n /path/to/nifi/nifi-1.9.2/conf/nifi
@ -4495,7 +4491,6 @@ $ ./bin/nifi.sh set-sensitive-properties-algorithm <algorithm>
The command reads the following flow configuration file properties from _nifi.properties_:
- `nifi.flow.configuration.file`
- `nifi.flow.configuration.json.file`
The command checks for the existence of each file and updates the sensitive property values found.
@ -4511,12 +4506,11 @@ The following command can be used to read an existing flow configuration and set
$ ./bin/nifi.sh set-sensitive-properties-key <sensitivePropertiesKey>
```
The command reads the following flow configuration file properties from _nifi.properties_:
The command reads the following flow configuration file property from _nifi.properties_:
- `nifi.flow.configuration.file`
- `nifi.flow.configuration.json.file`
The command checks for the existence of each file and updates the sensitive property values found.
The command checks for the existence of the file file and updates the sensitive property values found.
The minimum required length for a new sensitive properties key is 12 characters.

31
nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-client-dto/src/main/java/org/apache/nifi/web/api/dto/PortDTO.java
View File

@ -20,7 +20,6 @@ import io.swagger.annotations.ApiModelProperty;
import javax.xml.bind.annotation.XmlType;
import java.util.Collection;
import java.util.Set;
/**
* The details for a port within this NiFi flow.
@ -34,8 +33,6 @@ public class PortDTO extends ComponentDTO {
private String type;
private Boolean transmitting;
private Integer concurrentlySchedulableTaskCount;
private Set<String> userAccessControl;
private Set<String> groupAccessControl;
private Boolean allowRemoteAccess;
private String portFunction;
@ -129,34 +126,6 @@ public class PortDTO extends ComponentDTO {
this.transmitting = transmitting;
}
/**
* @return groups that are allowed to access this port
*/
@ApiModelProperty(
value = "The user groups that are allowed to access the port."
)
public Set<String> getGroupAccessControl() {
return groupAccessControl;
}
public void setGroupAccessControl(Set<String> groupAccessControl) {
this.groupAccessControl = groupAccessControl;
}
/**
* @return users that are allowed to access this port
*/
@ApiModelProperty(
value = "The users that are allowed to access the port."
)
public Set<String> getUserAccessControl() {
return userAccessControl;
}
public void setUserAccessControl(Set<String> userAccessControl) {
this.userAccessControl = userAccessControl;
}
/**
* Gets the validation errors from this port. These validation errors represent the problems with the port that must be resolved before it can be started.
*

170
nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-file-authorizer/src/main/java/org/apache/nifi/authorization/FileAccessPolicyProvider.java
View File

@ -25,12 +25,10 @@ import org.apache.nifi.authorization.exception.UninheritableAuthorizationsExcept
import org.apache.nifi.authorization.file.generated.Authorizations;
import org.apache.nifi.authorization.file.generated.Policies;
import org.apache.nifi.authorization.file.generated.Policy;
import org.apache.nifi.authorization.resource.ResourceFactory;
import org.apache.nifi.authorization.resource.ResourceType;
import org.apache.nifi.authorization.util.IdentityMapping;
import org.apache.nifi.authorization.util.IdentityMappingUtil;
import org.apache.nifi.components.PropertyValue;
import org.apache.nifi.user.generated.Users;
import org.apache.nifi.util.FlowInfo;
import org.apache.nifi.util.FlowParser;
import org.apache.nifi.util.NiFiProperties;
@ -131,7 +129,6 @@ public class FileAccessPolicyProvider implements ConfigurableAccessPolicyProvide
private File restoreAuthorizationsFile;
private String rootGroupId;
private String initialAdminIdentity;
private String legacyAuthorizedUsersFile;
private Set<String> nodeIdentities;
private String nodeGroupIdentifier;
private List<PortDTO> ports = new ArrayList<>();
@ -213,10 +210,6 @@ public class FileAccessPolicyProvider implements ConfigurableAccessPolicyProvide
final PropertyValue initialAdminIdentityProp = configurationContext.getProperty(PROP_INITIAL_ADMIN_IDENTITY);
initialAdminIdentity = initialAdminIdentityProp.isSet() ? IdentityMappingUtil.mapIdentity(initialAdminIdentityProp.getValue(), identityMappings) : null;
// get the value of the legacy authorized users file
final PropertyValue legacyAuthorizedUsersProp = configurationContext.getProperty(FileAuthorizer.PROP_LEGACY_AUTHORIZED_USERS_FILE);
legacyAuthorizedUsersFile = legacyAuthorizedUsersProp.isSet() ? legacyAuthorizedUsersProp.getValue() : null;
// extract any node identities
nodeIdentities = new HashSet<>();
for (Map.Entry<String,String> entry : configurationContext.getProperties().entrySet()) {
@ -594,20 +587,14 @@ public class FileAccessPolicyProvider implements ConfigurableAccessPolicyProvide
final AuthorizationsHolder authorizationsHolder = new AuthorizationsHolder(authorizations);
final boolean emptyAuthorizations = authorizationsHolder.getAllPolicies().isEmpty();
final boolean hasInitialAdminIdentity = (initialAdminIdentity != null && !StringUtils.isBlank(initialAdminIdentity));
final boolean hasLegacyAuthorizedUsers = (legacyAuthorizedUsersFile != null && !StringUtils.isBlank(legacyAuthorizedUsersFile));
// if we are starting fresh then we might need to populate an initial admin or convert legacy users
if (emptyAuthorizations) {
parseFlow();
if (hasInitialAdminIdentity && hasLegacyAuthorizedUsers) {
throw new AuthorizerCreationException("Cannot provide an Initial Admin Identity and a Legacy Authorized Users File");
} else if (hasInitialAdminIdentity) {
if (hasInitialAdminIdentity) {
logger.info("Populating authorizations for Initial Admin: " + initialAdminIdentity);
populateInitialAdmin(authorizations);
} else if (hasLegacyAuthorizedUsers) {
logger.info("Converting " + legacyAuthorizedUsersFile + " to new authorizations model");
convertLegacyAuthorizedUsers(authorizations);
}
populateNodes(authorizations);
@ -652,15 +639,7 @@ public class FileAccessPolicyProvider implements ConfigurableAccessPolicyProvide
*/
private void parseFlow() throws SAXException {
final FlowParser flowParser = new FlowParser();
final File flowConfigurationFile;
final File jsonFile = properties.getFlowConfigurationJsonFile();
if (jsonFile.exists()) {
flowConfigurationFile = jsonFile;
} else {
flowConfigurationFile = properties.getFlowConfigurationFile();
}
final File flowConfigurationFile = properties.getFlowConfigurationFile();
final FlowInfo flowInfo = flowParser.parse(flowConfigurationFile);
if (flowInfo != null) {
@ -741,151 +720,6 @@ public class FileAccessPolicyProvider implements ConfigurableAccessPolicyProvide
}
}
/**
* Unmarshalls an existing authorized-users.xml and converts the object model to the new model.
*
* @param authorizations the current Authorizations instance that policies will be added to
* @throws AuthorizerCreationException if the legacy authorized users file that was provided does not exist
* @throws JAXBException if the legacy authorized users file that was provided could not be unmarshalled
*/
private void convertLegacyAuthorizedUsers(final Authorizations authorizations) throws AuthorizerCreationException, JAXBException {
final File authorizedUsersFile = new File(legacyAuthorizedUsersFile);
if (!authorizedUsersFile.exists()) {
throw new AuthorizerCreationException("Legacy Authorized Users File '" + legacyAuthorizedUsersFile + "' does not exists");
}
final Unmarshaller unmarshaller = JAXB_USERS_CONTEXT.createUnmarshaller();
unmarshaller.setSchema(usersSchema);
final XMLStreamReader xsr;
try {
final XMLStreamReaderProvider provider = new StandardXMLStreamReaderProvider();
xsr = provider.getStreamReader(new StreamSource(authorizedUsersFile));
} catch (final ProcessingException e) {
logger.error("Encountered an error reading authorized users file: ", e);
throw new JAXBException("Error reading authorized users file", e);
}
final JAXBElement<Users> element = unmarshaller.unmarshal(
xsr, org.apache.nifi.user.generated.Users.class);
final org.apache.nifi.user.generated.Users users = element.getValue();
if (users.getUser().isEmpty()) {
logger.info("Legacy Authorized Users File contained no users, nothing to convert");
return;
}
// get all the user DNs into a list
List<String> userIdentities = new ArrayList<>();
for (org.apache.nifi.user.generated.User legacyUser : users.getUser()) {
userIdentities.add(IdentityMappingUtil.mapIdentity(legacyUser.getDn(), identityMappings));
}
// sort the list and pull out the first identity
Collections.sort(userIdentities);
final String seedIdentity = userIdentities.get(0);
// create mapping from Role to access policies
final Map<Role,Set<RoleAccessPolicy>> roleAccessPolicies = RoleAccessPolicy.getMappings(rootGroupId);
final List<Policy> allPolicies = new ArrayList<>();
for (org.apache.nifi.user.generated.User legacyUser : users.getUser()) {
// create the identifier of the new user based on the DN
final String legacyUserDn = IdentityMappingUtil.mapIdentity(legacyUser.getDn(), identityMappings);
final User user = userGroupProvider.getUserByIdentity(legacyUserDn);
if (user == null) {
throw new AuthorizerCreationException("Unable to locate legacy user " + legacyUserDn + " to seed policies.");
}
// create policies based on the given role
for (org.apache.nifi.user.generated.Role jaxbRole : legacyUser.getRole()) {
Role role = Role.valueOf(jaxbRole.getName());
Set<RoleAccessPolicy> policies = roleAccessPolicies.get(role);
for (RoleAccessPolicy roleAccessPolicy : policies) {
// get the matching policy, or create a new one
Policy policy = getOrCreatePolicy(
allPolicies,
seedIdentity,
roleAccessPolicy.getResource(),
roleAccessPolicy.getAction());
// add the user to the policy if it doesn't exist
addUserToPolicy(user.getIdentifier(), policy);
}
}
}
// convert any access controls on ports to the appropriate policies
for (PortDTO portDTO : ports) {
final Resource resource;
if (portDTO.getType() != null && portDTO.getType().equals("inputPort")) {
resource = ResourceFactory.getDataTransferResource(ResourceFactory.getComponentResource(ResourceType.InputPort, portDTO.getId(), portDTO.getName()));
} else {
resource = ResourceFactory.getDataTransferResource(ResourceFactory.getComponentResource(ResourceType.OutputPort, portDTO.getId(), portDTO.getName()));
}
if (portDTO.getUserAccessControl() != null) {
for (String userAccessControl : portDTO.getUserAccessControl()) {
// need to perform the identity mapping on the access control so it matches the identities in the User objects
final String mappedUserAccessControl = IdentityMappingUtil.mapIdentity(userAccessControl, identityMappings);
final User foundUser = userGroupProvider.getUserByIdentity(mappedUserAccessControl);
// couldn't find the user matching the access control so log a warning and skip
if (foundUser == null) {
logger.warn("Found port with user access control for {} but no user exists with this identity, skipping...",
new Object[] {mappedUserAccessControl});
continue;
}
// we found the user so create the appropriate policy and add the user to it
Policy policy = getOrCreatePolicy(
allPolicies,
seedIdentity,
resource.getIdentifier(),
WRITE_CODE);
addUserToPolicy(foundUser.getIdentifier(), policy);
}
}
if (portDTO.getGroupAccessControl() != null) {
for (String groupAccessControl : portDTO.getGroupAccessControl()) {
final String legacyGroupName = IdentityMappingUtil.mapIdentity(groupAccessControl, groupMappings);
// find a group where the name is the groupAccessControl
Group foundGroup = null;
for (Group group : userGroupProvider.getGroups()) {
if (group.getName().equals(legacyGroupName)) {
foundGroup = group;
break;
}
}
// couldn't find the group matching the access control so log a warning and skip
if (foundGroup == null) {
logger.warn("Found port with group access control for {} but no group exists with this name, skipping...",
new Object[] {legacyGroupName});
continue;
}
// we found the group so create the appropriate policy and add all the users to it
Policy policy = getOrCreatePolicy(
allPolicies,
seedIdentity,
resource.getIdentifier(),
WRITE_CODE);
addGroupToPolicy(IdentifierUtil.getIdentifier(legacyGroupName), policy);
}
}
}
authorizations.getPolicies().getPolicy().addAll(allPolicies);
}
/**
* Creates and adds an access policy for the given resource, identity, and actions to the specified authorizations.
*

8
nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-file-authorizer/src/main/java/org/apache/nifi/authorization/FileAuthorizer.java
View File

@ -39,8 +39,6 @@ public class FileAuthorizer extends AbstractPolicyBasedAuthorizer {
private static final String FILE_USER_GROUP_PROVIDER_ID = "file-user-group-provider";
private static final String FILE_ACCESS_POLICY_PROVIDER_ID = "file-access-policy-provider";
static final String PROP_LEGACY_AUTHORIZED_USERS_FILE = "Legacy Authorized Users File";
private FileUserGroupProvider userGroupProvider = new FileUserGroupProvider();
private FileAccessPolicyProvider accessPolicyProvider = new FileAccessPolicyProvider();
@ -93,9 +91,6 @@ public class FileAuthorizer extends AbstractPolicyBasedAuthorizer {
if (configurationProperties.containsKey(FileUserGroupProvider.PROP_TENANTS_FILE)) {
userGroupProperties.put(FileUserGroupProvider.PROP_TENANTS_FILE, configurationProperties.get(FileUserGroupProvider.PROP_TENANTS_FILE));
}
if (configurationProperties.containsKey(FileAuthorizer.PROP_LEGACY_AUTHORIZED_USERS_FILE)) {
userGroupProperties.put(FileAuthorizer.PROP_LEGACY_AUTHORIZED_USERS_FILE, configurationProperties.get(FileAuthorizer.PROP_LEGACY_AUTHORIZED_USERS_FILE));
}
// relay the relevant config
final Map<String, String> accessPolicyProperties = new HashMap<>();
@ -106,9 +101,6 @@ public class FileAuthorizer extends AbstractPolicyBasedAuthorizer {
if (configurationProperties.containsKey(FileAccessPolicyProvider.PROP_INITIAL_ADMIN_IDENTITY)) {
accessPolicyProperties.put(FileAccessPolicyProvider.PROP_INITIAL_ADMIN_IDENTITY, configurationProperties.get(FileAccessPolicyProvider.PROP_INITIAL_ADMIN_IDENTITY));
}
if (configurationProperties.containsKey(FileAuthorizer.PROP_LEGACY_AUTHORIZED_USERS_FILE)) {
accessPolicyProperties.put(FileAuthorizer.PROP_LEGACY_AUTHORIZED_USERS_FILE, configurationProperties.get(FileAuthorizer.PROP_LEGACY_AUTHORIZED_USERS_FILE));
}
// ensure all node identities are seeded into the user provider
configurationProperties.forEach((property, value) -> {

60
nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-file-authorizer/src/main/java/org/apache/nifi/authorization/FileUserGroupProvider.java
View File

@ -116,7 +116,6 @@ public class FileUserGroupProvider implements ConfigurableUserGroupProvider {
private NiFiProperties properties;
private File tenantsFile;
private File restoreTenantsFile;
private String legacyAuthorizedUsersFile;
private Set<String> initialUserIdentities;
private List<IdentityMapping> identityMappings;
private List<IdentityMapping> groupMappings;
@ -178,10 +177,6 @@ public class FileUserGroupProvider implements ConfigurableUserGroupProvider {
identityMappings = Collections.unmodifiableList(IdentityMappingUtil.getIdentityMappings(properties));
groupMappings = Collections.unmodifiableList(IdentityMappingUtil.getGroupMappings(properties));
// get the value of the legacy authorized users file
final PropertyValue legacyAuthorizedUsersProp = configurationContext.getProperty(FileAuthorizer.PROP_LEGACY_AUTHORIZED_USERS_FILE);
legacyAuthorizedUsersFile = legacyAuthorizedUsersProp.isSet() ? legacyAuthorizedUsersProp.getValue() : null;
// extract any node identities
initialUserIdentities = new HashSet<>();
for (Map.Entry<String,String> entry : configurationContext.getProperties().entrySet()) {
@ -692,14 +687,8 @@ public class FileUserGroupProvider implements ConfigurableUserGroupProvider {
final UserGroupHolder userGroupHolder = new UserGroupHolder(tenants);
final boolean emptyTenants = userGroupHolder.getAllUsers().isEmpty() && userGroupHolder.getAllGroups().isEmpty();
final boolean hasLegacyAuthorizedUsers = (legacyAuthorizedUsersFile != null && !StringUtils.isBlank(legacyAuthorizedUsersFile));
if (emptyTenants) {
if (hasLegacyAuthorizedUsers) {
logger.info("Loading users from legacy model " + legacyAuthorizedUsersFile + " into new users file.");
convertLegacyAuthorizedUsers(tenants);
}
populateInitialUsers(tenants);
// save any changes that were made and repopulate the holder
@ -740,55 +729,6 @@ public class FileUserGroupProvider implements ConfigurableUserGroupProvider {
}
}
/**
* Unmarshalls an existing authorized-users.xml and converts the object model to the new model.
*
* @param tenants the current Tenants instance users and groups will be added to
* @throws AuthorizerCreationException if the legacy authorized users file that was provided does not exist
* @throws JAXBException if the legacy authorized users file that was provided could not be unmarshalled
*/
private void convertLegacyAuthorizedUsers(final Tenants tenants) throws AuthorizerCreationException, JAXBException {
final File authorizedUsersFile = new File(legacyAuthorizedUsersFile);
if (!authorizedUsersFile.exists()) {
throw new AuthorizerCreationException("Legacy Authorized Users File '" + legacyAuthorizedUsersFile + "' does not exists");
}
final XMLStreamReaderProvider provider = new StandardXMLStreamReaderProvider();
final XMLStreamReader xsr;
try {
xsr = provider.getStreamReader(new StreamSource(authorizedUsersFile));
} catch (final ProcessingException e) {
throw new AuthorizerCreationException("Error converting the legacy authorizers file", e);
}
final Unmarshaller unmarshaller = JAXB_USERS_CONTEXT.createUnmarshaller();
unmarshaller.setSchema(usersSchema);
final JAXBElement<org.apache.nifi.user.generated.Users> element = unmarshaller.unmarshal(
xsr, org.apache.nifi.user.generated.Users.class);
final org.apache.nifi.user.generated.Users users = element.getValue();
if (users.getUser().isEmpty()) {
logger.info("Legacy Authorized Users File contained no users, nothing to convert");
return;
}
for (org.apache.nifi.user.generated.User legacyUser : users.getUser()) {
// create the identifier of the new user based on the DN
final String legacyUserDn = IdentityMappingUtil.mapIdentity(legacyUser.getDn(), identityMappings);
org.apache.nifi.authorization.file.tenants.generated.User user = getOrCreateUser(tenants, legacyUserDn);
// if there was a group name find or create the group and add the user to it
if (StringUtils.isNotBlank(legacyUser.getGroup())) {
final String legacyGroupName = IdentityMappingUtil.mapIdentity(legacyUser.getGroup(), groupMappings);
org.apache.nifi.authorization.file.tenants.generated.Group group = getOrCreateGroup(tenants, legacyGroupName);
org.apache.nifi.authorization.file.tenants.generated.Group.User groupUser = new org.apache.nifi.authorization.file.tenants.generated.Group.User();
groupUser.setIdentifier(user.getIdentifier());
group.getUser().add(groupUser);
}
}
}
/**
* Finds the User with the given identity, or creates a new one and adds it to the Tenants.
*

283
nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-file-authorizer/src/test/java/org/apache/nifi/authorization/FileAccessPolicyProviderTest.java
View File

@ -16,12 +16,11 @@
*/
package org.apache.nifi.authorization;
import org.apache.nifi.parameter.ParameterLookup;
import org.apache.nifi.attribute.expression.language.StandardPropertyValue;
import org.apache.nifi.authorization.exception.AuthorizerCreationException;
import org.apache.nifi.authorization.resource.ResourceFactory;
import org.apache.nifi.authorization.resource.ResourceType;
import org.apache.nifi.components.PropertyValue;
import org.apache.nifi.parameter.ParameterLookup;
import org.apache.nifi.util.NiFiProperties;
import org.apache.nifi.util.file.FileUtils;
import org.junit.jupiter.api.AfterEach;
@ -36,7 +35,6 @@ import java.io.FileOutputStream;
import java.io.IOException;
import java.nio.charset.StandardCharsets;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Map;
import java.util.Properties;
import java.util.Set;
@ -146,7 +144,6 @@ public class FileAccessPolicyProviderTest {
private File restoreAuthorizations;
private File restoreTenants;
private File flow;
private File flowJson;
private File flowNoPorts;
private File flowWithDns;
@ -170,21 +167,18 @@ public class FileAccessPolicyProviderTest {
restoreTenants = new File("target/restore/users.xml");
FileUtils.ensureDirectoryExistAndCanAccess(restoreTenants.getParentFile());
flow = new File("src/test/resources/flow.xml.gz");
flow = new File("src/test/resources/flow.json.gz");
FileUtils.ensureDirectoryExistAndCanAccess(flow.getParentFile());
flowJson = new File("src/test/resources/flow.json.gz");
flowNoPorts = new File("src/test/resources/flow-no-ports.xml.gz");
flowNoPorts = new File("src/test/resources/flow-no-ports.json.gz");
FileUtils.ensureDirectoryExistAndCanAccess(flowNoPorts.getParentFile());
flowWithDns = new File("src/test/resources/flow-with-dns.xml.gz");
flowWithDns = new File("src/test/resources/flow-with-dns.json.gz");
FileUtils.ensureDirectoryExistAndCanAccess(flowWithDns.getParentFile());
properties = mock(NiFiProperties.class);
when(properties.getRestoreDirectory()).thenReturn(restoreAuthorizations.getParentFile());
when(properties.getFlowConfigurationFile()).thenReturn(flow);
when(properties.getFlowConfigurationJsonFile()).thenReturn(flowJson);
userGroupProvider = new FileUserGroupProvider();
userGroupProvider.setNiFiProperties(properties);
@ -196,7 +190,6 @@ public class FileAccessPolicyProviderTest {
ParameterLookup.EMPTY));
when(configurationContext.getProperty(eq(FileUserGroupProvider.PROP_TENANTS_FILE))).thenReturn(new StandardPropertyValue(primaryTenants.getPath(), null, ParameterLookup.EMPTY));
when(configurationContext.getProperty(eq(FileAccessPolicyProvider.PROP_INITIAL_ADMIN_IDENTITY))).thenReturn(new StandardPropertyValue(null, null, ParameterLookup.EMPTY));
when(configurationContext.getProperty(eq(FileAuthorizer.PROP_LEGACY_AUTHORIZED_USERS_FILE))).thenReturn(new StandardPropertyValue(null, null, ParameterLookup.EMPTY));
when(configurationContext.getProperty(eq(FileAccessPolicyProvider.PROP_USER_GROUP_PROVIDER))).thenReturn(new StandardPropertyValue("user-group-provider", null,
ParameterLookup.EMPTY));
when(configurationContext.getProperties()).then((invocation) -> {
@ -212,11 +205,6 @@ public class FileAccessPolicyProviderTest {
properties.put(FileUserGroupProvider.PROP_TENANTS_FILE, tenantFile.getValue());
}
final PropertyValue legacyAuthFile = configurationContext.getProperty(FileAuthorizer.PROP_LEGACY_AUTHORIZED_USERS_FILE);
if (legacyAuthFile != null) {
properties.put(FileAuthorizer.PROP_LEGACY_AUTHORIZED_USERS_FILE, legacyAuthFile.getValue());
}
final PropertyValue initialAdmin = configurationContext.getProperty(FileAccessPolicyProvider.PROP_INITIAL_ADMIN_IDENTITY);
if (initialAdmin != null) {
properties.put(FileAccessPolicyProvider.PROP_INITIAL_ADMIN_IDENTITY, initialAdmin.getValue());
@ -280,262 +268,6 @@ public class FileAccessPolicyProviderTest {
deleteFile(restoreTenants);
}
@Test
public void testOnConfiguredWhenLegacyUsersFileProvidedWithOverlappingRoles() throws Exception {
when(configurationContext.getProperty(eq(FileAuthorizer.PROP_LEGACY_AUTHORIZED_USERS_FILE)))
.thenReturn(new StandardPropertyValue("src/test/resources/authorized-users-multirole.xml", null, ParameterLookup.EMPTY));
writeFile(primaryAuthorizations, EMPTY_AUTHORIZATIONS_CONCISE);
writeFile(primaryTenants, EMPTY_TENANTS_CONCISE);
userGroupProvider.onConfigured(configurationContext);
accessPolicyProvider.onConfigured(configurationContext);
assertNotNull(accessPolicyProvider.getAccessPolicy(ResourceType.Flow.getValue(), RequestAction.READ));
assertNotNull(accessPolicyProvider.getAccessPolicy(ResourceType.Controller.getValue(), RequestAction.READ));
assertNotNull(accessPolicyProvider.getAccessPolicy(ResourceType.Controller.getValue(), RequestAction.WRITE));
assertNotNull(accessPolicyProvider.getAccessPolicy(ResourceType.System.getValue(), RequestAction.READ));
assertNotNull(accessPolicyProvider.getAccessPolicy(ResourceType.ProcessGroup.getValue() + "/" + ROOT_GROUP_ID, RequestAction.READ));
assertNotNull(accessPolicyProvider.getAccessPolicy(ResourceType.ProcessGroup.getValue() + "/" + ROOT_GROUP_ID, RequestAction.WRITE));
}
@Test
public void testOnConfiguredWhenLegacyUsersFileProvidedAndFlowHasNoPorts() throws Exception {
properties = mock(NiFiProperties.class);
when(properties.getRestoreDirectory()).thenReturn(restoreAuthorizations.getParentFile());
when(properties.getFlowConfigurationFile()).thenReturn(flowNoPorts);
when(configurationContext.getProperty(eq(FileAuthorizer.PROP_LEGACY_AUTHORIZED_USERS_FILE)))
.thenReturn(new StandardPropertyValue("src/test/resources/authorized-users.xml", null, ParameterLookup.EMPTY));
writeFile(primaryAuthorizations, EMPTY_AUTHORIZATIONS_CONCISE);
writeFile(primaryTenants, EMPTY_TENANTS_CONCISE);
userGroupProvider.onConfigured(configurationContext);
accessPolicyProvider.onConfigured(configurationContext);
boolean foundDataTransferPolicy = false;
for (AccessPolicy policy : accessPolicyProvider.getAccessPolicies()) {
if (policy.getResource().contains(ResourceType.DataTransfer.name())) {
foundDataTransferPolicy = true;
break;
}
}
assertFalse(foundDataTransferPolicy);
}
@Test
public void testOnConfiguredWhenLegacyUsersFileProvided() throws Exception {
when(configurationContext.getProperty(eq(FileAuthorizer.PROP_LEGACY_AUTHORIZED_USERS_FILE)))
.thenReturn(new StandardPropertyValue("src/test/resources/authorized-users.xml", null, ParameterLookup.EMPTY));
writeFile(primaryAuthorizations, EMPTY_AUTHORIZATIONS_CONCISE);
writeFile(primaryTenants, EMPTY_TENANTS_CONCISE);
userGroupProvider.onConfigured(configurationContext);
accessPolicyProvider.onConfigured(configurationContext);
final User user1 = userGroupProvider.getUserByIdentity("user1");
final User user2 = userGroupProvider.getUserByIdentity("user2");
final User user3 = userGroupProvider.getUserByIdentity("user3");
final User user4 = userGroupProvider.getUserByIdentity("user4");
final User user5 = userGroupProvider.getUserByIdentity("user5");
final User user6 = userGroupProvider.getUserByIdentity("user6");
// verify one group got created
final Set<Group> groups = userGroupProvider.getGroups();
final Group group1 = groups.iterator().next();
// verify more than one policy got created
final Set<AccessPolicy> policies = accessPolicyProvider.getAccessPolicies();
assertTrue(policies.size() > 0);
// verify user1's policies
final Map<String,Set<RequestAction>> user1Policies = getResourceActions(policies, user1);
assertEquals(4, user1Policies.size());
assertTrue(user1Policies.containsKey(ResourceType.Flow.getValue()));
assertEquals(1, user1Policies.get(ResourceType.Flow.getValue()).size());
assertTrue(user1Policies.get(ResourceType.Flow.getValue()).contains(RequestAction.READ));
assertTrue(user1Policies.containsKey(ResourceType.ProcessGroup.getValue() + "/" + ROOT_GROUP_ID));
assertEquals(1, user1Policies.get(ResourceType.ProcessGroup.getValue() + "/" + ROOT_GROUP_ID).size());
assertTrue(user1Policies.get(ResourceType.ProcessGroup.getValue() + "/" + ROOT_GROUP_ID).contains(RequestAction.READ));
// verify user2's policies
final Map<String,Set<RequestAction>> user2Policies = getResourceActions(policies, user2);
assertEquals(3, user2Policies.size());
assertTrue(user2Policies.containsKey(ResourceType.Provenance.getValue()));
assertEquals(1, user2Policies.get(ResourceType.Provenance.getValue()).size());
assertTrue(user2Policies.get(ResourceType.Provenance.getValue()).contains(RequestAction.READ));
assertTrue(user2Policies.containsKey(ResourceType.ProvenanceData.getValue() + "/process-groups/" + ROOT_GROUP_ID));
assertEquals(1, user2Policies.get(ResourceType.ProvenanceData.getValue() + "/process-groups/" + ROOT_GROUP_ID).size());
assertTrue(user2Policies.get(ResourceType.ProvenanceData.getValue() + "/process-groups/" + ROOT_GROUP_ID).contains(RequestAction.READ));
assertTrue(user2Policies.containsKey(ResourceType.Data.getValue() + "/process-groups/" + ROOT_GROUP_ID));
assertEquals(1, user2Policies.get(ResourceType.Data.getValue() + "/process-groups/" + ROOT_GROUP_ID).size());
assertTrue(user2Policies.get(ResourceType.Data.getValue() + "/process-groups/" + ROOT_GROUP_ID).contains(RequestAction.READ));
// verify user3's policies
final Map<String,Set<RequestAction>> user3Policies = getResourceActions(policies, user3);
assertEquals(6, user3Policies.size());
assertTrue(user3Policies.containsKey(ResourceType.Flow.getValue()));
assertEquals(1, user3Policies.get(ResourceType.Flow.getValue()).size());
assertTrue(user3Policies.get(ResourceType.Flow.getValue()).contains(RequestAction.READ));
assertTrue(user3Policies.containsKey(ResourceType.ProcessGroup.getValue() + "/" + ROOT_GROUP_ID));
assertEquals(2, user3Policies.get(ResourceType.ProcessGroup.getValue() + "/" + ROOT_GROUP_ID).size());
assertTrue(user3Policies.get(ResourceType.ProcessGroup.getValue() + "/" + ROOT_GROUP_ID).contains(RequestAction.WRITE));
// verify user4's policies
final Map<String,Set<RequestAction>> user4Policies = getResourceActions(policies, user4);
assertEquals(6, user4Policies.size());
assertTrue(user4Policies.containsKey(ResourceType.Flow.getValue()));
assertEquals(1, user4Policies.get(ResourceType.Flow.getValue()).size());
assertTrue(user4Policies.get(ResourceType.Flow.getValue()).contains(RequestAction.READ));
assertTrue(user4Policies.containsKey(ResourceType.ProcessGroup.getValue() + "/" + ROOT_GROUP_ID));
assertEquals(1, user4Policies.get(ResourceType.ProcessGroup.getValue() + "/" + ROOT_GROUP_ID).size());
assertTrue(user4Policies.get(ResourceType.ProcessGroup.getValue() + "/" + ROOT_GROUP_ID).contains(RequestAction.READ));
assertTrue(user4Policies.containsKey(ResourceType.Tenant.getValue()));
assertEquals(2, user4Policies.get(ResourceType.Tenant.getValue()).size());
assertTrue(user4Policies.get(ResourceType.Tenant.getValue()).contains(RequestAction.WRITE));
assertTrue(user4Policies.containsKey(ResourceType.Policy.getValue()));
assertEquals(2, user4Policies.get(ResourceType.Policy.getValue()).size());
assertTrue(user4Policies.get(ResourceType.Policy.getValue()).contains(RequestAction.WRITE));
// verify user5's policies
final Map<String,Set<RequestAction>> user5Policies = getResourceActions(policies, user5);
assertEquals(2, user5Policies.size());
assertTrue(user5Policies.containsKey(ResourceType.Proxy.getValue()));
assertEquals(1, user5Policies.get(ResourceType.Proxy.getValue()).size());
assertTrue(user5Policies.get(ResourceType.Proxy.getValue()).contains(RequestAction.WRITE));
// verify user6's policies
final Map<String,Set<RequestAction>> user6Policies = getResourceActions(policies, user6);
assertEquals(3, user6Policies.size());
assertTrue(user6Policies.containsKey(ResourceType.SiteToSite.getValue()));
assertEquals(1, user6Policies.get(ResourceType.SiteToSite.getValue()).size());
assertTrue(user6Policies.get(ResourceType.SiteToSite.getValue()).contains(RequestAction.READ));
final Resource inputPortResource = ResourceFactory.getDataTransferResource(
ResourceFactory.getComponentResource(ResourceType.InputPort, "2f7d1606-b090-4be7-a592-a5b70fb55531", "TCP Input"));
final AccessPolicy inputPortPolicy = accessPolicyProvider.getAccessPolicy(inputPortResource.getIdentifier(), RequestAction.WRITE);
assertNotNull(inputPortPolicy);
assertEquals(1, inputPortPolicy.getUsers().size());
assertTrue(inputPortPolicy.getUsers().contains(user6.getIdentifier()));
assertEquals(1, inputPortPolicy.getGroups().size());
assertTrue(inputPortPolicy.getGroups().contains(group1.getIdentifier()));
final Resource outputPortResource = ResourceFactory.getDataTransferResource(
ResourceFactory.getComponentResource(ResourceType.OutputPort, "2f7d1606-b090-4be7-a592-a5b70fb55532", "TCP Output"));
final AccessPolicy outputPortPolicy = accessPolicyProvider.getAccessPolicy(outputPortResource.getIdentifier(), RequestAction.WRITE);
assertNotNull(outputPortPolicy);
assertEquals(1, outputPortPolicy.getUsers().size());
assertTrue(outputPortPolicy.getUsers().contains(user4.getIdentifier()));
}
private Map<String,Set<RequestAction>> getResourceActions(final Set<AccessPolicy> policies, final User user) {
Map<String,Set<RequestAction>> resourceActionMap = new HashMap<>();
for (AccessPolicy accessPolicy : policies) {
if (accessPolicy.getUsers().contains(user.getIdentifier())) {
Set<RequestAction> actions = resourceActionMap.get(accessPolicy.getResource());
if (actions == null) {
actions = new HashSet<>();
resourceActionMap.put(accessPolicy.getResource(), actions);
}
actions.add(accessPolicy.getAction());
}
}
return resourceActionMap;
}
@Test
public void testOnConfiguredWhenLegacyUsersFileProvidedWithIdentityMappings() throws Exception {
final Properties props = new Properties();
props.setProperty("nifi.security.identity.mapping.pattern.dn1", "^CN=(.*?), OU=(.*?), O=(.*?), L=(.*?), ST=(.*?), C=(.*?)$");
props.setProperty("nifi.security.identity.mapping.value.dn1", "$1");
properties = getNiFiProperties(props);
when(properties.getRestoreDirectory()).thenReturn(restoreAuthorizations.getParentFile());
when(properties.getFlowConfigurationFile()).thenReturn(flowWithDns);
when(properties.getFlowConfigurationJsonFile()).thenReturn(flowJson);
userGroupProvider.setNiFiProperties(properties);
accessPolicyProvider.setNiFiProperties(properties);
when(configurationContext.getProperty(eq(FileAuthorizer.PROP_LEGACY_AUTHORIZED_USERS_FILE)))
.thenReturn(new StandardPropertyValue("src/test/resources/authorized-users-with-dns.xml", null, ParameterLookup.EMPTY));
writeFile(primaryAuthorizations, EMPTY_AUTHORIZATIONS_CONCISE);
writeFile(primaryTenants, EMPTY_TENANTS_CONCISE);
userGroupProvider.onConfigured(configurationContext);
accessPolicyProvider.onConfigured(configurationContext);
final User user4 = userGroupProvider.getUserByIdentity("user4");
final User user6 = userGroupProvider.getUserByIdentity("user6");
// verify one group got created
final Set<Group> groups = userGroupProvider.getGroups();
final Group group1 = groups.iterator().next();
final Resource inputPortResource = ResourceFactory.getDataTransferResource(
ResourceFactory.getComponentResource(ResourceType.InputPort, "2f7d1606-b090-4be7-a592-a5b70fb55531", "TCP Input"));
final AccessPolicy inputPortPolicy = accessPolicyProvider.getAccessPolicy(inputPortResource.getIdentifier(), RequestAction.WRITE);
assertNotNull(inputPortPolicy);
assertEquals(1, inputPortPolicy.getUsers().size());
assertTrue(inputPortPolicy.getUsers().contains(user6.getIdentifier()));
assertEquals(1, inputPortPolicy.getGroups().size());
assertTrue(inputPortPolicy.getGroups().contains(group1.getIdentifier()));
final Resource outputPortResource = ResourceFactory.getDataTransferResource(
ResourceFactory.getComponentResource(ResourceType.OutputPort, "2f7d1606-b090-4be7-a592-a5b70fb55532", "TCP Output"));
final AccessPolicy outputPortPolicy = accessPolicyProvider.getAccessPolicy(outputPortResource.getIdentifier(), RequestAction.WRITE);
assertNotNull(outputPortPolicy);
assertEquals(1, outputPortPolicy.getUsers().size());
assertTrue(outputPortPolicy.getUsers().contains(user4.getIdentifier()));
}
@Test
public void testOnConfiguredWhenBadLegacyUsersFileProvided() throws Exception {
when(configurationContext.getProperty(eq(FileAuthorizer.PROP_LEGACY_AUTHORIZED_USERS_FILE)))
.thenReturn(new StandardPropertyValue("src/test/resources/does-not-exist.xml", null, ParameterLookup.EMPTY));
writeFile(primaryAuthorizations, EMPTY_AUTHORIZATIONS_CONCISE);
writeFile(primaryTenants, EMPTY_TENANTS_CONCISE);
assertThrows(AuthorizerCreationException.class,
() -> accessPolicyProvider.onConfigured(configurationContext));
}
@Test
public void testOnConfiguredWhenInitialAdminAndLegacyUsersProvided() throws Exception {
final String adminIdentity = "admin-user";
when(configurationContext.getProperty(eq(FileAccessPolicyProvider.PROP_INITIAL_ADMIN_IDENTITY)))
.thenReturn(new StandardPropertyValue(adminIdentity, null, ParameterLookup.EMPTY));
when(configurationContext.getProperty(eq(FileAuthorizer.PROP_LEGACY_AUTHORIZED_USERS_FILE)))
.thenReturn(new StandardPropertyValue("src/test/resources/authorized-users.xml", null, ParameterLookup.EMPTY));
writeFile(primaryAuthorizations, EMPTY_AUTHORIZATIONS_CONCISE);
writeFile(primaryTenants, EMPTY_TENANTS_CONCISE);
assertThrows(AuthorizerCreationException.class,
() -> accessPolicyProvider.onConfigured(configurationContext));
}
@Test
public void testOnConfiguredWhenInitialAdminNotProvided() throws Exception {
writeFile(primaryAuthorizations, EMPTY_AUTHORIZATIONS_CONCISE);
@ -586,8 +318,7 @@ public class FileAccessPolicyProviderTest {
// setup NiFi properties to return a file that does not exist
properties = mock(NiFiProperties.class);
when(properties.getRestoreDirectory()).thenReturn(restoreAuthorizations.getParentFile());
when(properties.getFlowConfigurationFile()).thenReturn(new File("src/test/resources/does-not-exist.xml.gz"));
when(properties.getFlowConfigurationJsonFile()).thenReturn(flowJson);
when(properties.getFlowConfigurationFile()).thenReturn(new File("src/test/resources/does-not-exist.json.gz"));
userGroupProvider.setNiFiProperties(properties);
accessPolicyProvider.setNiFiProperties(properties);
@ -627,8 +358,7 @@ public class FileAccessPolicyProviderTest {
// setup NiFi properties to return a file that does not exist
properties = mock(NiFiProperties.class);
when(properties.getRestoreDirectory()).thenReturn(restoreAuthorizations.getParentFile());
when(properties.getFlowConfigurationFile()).thenReturn(null);
when(properties.getFlowConfigurationJsonFile()).thenReturn(flowJson);
when(properties.getFlowConfigurationFile()).thenReturn(new File("src/test/resources/does-not-exist.json.gz"));
userGroupProvider.setNiFiProperties(properties);
accessPolicyProvider.setNiFiProperties(properties);
@ -672,7 +402,6 @@ public class FileAccessPolicyProviderTest {
properties = getNiFiProperties(props);
when(properties.getRestoreDirectory()).thenReturn(restoreAuthorizations.getParentFile());
when(properties.getFlowConfigurationFile()).thenReturn(flow);
when(properties.getFlowConfigurationJsonFile()).thenReturn(flowJson);
userGroupProvider.setNiFiProperties(properties);
accessPolicyProvider.setNiFiProperties(properties);

316
nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-file-authorizer/src/test/java/org/apache/nifi/authorization/FileAuthorizerTest.java
View File

@ -17,19 +17,19 @@
package org.apache.nifi.authorization;
import org.apache.commons.lang3.SystemUtils;
import org.apache.nifi.parameter.ParameterLookup;
import org.apache.nifi.attribute.expression.language.StandardPropertyValue;
import org.apache.nifi.authorization.AuthorizationResult.Result;
import org.apache.nifi.authorization.exception.AuthorizerCreationException;
import org.apache.nifi.authorization.resource.ResourceFactory;
import org.apache.nifi.authorization.resource.ResourceType;
import org.apache.nifi.components.PropertyValue;
import org.apache.nifi.parameter.ParameterLookup;
import org.apache.nifi.util.NiFiProperties;
import org.apache.nifi.util.file.FileUtils;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.Assumptions;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.BeforeAll;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.mockito.Mockito;
import org.mockito.invocation.InvocationOnMock;
@ -40,7 +40,6 @@ import java.io.FileOutputStream;
import java.io.IOException;
import java.nio.charset.StandardCharsets;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Map;
import java.util.Properties;
import java.util.Set;
@ -157,7 +156,6 @@ public class FileAuthorizerTest {
private File restoreAuthorizations;
private File restoreTenants;
private File flow;
private File flowJson;
private File flowNoPorts;
private File flowWithDns;
@ -186,21 +184,18 @@ public class FileAuthorizerTest {
restoreTenants = new File("target/restore/users.xml");
FileUtils.ensureDirectoryExistAndCanAccess(restoreTenants.getParentFile());
flow = new File("src/test/resources/flow.xml.gz");
flow = new File("src/test/resources/flow.json.gz");
FileUtils.ensureDirectoryExistAndCanAccess(flow.getParentFile());
flowJson = new File("src/test/resources/flow.json.gz");
flowNoPorts = new File("src/test/resources/flow-no-ports.xml.gz");
flowNoPorts = new File("src/test/resources/flow-no-ports.json.gz");
FileUtils.ensureDirectoryExistAndCanAccess(flowNoPorts.getParentFile());
flowWithDns = new File("src/test/resources/flow-with-dns.xml.gz");
flowWithDns = new File("src/test/resources/flow-with-dns.json.gz");
FileUtils.ensureDirectoryExistAndCanAccess(flowWithDns.getParentFile());
properties = mock(NiFiProperties.class);
when(properties.getRestoreDirectory()).thenReturn(restoreAuthorizations.getParentFile());
when(properties.getFlowConfigurationFile()).thenReturn(flow);
when(properties.getFlowConfigurationJsonFile()).thenReturn(flowJson);
configurationContext = mock(AuthorizerConfigurationContext.class);
when(configurationContext.getProperty(Mockito.eq(FileAccessPolicyProvider.PROP_AUTHORIZATIONS_FILE))).thenReturn(new StandardPropertyValue(primaryAuthorizations.getPath(), null,
@ -219,11 +214,6 @@ public class FileAuthorizerTest {
properties.put(FileUserGroupProvider.PROP_TENANTS_FILE, tenantFile.getValue());
}
final PropertyValue legacyAuthFile = configurationContext.getProperty(FileAuthorizer.PROP_LEGACY_AUTHORIZED_USERS_FILE);
if (legacyAuthFile != null) {
properties.put(FileAuthorizer.PROP_LEGACY_AUTHORIZED_USERS_FILE, legacyAuthFile.getValue());
}
final PropertyValue initialAdmin = configurationContext.getProperty(FileAccessPolicyProvider.PROP_INITIAL_ADMIN_IDENTITY);
if (initialAdmin != null) {
properties.put(FileAccessPolicyProvider.PROP_INITIAL_ADMIN_IDENTITY, initialAdmin.getValue());
@ -256,294 +246,6 @@ public class FileAuthorizerTest {
deleteFile(restoreTenants);
}
@Test
public void testOnConfiguredWhenLegacyUsersFileProvidedWithOverlappingRoles() throws Exception {
when(configurationContext.getProperty(Mockito.eq(FileAuthorizer.PROP_LEGACY_AUTHORIZED_USERS_FILE)))
.thenReturn(new StandardPropertyValue("src/test/resources/authorized-users-multirole.xml", null, ParameterLookup.EMPTY));
writeFile(primaryAuthorizations, EMPTY_AUTHORIZATIONS_CONCISE);
writeFile(primaryTenants, EMPTY_TENANTS_CONCISE);
authorizer.onConfigured(configurationContext);
final Set<User> users = authorizer.getUsers();
assertEquals(1, users.size());
UsersAndAccessPolicies usersAndAccessPolicies = authorizer.getUsersAndAccessPolicies();
assertNotNull(usersAndAccessPolicies.getAccessPolicy(ResourceType.Flow.getValue(), RequestAction.READ));
assertNotNull(usersAndAccessPolicies.getAccessPolicy(ResourceType.Controller.getValue(), RequestAction.READ));
assertNotNull(usersAndAccessPolicies.getAccessPolicy(ResourceType.Controller.getValue(), RequestAction.WRITE));
assertNotNull(usersAndAccessPolicies.getAccessPolicy(ResourceType.System.getValue(), RequestAction.READ));
assertNotNull(usersAndAccessPolicies.getAccessPolicy(ResourceType.ProcessGroup.getValue() + "/" + ROOT_GROUP_ID, RequestAction.READ));
assertNotNull(usersAndAccessPolicies.getAccessPolicy(ResourceType.ProcessGroup.getValue() + "/" + ROOT_GROUP_ID, RequestAction.WRITE));
}
@Test
public void testOnConfiguredWhenLegacyUsersFileProvidedAndFlowHasNoPorts() throws Exception {
properties = mock(NiFiProperties.class);
when(properties.getRestoreDirectory()).thenReturn(restoreAuthorizations.getParentFile());
when(properties.getFlowConfigurationFile()).thenReturn(flowNoPorts);
when(configurationContext.getProperty(Mockito.eq(FileAuthorizer.PROP_LEGACY_AUTHORIZED_USERS_FILE)))
.thenReturn(new StandardPropertyValue("src/test/resources/authorized-users.xml", null, ParameterLookup.EMPTY));
writeFile(primaryAuthorizations, EMPTY_AUTHORIZATIONS_CONCISE);
writeFile(primaryTenants, EMPTY_TENANTS_CONCISE);
authorizer.onConfigured(configurationContext);
boolean foundDataTransferPolicy = false;
for (AccessPolicy policy : authorizer.getAccessPolicies()) {
if (policy.getResource().contains(ResourceType.DataTransfer.name())) {
foundDataTransferPolicy = true;
break;
}
}
assertFalse(foundDataTransferPolicy);
}
@Test
public void testOnConfiguredWhenLegacyUsersFileProvided() throws Exception {
when(configurationContext.getProperty(Mockito.eq(FileAuthorizer.PROP_LEGACY_AUTHORIZED_USERS_FILE)))
.thenReturn(new StandardPropertyValue("src/test/resources/authorized-users.xml", null, ParameterLookup.EMPTY));
writeFile(primaryAuthorizations, EMPTY_AUTHORIZATIONS_CONCISE);
writeFile(primaryTenants, EMPTY_TENANTS_CONCISE);
authorizer.onConfigured(configurationContext);
// verify all users got created correctly
final Set<User> users = authorizer.getUsers();
assertEquals(6, users.size());
final User user1 = authorizer.getUserByIdentity("user1");
assertNotNull(user1);
final User user2 = authorizer.getUserByIdentity("user2");
assertNotNull(user2);
final User user3 = authorizer.getUserByIdentity("user3");
assertNotNull(user3);
final User user4 = authorizer.getUserByIdentity("user4");
assertNotNull(user4);
final User user5 = authorizer.getUserByIdentity("user5");
assertNotNull(user5);
final User user6 = authorizer.getUserByIdentity("user6");
assertNotNull(user6);
// verify one group got created
final Set<Group> groups = authorizer.getGroups();
assertEquals(1, groups.size());
final Group group1 = groups.iterator().next();
assertEquals("group1", group1.getName());
// verify more than one policy got created
final Set<AccessPolicy> policies = authorizer.getAccessPolicies();
assertTrue(policies.size() > 0);
// verify user1's policies
final Map<String,Set<RequestAction>> user1Policies = getResourceActions(policies, user1);
assertEquals(4, user1Policies.size());
assertTrue(user1Policies.containsKey(ResourceType.Flow.getValue()));
assertEquals(1, user1Policies.get(ResourceType.Flow.getValue()).size());
assertTrue(user1Policies.get(ResourceType.Flow.getValue()).contains(RequestAction.READ));
assertTrue(user1Policies.containsKey(ResourceType.ProcessGroup.getValue() + "/" + ROOT_GROUP_ID));
assertEquals(1, user1Policies.get(ResourceType.ProcessGroup.getValue() + "/" + ROOT_GROUP_ID).size());
assertTrue(user1Policies.get(ResourceType.ProcessGroup.getValue() + "/" + ROOT_GROUP_ID).contains(RequestAction.READ));
// verify user2's policies
final Map<String,Set<RequestAction>> user2Policies = getResourceActions(policies, user2);
assertEquals(3, user2Policies.size());
assertTrue(user2Policies.containsKey(ResourceType.Provenance.getValue()));
assertEquals(1, user2Policies.get(ResourceType.Provenance.getValue()).size());
assertTrue(user2Policies.get(ResourceType.Provenance.getValue()).contains(RequestAction.READ));
assertTrue(user2Policies.containsKey(ResourceType.ProvenanceData.getValue() + "/process-groups/" + ROOT_GROUP_ID));
assertEquals(1, user2Policies.get(ResourceType.ProvenanceData.getValue() + "/process-groups/" + ROOT_GROUP_ID).size());
assertTrue(user2Policies.get(ResourceType.ProvenanceData.getValue() + "/process-groups/" + ROOT_GROUP_ID).contains(RequestAction.READ));
assertTrue(user2Policies.containsKey(ResourceType.Data.getValue() + "/process-groups/" + ROOT_GROUP_ID));
assertEquals(1, user2Policies.get(ResourceType.Data.getValue() + "/process-groups/" + ROOT_GROUP_ID).size());
assertTrue(user2Policies.get(ResourceType.Data.getValue() + "/process-groups/" + ROOT_GROUP_ID).contains(RequestAction.READ));
// verify user3's policies
final Map<String,Set<RequestAction>> user3Policies = getResourceActions(policies, user3);
assertEquals(6, user3Policies.size());
assertTrue(user3Policies.containsKey(ResourceType.Flow.getValue()));
assertEquals(1, user3Policies.get(ResourceType.Flow.getValue()).size());
assertTrue(user3Policies.get(ResourceType.Flow.getValue()).contains(RequestAction.READ));
assertTrue(user3Policies.containsKey(ResourceType.ProcessGroup.getValue() + "/" + ROOT_GROUP_ID));
assertEquals(2, user3Policies.get(ResourceType.ProcessGroup.getValue() + "/" + ROOT_GROUP_ID).size());
assertTrue(user3Policies.get(ResourceType.ProcessGroup.getValue() + "/" + ROOT_GROUP_ID).contains(RequestAction.WRITE));
// verify user4's policies
final Map<String,Set<RequestAction>> user4Policies = getResourceActions(policies, user4);
assertEquals(6, user4Policies.size());
assertTrue(user4Policies.containsKey(ResourceType.Flow.getValue()));
assertEquals(1, user4Policies.get(ResourceType.Flow.getValue()).size());
assertTrue(user4Policies.get(ResourceType.Flow.getValue()).contains(RequestAction.READ));
assertTrue(user4Policies.containsKey(ResourceType.ProcessGroup.getValue() + "/" + ROOT_GROUP_ID));
assertEquals(1, user4Policies.get(ResourceType.ProcessGroup.getValue() + "/" + ROOT_GROUP_ID).size());
assertTrue(user4Policies.get(ResourceType.ProcessGroup.getValue() + "/" + ROOT_GROUP_ID).contains(RequestAction.READ));
assertTrue(user4Policies.containsKey(ResourceType.Tenant.getValue()));
assertEquals(2, user4Policies.get(ResourceType.Tenant.getValue()).size());
assertTrue(user4Policies.get(ResourceType.Tenant.getValue()).contains(RequestAction.WRITE));
assertTrue(user4Policies.containsKey(ResourceType.Policy.getValue()));
assertEquals(2, user4Policies.get(ResourceType.Policy.getValue()).size());
assertTrue(user4Policies.get(ResourceType.Policy.getValue()).contains(RequestAction.WRITE));
// verify user5's policies
final Map<String,Set<RequestAction>> user5Policies = getResourceActions(policies, user5);
assertEquals(2, user5Policies.size());
assertTrue(user5Policies.containsKey(ResourceType.Proxy.getValue()));
assertEquals(1, user5Policies.get(ResourceType.Proxy.getValue()).size());
assertTrue(user5Policies.get(ResourceType.Proxy.getValue()).contains(RequestAction.WRITE));
// verify user6's policies
final Map<String,Set<RequestAction>> user6Policies = getResourceActions(policies, user6);
assertEquals(3, user6Policies.size());
assertTrue(user6Policies.containsKey(ResourceType.SiteToSite.getValue()));
assertEquals(1, user6Policies.get(ResourceType.SiteToSite.getValue()).size());
assertTrue(user6Policies.get(ResourceType.SiteToSite.getValue()).contains(RequestAction.READ));
final Resource inputPortResource = ResourceFactory.getDataTransferResource(
ResourceFactory.getComponentResource(ResourceType.InputPort, "2f7d1606-b090-4be7-a592-a5b70fb55531", "TCP Input"));
final AccessPolicy inputPortPolicy = authorizer.getUsersAndAccessPolicies().getAccessPolicy(inputPortResource.getIdentifier(), RequestAction.WRITE);
assertNotNull(inputPortPolicy);
assertEquals(1, inputPortPolicy.getUsers().size());
assertTrue(inputPortPolicy.getUsers().contains(user6.getIdentifier()));
assertEquals(1, inputPortPolicy.getGroups().size());
assertTrue(inputPortPolicy.getGroups().contains(group1.getIdentifier()));
final Resource outputPortResource = ResourceFactory.getDataTransferResource(
ResourceFactory.getComponentResource(ResourceType.OutputPort, "2f7d1606-b090-4be7-a592-a5b70fb55532", "TCP Output"));
final AccessPolicy outputPortPolicy = authorizer.getUsersAndAccessPolicies().getAccessPolicy(outputPortResource.getIdentifier(), RequestAction.WRITE);
assertNotNull(outputPortPolicy);
assertEquals(1, outputPortPolicy.getUsers().size());
assertTrue(outputPortPolicy.getUsers().contains(user4.getIdentifier()));
}
private Map<String,Set<RequestAction>> getResourceActions(final Set<AccessPolicy> policies, final User user) {
Map<String,Set<RequestAction>> resourceActionMap = new HashMap<>();
for (AccessPolicy accessPolicy : policies) {
if (accessPolicy.getUsers().contains(user.getIdentifier())) {
Set<RequestAction> actions = resourceActionMap.get(accessPolicy.getResource());
if (actions == null) {
actions = new HashSet<>();
resourceActionMap.put(accessPolicy.getResource(), actions);
}
actions.add(accessPolicy.getAction());
}
}
return resourceActionMap;
}
@Test
public void testOnConfiguredWhenLegacyUsersFileProvidedWithIdentityMappings() throws Exception {
final Properties props = new Properties();
props.setProperty("nifi.security.identity.mapping.pattern.dn1", "^CN=(.*?), OU=(.*?), O=(.*?), L=(.*?), ST=(.*?), C=(.*?)$");
props.setProperty("nifi.security.identity.mapping.value.dn1", "$1");
props.setProperty("nifi.security.group.mapping.pattern.anygroup", "^(.*)$");
props.setProperty("nifi.security.group.mapping.value.anygroup", "$1");
props.setProperty("nifi.security.group.mapping.transform.anygroup", "UPPER");
properties = getNiFiProperties(props);
when(properties.getRestoreDirectory()).thenReturn(restoreAuthorizations.getParentFile());
when(properties.getFlowConfigurationFile()).thenReturn(flowWithDns);
when(properties.getFlowConfigurationJsonFile()).thenReturn(flowJson);
authorizer.setNiFiProperties(properties);
when(configurationContext.getProperty(Mockito.eq(FileAuthorizer.PROP_LEGACY_AUTHORIZED_USERS_FILE)))
.thenReturn(new StandardPropertyValue("src/test/resources/authorized-users-with-dns.xml", null, ParameterLookup.EMPTY));
writeFile(primaryAuthorizations, EMPTY_AUTHORIZATIONS_CONCISE);
writeFile(primaryTenants, EMPTY_TENANTS_CONCISE);
authorizer.onConfigured(configurationContext);
final User user1 = authorizer.getUserByIdentity("user1");
assertNotNull(user1);
final User user2 = authorizer.getUserByIdentity("user2");
assertNotNull(user2);
final User user3 = authorizer.getUserByIdentity("user3");
assertNotNull(user3);
final User user4 = authorizer.getUserByIdentity("user4");
assertNotNull(user4);
final User user5 = authorizer.getUserByIdentity("user5");
assertNotNull(user5);
final User user6 = authorizer.getUserByIdentity("user6");
assertNotNull(user6);
// verify one group got created
final Set<Group> groups = authorizer.getGroups();
assertEquals(1, groups.size());
final Group group1 = groups.iterator().next();
assertEquals("GROUP1", group1.getName());
final Resource inputPortResource = ResourceFactory.getDataTransferResource(
ResourceFactory.getComponentResource(ResourceType.InputPort, "2f7d1606-b090-4be7-a592-a5b70fb55531", "TCP Input"));
final AccessPolicy inputPortPolicy = authorizer.getUsersAndAccessPolicies().getAccessPolicy(inputPortResource.getIdentifier(), RequestAction.WRITE);
assertNotNull(inputPortPolicy);
assertEquals(1, inputPortPolicy.getUsers().size());
assertTrue(inputPortPolicy.getUsers().contains(user6.getIdentifier()));
assertEquals(1, inputPortPolicy.getGroups().size());
assertTrue(inputPortPolicy.getGroups().contains(group1.getIdentifier()));
final Resource outputPortResource = ResourceFactory.getDataTransferResource(
ResourceFactory.getComponentResource(ResourceType.OutputPort, "2f7d1606-b090-4be7-a592-a5b70fb55532", "TCP Output"));
final AccessPolicy outputPortPolicy = authorizer.getUsersAndAccessPolicies().getAccessPolicy(outputPortResource.getIdentifier(), RequestAction.WRITE);
assertNotNull(outputPortPolicy);
assertEquals(1, outputPortPolicy.getUsers().size());
assertTrue(outputPortPolicy.getUsers().contains(user4.getIdentifier()));
}
@Test
public void testOnConfiguredWhenBadLegacyUsersFileProvided() throws Exception {
when(configurationContext.getProperty(Mockito.eq(FileAuthorizer.PROP_LEGACY_AUTHORIZED_USERS_FILE)))
.thenReturn(new StandardPropertyValue("src/test/resources/does-not-exist.xml", null, ParameterLookup.EMPTY));
writeFile(primaryAuthorizations, EMPTY_AUTHORIZATIONS_CONCISE);
writeFile(primaryTenants, EMPTY_TENANTS_CONCISE);
assertThrows(AuthorizerCreationException.class,
() -> authorizer.onConfigured(configurationContext));
}
@Test
public void testOnConfiguredWhenInitialAdminAndLegacyUsersProvided() throws Exception {
final String adminIdentity = "admin-user";
when(configurationContext.getProperty(Mockito.eq(FileAccessPolicyProvider.PROP_INITIAL_ADMIN_IDENTITY)))
.thenReturn(new StandardPropertyValue(adminIdentity, null, ParameterLookup.EMPTY));
when(configurationContext.getProperty(Mockito.eq(FileAuthorizer.PROP_LEGACY_AUTHORIZED_USERS_FILE)))
.thenReturn(new StandardPropertyValue("src/test/resources/authorized-users.xml", null, ParameterLookup.EMPTY));
writeFile(primaryAuthorizations, EMPTY_AUTHORIZATIONS_CONCISE);
writeFile(primaryTenants, EMPTY_TENANTS_CONCISE);
assertThrows(AuthorizerCreationException.class,
() -> authorizer.onConfigured(configurationContext));
}
@Test
public void testOnConfiguredWhenInitialAdminNotProvided() throws Exception {
writeFile(primaryAuthorizations, EMPTY_AUTHORIZATIONS_CONCISE);
@ -595,8 +297,7 @@ public class FileAuthorizerTest {
// setup NiFi properties to return a file that does not exist
properties = mock(NiFiProperties.class);
when(properties.getRestoreDirectory()).thenReturn(restoreAuthorizations.getParentFile());
when(properties.getFlowConfigurationFile()).thenReturn(new File("src/test/resources/does-not-exist.xml.gz"));
when(properties.getFlowConfigurationJsonFile()).thenReturn(flowJson);
when(properties.getFlowConfigurationFile()).thenReturn(new File("src/test/resources/does-not-exist.json.gz"));
authorizer.setNiFiProperties(properties);
final String adminIdentity = "admin-user";
@ -634,8 +335,7 @@ public class FileAuthorizerTest {
// setup NiFi properties to return a file that does not exist
properties = mock(NiFiProperties.class);
when(properties.getRestoreDirectory()).thenReturn(restoreAuthorizations.getParentFile());
when(properties.getFlowConfigurationFile()).thenReturn(null);
when(properties.getFlowConfigurationJsonFile()).thenReturn(flowJson);
when(properties.getFlowConfigurationFile()).thenReturn(new File("src/test/resources/does-not-exist.json.gz"));
authorizer.setNiFiProperties(properties);
final String adminIdentity = "admin-user";
@ -677,7 +377,6 @@ public class FileAuthorizerTest {
properties = getNiFiProperties(props);
when(properties.getRestoreDirectory()).thenReturn(restoreAuthorizations.getParentFile());
when(properties.getFlowConfigurationFile()).thenReturn(flow);
when(properties.getFlowConfigurationJsonFile()).thenReturn(flowJson);
authorizer.setNiFiProperties(properties);
final String adminIdentity = "CN=localhost, OU=Apache NiFi, O=Apache, L=Santa Monica, ST=CA, C=US";
@ -772,7 +471,6 @@ public class FileAuthorizerTest {
properties = getNiFiProperties(props);
when(properties.getRestoreDirectory()).thenReturn(restoreAuthorizations.getParentFile());
when(properties.getFlowConfigurationFile()).thenReturn(flow);
when(properties.getFlowConfigurationJsonFile()).thenReturn(flowJson);
authorizer.setNiFiProperties(properties);
final String adminIdentity = "CN=user1, OU=Apache NiFi, O=Apache, L=Santa Monica, ST=CA, C=US";

141
nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-file-authorizer/src/test/java/org/apache/nifi/authorization/FileUserGroupProviderTest.java
View File

@ -111,7 +111,6 @@ public class FileUserGroupProviderTest {
when(properties.getRestoreDirectory()).thenReturn(restoreTenants.getParentFile());
configurationContext = mock(AuthorizerConfigurationContext.class);
when(configurationContext.getProperty(eq(FileAuthorizer.PROP_LEGACY_AUTHORIZED_USERS_FILE))).thenReturn(new StandardPropertyValue(null, null, ParameterLookup.EMPTY));
when(configurationContext.getProperty(eq(FileUserGroupProvider.PROP_TENANTS_FILE))).thenReturn(new StandardPropertyValue(primaryTenants.getPath(), null, ParameterLookup.EMPTY));
when(configurationContext.getProperties()).then((invocation) -> {
final Map<String, String> properties = new HashMap<>();
@ -121,11 +120,6 @@ public class FileUserGroupProviderTest {
properties.put(FileUserGroupProvider.PROP_TENANTS_FILE, tenantFile.getValue());
}
final PropertyValue legacyAuthFile = configurationContext.getProperty(FileAuthorizer.PROP_LEGACY_AUTHORIZED_USERS_FILE);
if (legacyAuthFile != null) {
properties.put(FileAuthorizer.PROP_LEGACY_AUTHORIZED_USERS_FILE, legacyAuthFile.getValue());
}
int i = 1;
while (true) {
final String key = FileUserGroupProvider.PROP_INITIAL_USER_IDENTITY_PREFIX + i++;
@ -151,141 +145,6 @@ public class FileUserGroupProviderTest {
deleteFile(restoreTenants);
}
@Test
public void testOnConfiguredWhenLegacyUsersFileProvided() throws Exception {
when(configurationContext.getProperty(eq(FileAuthorizer.PROP_LEGACY_AUTHORIZED_USERS_FILE)))
.thenReturn(new StandardPropertyValue("src/test/resources/authorized-users.xml", null, ParameterLookup.EMPTY));
writeFile(primaryTenants, EMPTY_TENANTS_CONCISE);
userGroupProvider.onConfigured(configurationContext);
// verify all users got created correctly
final Set<User> users = userGroupProvider.getUsers();
assertEquals(6, users.size());
final User user1 = userGroupProvider.getUserByIdentity("user1");
assertNotNull(user1);
final User user2 = userGroupProvider.getUserByIdentity("user2");
assertNotNull(user2);
final User user3 = userGroupProvider.getUserByIdentity("user3");
assertNotNull(user3);
final User user4 = userGroupProvider.getUserByIdentity("user4");
assertNotNull(user4);
final User user5 = userGroupProvider.getUserByIdentity("user5");
assertNotNull(user5);
final User user6 = userGroupProvider.getUserByIdentity("user6");
assertNotNull(user6);
// verify one group got created
final Set<Group> groups = userGroupProvider.getGroups();
assertEquals(1, groups.size());
final Group group1 = groups.iterator().next();
assertEquals("group1", group1.getName());
}
@Test
public void testOnConfiguredWhenLegacyUsersFileProvidedWithIdentityMappings() throws Exception {
final Properties props = new Properties();
props.setProperty("nifi.security.identity.mapping.pattern.dn1", "^CN=(.*?), OU=(.*?), O=(.*?), L=(.*?), ST=(.*?), C=(.*?)$");
props.setProperty("nifi.security.identity.mapping.value.dn1", "$1");
properties = getNiFiProperties(props);
when(properties.getRestoreDirectory()).thenReturn(restoreTenants.getParentFile());
userGroupProvider.setNiFiProperties(properties);
when(configurationContext.getProperty(eq(FileAuthorizer.PROP_LEGACY_AUTHORIZED_USERS_FILE)))
.thenReturn(new StandardPropertyValue("src/test/resources/authorized-users-with-dns.xml", null, ParameterLookup.EMPTY));
writeFile(primaryTenants, EMPTY_TENANTS_CONCISE);
userGroupProvider.onConfigured(configurationContext);
final User user1 = userGroupProvider.getUserByIdentity("user1");
assertNotNull(user1);
final User user2 = userGroupProvider.getUserByIdentity("user2");
assertNotNull(user2);
final User user3 = userGroupProvider.getUserByIdentity("user3");
assertNotNull(user3);
final User user4 = userGroupProvider.getUserByIdentity("user4");
assertNotNull(user4);
final User user5 = userGroupProvider.getUserByIdentity("user5");
assertNotNull(user5);
final User user6 = userGroupProvider.getUserByIdentity("user6");
assertNotNull(user6);
// verify one group got created
final Set<Group> groups = userGroupProvider.getGroups();
assertEquals(1, groups.size());
final Group group1 = groups.iterator().next();
assertEquals("group1", group1.getName());
}
@Test
public void testOnConfiguredWhenLegacyUsersFileProvidedWithIdentityMappingsAndTransforms() throws Exception {
final Properties props = new Properties();
props.setProperty("nifi.security.identity.mapping.pattern.dn1", "^CN=(.*?), OU=(.*?), O=(.*?), L=(.*?), ST=(.*?), C=(.*?)$");
props.setProperty("nifi.security.identity.mapping.value.dn1", "$1");
props.setProperty("nifi.security.identity.mapping.transform.dn1", "UPPER");
props.setProperty("nifi.security.group.mapping.pattern.anygroup", "^(.*)$");
props.setProperty("nifi.security.group.mapping.value.anygroup", "$1");
props.setProperty("nifi.security.group.mapping.transform.anygroup", "UPPER");
properties = getNiFiProperties(props);
when(properties.getRestoreDirectory()).thenReturn(restoreTenants.getParentFile());
userGroupProvider.setNiFiProperties(properties);
when(configurationContext.getProperty(eq(FileAuthorizer.PROP_LEGACY_AUTHORIZED_USERS_FILE)))
.thenReturn(new StandardPropertyValue("src/test/resources/authorized-users-with-dns.xml", null, ParameterLookup.EMPTY));
writeFile(primaryTenants, EMPTY_TENANTS_CONCISE);
userGroupProvider.onConfigured(configurationContext);
final User user1 = userGroupProvider.getUserByIdentity("USER1");
assertNotNull(user1);
final User user2 = userGroupProvider.getUserByIdentity("USER2");
assertNotNull(user2);
final User user3 = userGroupProvider.getUserByIdentity("USER3");
assertNotNull(user3);
final User user4 = userGroupProvider.getUserByIdentity("USER4");
assertNotNull(user4);
final User user5 = userGroupProvider.getUserByIdentity("USER5");
assertNotNull(user5);
final User user6 = userGroupProvider.getUserByIdentity("USER6");
assertNotNull(user6);
// verify one group got created
final Set<Group> groups = userGroupProvider.getGroups();
assertEquals(1, groups.size());
final Group group1 = groups.iterator().next();
assertEquals("GROUP1", group1.getName());
}
@Test
public void testOnConfiguredWhenBadLegacyUsersFileProvided() throws Exception {
when(configurationContext.getProperty(eq(FileAuthorizer.PROP_LEGACY_AUTHORIZED_USERS_FILE)))
.thenReturn(new StandardPropertyValue("src/test/resources/does-not-exist.xml", null, ParameterLookup.EMPTY));
writeFile(primaryTenants, EMPTY_TENANTS_CONCISE);
assertThrows(AuthorizerCreationException.class,
() -> userGroupProvider.onConfigured(configurationContext));
}
@Test
public void testOnConfiguredWhenInitialUsersNotProvided() throws Exception {
writeFile(primaryTenants, EMPTY_TENANTS_CONCISE);

BIN
nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-file-authorizer/src/test/resources/flow-no-ports.json.gz Normal file
View File

Binary file not shown.

BIN
nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-file-authorizer/src/test/resources/flow-no-ports.xml.gz
View File

Binary file not shown.

BIN
nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-file-authorizer/src/test/resources/flow-with-dns.json.gz Normal file
View File

Binary file not shown.

BIN
nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-file-authorizer/src/test/resources/flow-with-dns.xml.gz
View File

Binary file not shown.

BIN
nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-file-authorizer/src/test/resources/flow.json.gz Normal file
View File

Binary file not shown.

BIN
nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-file-authorizer/src/test/resources/flow.xml.gz
View File

Binary file not shown.

10
nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-cluster-protocol/src/main/java/org/apache/nifi/cluster/protocol/StandardDataFlow.java
View File

@ -60,7 +60,7 @@ public class StandardDataFlow implements Serializable, DataFlow {
* Constructs an instance.
*
* @param flow a valid flow as bytes, which cannot be null
* @param snippetBytes an XML representation of snippets. May be null.
* @param snippetBytes a JSON representation of snippets. May be null.
* @param authorizerFingerprint the bytes of the Authorizer's fingerprint. May be null when using an external Authorizer.
* @param missingComponentIds the ids of components that were created as missing ghost components
*
@ -161,12 +161,4 @@ public class StandardDataFlow implements Serializable, DataFlow {
throw new FlowSerializationException("Could not parse flow as a VersionedDataflow", e);
}
}
public boolean isXml() {
if (flow == null || flow.length == 0) {
return true;
}
return flow[0] == '<';
}
}

4
nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-cluster/src/main/java/org/apache/nifi/cluster/coordination/flow/PopularVoteFlowElection.java
View File

@ -20,7 +20,7 @@ package org.apache.nifi.cluster.coordination.flow;
import org.apache.commons.codec.digest.DigestUtils;
import org.apache.nifi.cluster.protocol.DataFlow;
import org.apache.nifi.cluster.protocol.NodeIdentifier;
import org.apache.nifi.controller.serialization.StandardFlowSynchronizer;
import org.apache.nifi.controller.serialization.VersionedFlowSynchronizer;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@ -239,7 +239,7 @@ public class PopularVoteFlowElection implements FlowElection {
}
public boolean isFlowEmpty() {
return StandardFlowSynchronizer.isFlowEmpty(dataFlow);
return VersionedFlowSynchronizer.isFlowEmpty(dataFlow);
}
public Set<NodeIdentifier> getNodes() {

14
nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-cluster/src/test/java/org/apache/nifi/cluster/coordination/flow/TestPopularVoteFlowElection.java
View File

@ -40,7 +40,7 @@ public class TestPopularVoteFlowElection {
@Test
public void testOnlyEmptyFlows() throws IOException {
final PopularVoteFlowElection election = new PopularVoteFlowElection(1, TimeUnit.MINUTES, 3);
final byte[] flow = Files.readAllBytes(Paths.get("src/test/resources/conf/empty-flow.xml"));
final byte[] flow = Files.readAllBytes(Paths.get("src/test/resources/conf/empty-flow.json"));
assertFalse(election.isElectionComplete());
assertNull(election.getElectedDataFlow());
@ -62,8 +62,8 @@ public class TestPopularVoteFlowElection {
@Test
public void testDifferentEmptyFlows() throws IOException {
final PopularVoteFlowElection election = new PopularVoteFlowElection(1, TimeUnit.MINUTES, 3);
final byte[] flow1 = Files.readAllBytes(Paths.get("src/test/resources/conf/empty-flow.xml"));
final byte[] flow2 = Files.readAllBytes(Paths.get("src/test/resources/conf/different-empty-flow.xml"));
final byte[] flow1 = Files.readAllBytes(Paths.get("src/test/resources/conf/empty-flow.json"));
final byte[] flow2 = Files.readAllBytes(Paths.get("src/test/resources/conf/different-empty-flow.json"));
assertFalse(election.isElectionComplete());
assertNull(election.getElectedDataFlow());
@ -87,8 +87,8 @@ public class TestPopularVoteFlowElection {
@Test
public void testEmptyFlowIgnoredIfNonEmptyFlowExists() throws IOException {
final PopularVoteFlowElection election = new PopularVoteFlowElection(1, TimeUnit.MINUTES, 8);
final byte[] emptyFlow = Files.readAllBytes(Paths.get("src/test/resources/conf/empty-flow.xml"));
final byte[] nonEmptyFlow = Files.readAllBytes(Paths.get("src/test/resources/conf/non-empty-flow.xml"));
final byte[] emptyFlow = Files.readAllBytes(Paths.get("src/test/resources/conf/empty-flow.json"));
final byte[] nonEmptyFlow = Files.readAllBytes(Paths.get("src/test/resources/conf/non-empty-flow.json"));
for (int i = 0; i < 8; i++) {
assertFalse(election.isElectionComplete());
@ -114,8 +114,8 @@ public class TestPopularVoteFlowElection {
@Test
public void testAutoGeneratedVsPopulatedFlowElection() throws IOException {
final PopularVoteFlowElection election = new PopularVoteFlowElection(1, TimeUnit.MINUTES, 4);
final byte[] emptyFlow = Files.readAllBytes(Paths.get("src/test/resources/conf/auto-generated-empty-flow.xml"));
final byte[] nonEmptyFlow = Files.readAllBytes(Paths.get("src/test/resources/conf/reporting-task-flow.xml"));
final byte[] emptyFlow = Files.readAllBytes(Paths.get("src/test/resources/conf/auto-generated-empty-flow.json"));
final byte[] nonEmptyFlow = Files.readAllBytes(Paths.get("src/test/resources/conf/reporting-task-flow.json"));
for (int i = 0; i < 4; i++) {
assertFalse(election.isElectionComplete());

1
nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-cluster/src/test/resources/conf/auto-generated-empty-flow.json Normal file
View File

@ -0,0 +1 @@
{"encodingVersion":{"majorVersion":2,"minorVersion":0},"maxTimerDrivenThreadCount":10,"registries":[],"parameterContexts":[],"parameterProviders":[],"controllerServices":[],"reportingTasks":[],"templates":[],"rootGroup":{"identifier":"0bc6db14-095b-392d-a063-0df1646ddf72","instanceIdentifier":"ee207cce-015d-1000-30bf-36cd2fd1ea5c","name":"NiFi Flow","comments":"","position":{"x":0.0,"y":0.0},"processGroups":[],"remoteProcessGroups":[],"processors":[],"inputPorts":[],"outputPorts":[],"connections":[],"labels":[],"funnels":[],"controllerServices":[],"variables":{},"defaultFlowFileExpiration":"0 sec","defaultBackPressureObjectThreshold":10000,"defaultBackPressureDataSizeThreshold":"1 GB","scheduledState":"ENABLED","executionEngine":"INHERITED","maxConcurrentTasks":1,"statelessFlowTimeout":"1 min","componentType":"PROCESS_GROUP","flowFileConcurrency":"UNBOUNDED","flowFileOutboundPolicy":"STREAM_WHEN_AVAILABLE"}}

27
nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-cluster/src/test/resources/conf/auto-generated-empty-flow.xml
View File

@ -1,27 +0,0 @@
<?xml version="1.0" encoding="UTF-8" standalone="no"?>
<!--
Licensed to the Apache Software Foundation (ASF) under one or more
contributor license agreements. See the NOTICE file distributed with
this work for additional information regarding copyright ownership.
The ASF licenses this file to You under the Apache License, Version 2.0
(the "License"); you may not use this file except in compliance with
the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
-->
<flowController encoding-version="1.1">
<maxTimerDrivenThreadCount>10</maxTimerDrivenThreadCount>
<maxEventDrivenThreadCount>5</maxEventDrivenThreadCount>
<rootGroup>
<id>ee207cce-015d-1000-30bf-36cd2fd1ea5c</id>
<name>NiFi Flow</name>
<position x="0.0" y="0.0"/>
<comment/>
</rootGroup>
<controllerServices/>
<reportingTasks/>
</flowController>

64
nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-cluster/src/test/resources/conf/controller-service-flow.json Normal file
View File

@ -0,0 +1,64 @@
{
"encodingVersion": {
"majorVersion": 2,
"minorVersion": 0
},
"maxTimerDrivenThreadCount": 10,
"registries": [],
"parameterContexts": [],
"parameterProviders": [],
"controllerServices": [],
"reportingTasks": [],
"templates": [],
"rootGroup": {
"identifier": "3ddb274d-5d53-3a68-8db3-1e95b702852d",
"instanceIdentifier": "778f676e-6542-4c18-9d06-24b6fd3a1b29",
"name": "NiFi Flow",
"position": {
"x": 0,
"y": 0
},
"processGroups": [],
"remoteProcessGroups": [],
"processors": [],
"inputPorts": [],
"outputPorts": [],
"connections": [],
"labels": [],
"funnels": [],
"controllerServices": [
{
"identifier": "e5c9d095-39ab-311a-80a7-260efcdd0498",
"instanceIdentifier": "edf22ee5-376a-46dc-a38a-919351124457",
"name": "ControllerService",
"comments": "",
"type": "org.apache.nifi.controller.service.mock.ServiceD",
"bundle": {
"group": "default",
"artifact": "unknown",
"version": "unversioned"
},
"properties": {
"Foo1": "enc{265708cd5e497f7581a37f48445f634879020a73382e2e761cc55587b8a2e55170774348}"
},
"propertyDescriptors": {},
"controllerServiceApis": [],
"scheduledState": "DISABLED",
"bulletinLevel": "WARN",
"componentType": "CONTROLLER_SERVICE",
"groupIdentifier": "3ddb274d-5d53-3a68-8db3-1e95b702852d"
}
],
"variables": {},
"defaultFlowFileExpiration": "0 sec",
"defaultBackPressureObjectThreshold": 10000,
"defaultBackPressureDataSizeThreshold": "1 GB",
"scheduledState": "ENABLED",
"executionEngine": "INHERITED",
"maxConcurrentTasks": 1,
"statelessFlowTimeout": "1 min",
"componentType": "PROCESS_GROUP",
"flowFileConcurrency": "UNBOUNDED",
"flowFileOutboundPolicy": "STREAM_WHEN_AVAILABLE"
}
}

37
nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-cluster/src/test/resources/conf/controller-service-flow.xml
View File

@ -1,37 +0,0 @@
<?xml version="1.0" encoding="UTF-8" standalone="no"?>
<!--
Licensed to the Apache Software Foundation (ASF) under one or more
contributor license agreements. See the NOTICE file distributed with
this work for additional information regarding copyright ownership.
The ASF licenses this file to You under the Apache License, Version 2.0
(the "License"); you may not use this file except in compliance with
the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
-->
<flowController>
<maxTimerDrivenThreadCount>10</maxTimerDrivenThreadCount>
<maxEventDrivenThreadCount>5</maxEventDrivenThreadCount>
<rootGroup>
<id>778f676e-6542-4c18-9d06-24b6fd3a1b29</id>
<name>NiFi Flow</name>
<position x="0.0" y="0.0"/>
</rootGroup>
<controllerServices>
<controllerService>
<id>edf22ee5-376a-46dc-a38a-919351124457</id>
<name>ControllerService</name>
<comment/>
<class>org.apache.nifi.controller.service.mock.ServiceD</class>
<enabled>false</enabled>
<property>
<name>Foo1</name>
<value>Bar1</value>
</property>
</controllerService>
</controllerServices>
</flowController>

1
nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-cluster/src/test/resources/conf/different-empty-flow.json Normal file
View File

@ -0,0 +1 @@
{"encodingVersion":{"majorVersion":2,"minorVersion":0},"maxTimerDrivenThreadCount":10,"registries":[],"parameterContexts":[],"parameterProviders":[],"controllerServices":[],"reportingTasks":[],"templates":[],"rootGroup":{"identifier":"38c6cbd2-8bf1-3507-8d07-0980dd1fb595","instanceIdentifier":"11111111-1111-1111-1111-111111111111","name":"Empty NiFi Flow","comments":"","position":{"x":0.0,"y":0.0},"processGroups":[],"remoteProcessGroups":[],"processors":[],"inputPorts":[],"outputPorts":[],"connections":[],"labels":[],"funnels":[],"controllerServices":[],"variables":{},"defaultFlowFileExpiration":"0 sec","defaultBackPressureObjectThreshold":10000,"defaultBackPressureDataSizeThreshold":"1 GB","scheduledState":"ENABLED","executionEngine":"INHERITED","maxConcurrentTasks":1,"statelessFlowTimeout":"1 min","componentType":"PROCESS_GROUP","flowFileConcurrency":"UNBOUNDED","flowFileOutboundPolicy":"STREAM_WHEN_AVAILABLE"}}

27
nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-cluster/src/test/resources/conf/different-empty-flow.xml
View File

@ -1,27 +0,0 @@
<?xml version="1.0" encoding="UTF-8" standalone="no"?>
<!--
Licensed to the Apache Software Foundation (ASF) under one or more
contributor license agreements. See the NOTICE file distributed with
this work for additional information regarding copyright ownership.
The ASF licenses this file to You under the Apache License, Version 2.0
(the "License"); you may not use this file except in compliance with
the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
-->
<flowController encoding-version="1.0">
<maxTimerDrivenThreadCount>10</maxTimerDrivenThreadCount>
<maxEventDrivenThreadCount>5</maxEventDrivenThreadCount>
<rootGroup>
<id>11111111-1111-1111-1111-111111111111</id>
<name>Empty NiFi Flow</name>
<position x="0.0" y="0.0"/>
<comment/>
</rootGroup>
<controllerServices/>
<reportingTasks/>
</flowController>

1
nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-cluster/src/test/resources/conf/empty-flow.json Normal file
View File

@ -0,0 +1 @@
{"encodingVersion":{"majorVersion":2,"minorVersion":0},"maxTimerDrivenThreadCount":10,"registries":[],"parameterContexts":[],"parameterProviders":[],"controllerServices":[],"reportingTasks":[],"templates":[],"rootGroup":{"identifier":"9f89c84a-559f-3736-b6a4-7ff8daed0d33","instanceIdentifier":"00000000-0000-0000-0000-000000000000","name":"Empty NiFi Flow","comments":"","position":{"x":0.0,"y":0.0},"processGroups":[],"remoteProcessGroups":[],"processors":[],"inputPorts":[],"outputPorts":[],"connections":[],"labels":[],"funnels":[],"controllerServices":[],"variables":{},"defaultFlowFileExpiration":"0 sec","defaultBackPressureObjectThreshold":10000,"defaultBackPressureDataSizeThreshold":"1 GB","scheduledState":"ENABLED","executionEngine":"INHERITED","maxConcurrentTasks":1,"statelessFlowTimeout":"1 min","componentType":"PROCESS_GROUP","flowFileConcurrency":"UNBOUNDED","flowFileOutboundPolicy":"STREAM_WHEN_AVAILABLE"}}

27
nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-cluster/src/test/resources/conf/empty-flow.xml
View File

@ -1,27 +0,0 @@
<?xml version="1.0" encoding="UTF-8" standalone="no"?>
<!--
Licensed to the Apache Software Foundation (ASF) under one or more
contributor license agreements. See the NOTICE file distributed with
this work for additional information regarding copyright ownership.
The ASF licenses this file to You under the Apache License, Version 2.0
(the "License"); you may not use this file except in compliance with
the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
-->
<flowController encoding-version="1.0">
<maxTimerDrivenThreadCount>10</maxTimerDrivenThreadCount>
<maxEventDrivenThreadCount>5</maxEventDrivenThreadCount>
<rootGroup>
<id>00000000-0000-0000-0000-000000000000</id>
<name>Empty NiFi Flow</name>
<position x="0.0" y="0.0"/>
<comment/>
</rootGroup>
<controllerServices/>
<reportingTasks/>
</flowController>

2
nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-cluster/src/test/resources/conf/nifi.properties
View File

@ -14,7 +14,7 @@
# limitations under the License.
# Core Properties #
nifi.flow.configuration.file=./target/flow.xml.gz
nifi.flow.configuration.file=./target/flow.json.gz
nifi.flow.configuration.archive.dir=./target/archive/
nifi.flowcontroller.autoResumeState=true
nifi.flowcontroller.graceful.shutdown.period=10 sec

1
nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-cluster/src/test/resources/conf/non-empty-flow.json Normal file
View File

@ -0,0 +1 @@
{"encodingVersion":{"majorVersion":2,"minorVersion":0},"maxTimerDrivenThreadCount":10,"registries":[],"parameterContexts":[],"parameterProviders":[],"controllerServices":[],"reportingTasks":[],"templates":[],"rootGroup":{"identifier":"9f89c84a-559f-3736-b6a4-7ff8daed0d33","instanceIdentifier":"00000000-0000-0000-0000-000000000000","name":"Integration Test Flow","comments":"","position":{"x":0.0,"y":0.0},"processGroups":[],"remoteProcessGroups":[],"processors":[],"inputPorts":[],"outputPorts":[],"connections":[],"labels":[],"funnels":[],"controllerServices":[],"variables":{},"defaultFlowFileExpiration":"0 sec","defaultBackPressureObjectThreshold":10000,"defaultBackPressureDataSizeThreshold":"1 GB","scheduledState":"ENABLED","executionEngine":"INHERITED","maxConcurrentTasks":1,"statelessFlowTimeout":"1 min","componentType":"PROCESS_GROUP","flowFileConcurrency":"UNBOUNDED","flowFileOutboundPolicy":"STREAM_WHEN_AVAILABLE"}}

26
nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-cluster/src/test/resources/conf/non-empty-flow.xml
View File

@ -1,26 +0,0 @@
<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
<!--
Licensed to the Apache Software Foundation (ASF) under one or more
contributor license agreements. See the NOTICE file distributed with
this work for additional information regarding copyright ownership.
The ASF licenses this file to You under the Apache License, Version 2.0
(the "License"); you may not use this file except in compliance with
the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
-->
<flowController>
<maxTimerDrivenThreadCount>10</maxTimerDrivenThreadCount>
<maxEventDrivenThreadCount>5</maxEventDrivenThreadCount>
<rootGroup>
<id>00000000-0000-0000-0000-000000000000</id>
<name>Integration Test Flow</name>
<position x="0.0" y="0.0"/>
<comment/>
<parameterContextId>1234</parameterContextId>
</rootGroup>
</flowController>

1
nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-cluster/src/test/resources/conf/reporting-task-flow.json Normal file
View File

@ -0,0 +1 @@
{"encodingVersion":{"majorVersion":2,"minorVersion":0},"maxTimerDrivenThreadCount":10,"registries":[],"parameterContexts":[],"parameterProviders":[],"controllerServices":[],"reportingTasks":[{"identifier":"3b80ba0f-a6c0-48db-b721-4dbc04cef28e","instanceIdentifier":"3b80ba0f-a6c0-48db-b721-4dbc04cef28e","name":"AmbariReportingTask","comments":"","type":"org.apache.nifi.reporting.ambari.AmbariReportingTask","bundle":{"group":"org.apache.nifi","artifact":"nifi-standard-nar","version":"1.1.0"},"properties":{"Application ID":"enc{398dad79bd3a82ae9c8c3dfb8627b023d970ee4740813fc452b5a930f2475eeb7b82c1c69755b7ad8d67f63a02149cce318a5edb95ae4203}","Hostname":"enc{e9364c72126486f3d1aa1e454a8382ab3fd561456003e614f054fb8d143193428e9e4cc1ddf1b347c8a20ee884c953f110}","Metrics Collector URL":"enc{1ee753dc18ce639e890c3a8649817929b275f3b803ab751776a499fdc770e65521ec14785f818fa5118ec8d4bee1c572f8af1771803e246b17012dc8a44b99}"},"propertyDescriptors":{},"scheduledState":"RUNNING","schedulingPeriod":"{{nifi_ambari_reporting_frequency}}","schedulingStrategy":"TIMER_DRIVEN","componentType":"REPORTING_TASK"}],"templates":[],"rootGroup":{"identifier":"46f95fe5-3024-31d8-aee8-2537d68316f4","instanceIdentifier":"7c84501d-d10c-407c-b9f3-1d80e38fe36a","name":"NiFi Flow","comments":"","position":{"x":0.0,"y":0.0},"processGroups":[],"remoteProcessGroups":[],"processors":[],"inputPorts":[],"outputPorts":[],"connections":[],"labels":[],"funnels":[],"controllerServices":[],"variables":{},"defaultFlowFileExpiration":"0 sec","defaultBackPressureObjectThreshold":10000,"defaultBackPressureDataSizeThreshold":"1 GB","scheduledState":"ENABLED","executionEngine":"INHERITED","maxConcurrentTasks":1,"statelessFlowTimeout":"1 min","componentType":"PROCESS_GROUP","flowFileConcurrency":"UNBOUNDED","flowFileOutboundPolicy":"STREAM_WHEN_AVAILABLE"}}

54
nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-cluster/src/test/resources/conf/reporting-task-flow.xml
View File

@ -1,54 +0,0 @@
<?xml version="1.0" encoding="UTF-8" standalone="no"?>
<!--
Licensed to the Apache Software Foundation (ASF) under one or more
contributor license agreements. See the NOTICE file distributed with
this work for additional information regarding copyright ownership.
The ASF licenses this file to You under the Apache License, Version 2.0
(the "License"); you may not use this file except in compliance with
the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
-->
<flowController>
<maxTimerDrivenThreadCount>10</maxTimerDrivenThreadCount>
<maxEventDrivenThreadCount>5</maxEventDrivenThreadCount>
<rootGroup>
<id>7c84501d-d10c-407c-b9f3-1d80e38fe36a</id>
<name>NiFi Flow</name>
<position x="0.0" y="0.0"/>
<comment/>
</rootGroup>
<controllerServices/>
<reportingTasks>
<reportingTask>
<id>3b80ba0f-a6c0-48db-b721-4dbc04cef28e</id>
<name>AmbariReportingTask</name>
<comment/>
<class>org.apache.nifi.reporting.ambari.AmbariReportingTask</class>
<bundle>
<group>org.apache.nifi</group>
<artifact>nifi-standard-nar</artifact>
<version>1.1.0</version>
</bundle>
<schedulingPeriod>{{nifi_ambari_reporting_frequency}}</schedulingPeriod>
<scheduledState>RUNNING</scheduledState>
<schedulingStrategy>TIMER_DRIVEN</schedulingStrategy>
<property>
<name>Metrics Collector URL</name>
<value>${ambari.metrics.collector.url}</value>
</property>
<property>
<name>Application ID</name>
<value>${ambari.application.id}</value>
</property>
<property>
<name>Hostname</name>
<value>${hostname(true)}</value>
</property>
</reportingTask>
</reportingTasks>
</flowController>

2
nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/main/java/org/apache/nifi/flow/synchronization/StandardVersionedComponentSynchronizer.java
View File

@ -1472,7 +1472,7 @@ public class StandardVersionedComponentSynchronizer implements VersionedComponen
return decryptor.decrypt(value.substring(ENC_PREFIX.length(), value.length() - ENC_SUFFIX.length()));
} catch (EncryptionException e) {
final String moreDescriptiveMessage = "There was a problem decrypting a sensitive flow configuration value. " +
"Check that the nifi.sensitive.props.key value in nifi.properties matches the value used to encrypt the flow.xml.gz file";
"Check that the nifi.sensitive.props.key value in nifi.properties matches the value used to encrypt the flow.json.gz file";
throw new EncryptionException(moreDescriptiveMessage, e);
}
} else {

2
nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/test/resources/conf/nifi.properties
View File

@ -14,7 +14,7 @@
# limitations under the License.
# Core Properties #
nifi.flow.configuration.file=./target/flow.xml.gz
nifi.flow.configuration.file=./target/flow.json.gz
nifi.flow.configuration.archive.dir=./target/archive/
nifi.flowcontroller.autoResumeState=true
nifi.flowcontroller.graceful.shutdown.period=10 sec

5
nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core-api/src/main/java/org/apache/nifi/cluster/protocol/DataFlow.java
View File

@ -50,9 +50,4 @@ public interface DataFlow {
* @return the component ids of components that were created as a missing ghost component
*/
Set<String> getMissingComponents();
/**
* @return <code>true</code> if the contents are empty or are made up of XML, <code>false</code> if the contents are JSON
*/
boolean isXml();
}

10
nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core-api/src/main/java/org/apache/nifi/remote/PublicPort.java
View File

@ -23,8 +23,6 @@ import org.apache.nifi.remote.exception.NotAuthorizedException;
import org.apache.nifi.remote.exception.RequestExpiredException;
import org.apache.nifi.remote.protocol.ServerProtocol;
import java.util.Set;
/**
* Represents an input or output port that can receive or transfer data via Site-to-Site protocol.
*/
@ -32,14 +30,6 @@ public interface PublicPort extends Port {
boolean isTransmitting();
void setGroupAccessControl(Set<String> groups);
Set<String> getGroupAccessControl();
void setUserAccessControl(Set<String> users);
Set<String> getUserAccessControl();
/**
* Verifies that the specified user is authorized to interact with this port
* and returns a {@link PortAuthorizationResult} indicating why the user is

2
nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core-api/src/main/java/org/apache/nifi/services/FlowService.java
View File

@ -86,7 +86,7 @@ public interface FlowService extends LifeCycle {
void copyCurrentFlow(OutputStream os) throws IOException;
/**
* Copies the contents of the current flow.xml.gz to the given file, overwriting the file if it exists
* Copies the contents of the current flow.json.gz to the given file, overwriting the file if it exists
* @param file the file to write the current flow to
* @throws IOException if unable to read the current flow or unable to write to the given file
*/

31
nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/FlowSerializationStrategy.java
View File

@ -1,31 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.nifi.controller;
public enum FlowSerializationStrategy {
WRITE_XML_ONLY,
WRITE_JSON_ONLY,
WRITE_XML_AND_JSON;
public boolean writesJson() {
return this == WRITE_JSON_ONLY || this == WRITE_XML_AND_JSON;
}
public boolean writesXml() {
return this == WRITE_XML_ONLY || this == WRITE_XML_AND_JSON;
}
}

21
nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/StandardFlowService.java
View File

@ -52,7 +52,7 @@ import org.apache.nifi.connectable.Connection;
import org.apache.nifi.controller.flow.FlowManager;
import org.apache.nifi.controller.serialization.FlowSerializationException;
import org.apache.nifi.controller.serialization.FlowSynchronizationException;
import org.apache.nifi.controller.serialization.StandardFlowSynchronizer;
import org.apache.nifi.controller.serialization.VersionedFlowSynchronizer;
import org.apache.nifi.controller.status.ProcessGroupStatus;
import org.apache.nifi.engine.FlowEngine;
import org.apache.nifi.events.BulletinFactory;
@ -111,7 +111,7 @@ public class StandardFlowService implements FlowService, ProtocolHandler {
private final boolean autoResumeState;
private final Authorizer authorizer;
// Lock is used to protect the flow.xml file.
// Lock is used to protect the flow.json.gz file.
private final ReentrantReadWriteLock rwLock = new ReentrantReadWriteLock();
private final Lock readLock = rwLock.readLock();
private final Lock writeLock = rwLock.writeLock();
@ -150,11 +150,9 @@ public class StandardFlowService implements FlowService, ProtocolHandler {
final FlowController controller,
final NiFiProperties nifiProperties,
final RevisionManager revisionManager,
final Authorizer authorizer,
final FlowSerializationStrategy serializationStrategy) throws IOException {
final Authorizer authorizer) throws IOException {
return new StandardFlowService(controller, nifiProperties, null, false, null, revisionManager, authorizer,
serializationStrategy);
return new StandardFlowService(controller, nifiProperties, null, false, null, revisionManager, authorizer);
}
public static StandardFlowService createClusteredInstance(
@ -165,8 +163,7 @@ public class StandardFlowService implements FlowService, ProtocolHandler {
final RevisionManager revisionManager,
final Authorizer authorizer) throws IOException {
return new StandardFlowService(controller, nifiProperties, senderListener, true, coordinator, revisionManager, authorizer,
FlowSerializationStrategy.WRITE_XML_AND_JSON);
return new StandardFlowService(controller, nifiProperties, senderListener, true, coordinator, revisionManager, authorizer);
}
private StandardFlowService(
@ -176,17 +173,15 @@ public class StandardFlowService implements FlowService, ProtocolHandler {
final boolean configuredForClustering,
final ClusterCoordinator clusterCoordinator,
final RevisionManager revisionManager,
final Authorizer authorizer,
final FlowSerializationStrategy serializationStrategy) throws IOException {
final Authorizer authorizer) throws IOException {
this.nifiProperties = nifiProperties;
this.controller = controller;
gracefulShutdownSeconds = (int) FormatUtils.getTimeDuration(nifiProperties.getProperty(NiFiProperties.FLOW_CONTROLLER_GRACEFUL_SHUTDOWN_PERIOD), TimeUnit.SECONDS);
autoResumeState = nifiProperties.getAutoResumeState();
dao = new StandardFlowConfigurationDAO(nifiProperties, controller.getExtensionManager(), serializationStrategy);
dao = new StandardFlowConfigurationDAO(nifiProperties, controller.getExtensionManager());
this.clusterCoordinator = clusterCoordinator;
if (clusterCoordinator != null) {
clusterCoordinator.setFlowService(this);
@ -465,7 +460,7 @@ public class StandardFlowService implements FlowService, ProtocolHandler {
* the response will be null and we should load the local dataflow
* and heartbeat until a manager is located.
*/
final boolean localFlowEmpty = StandardFlowSynchronizer.isFlowEmpty(proposedFlow);
final boolean localFlowEmpty = VersionedFlowSynchronizer.isFlowEmpty(proposedFlow);
final ConnectionResponse response = connect(true, localFlowEmpty, proposedFlow);
// obtain write lock while we are updating the controller. We need to ensure that we don't

13
nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/StandardFlowSnippet.java
View File

@ -46,7 +46,6 @@ import org.apache.nifi.processor.Relationship;
import org.apache.nifi.processor.StandardProcessContext;
import org.apache.nifi.registry.flow.StandardVersionControlInformation;
import org.apache.nifi.registry.flow.VersionControlInformation;
import org.apache.nifi.remote.PublicPort;
import org.apache.nifi.remote.StandardRemoteProcessGroupPortDescriptor;
import org.apache.nifi.remote.protocol.SiteToSiteTransportProtocol;
import org.apache.nifi.scheduling.ExecutionNode;
@ -309,12 +308,6 @@ public class StandardFlowSnippet implements FlowSnippet {
if (group.isRootGroup() || Boolean.TRUE.equals(portDTO.getAllowRemoteAccess())) {
final String portName = generatePublicInputPortName(flowManager, portDTO.getName());
inputPort = flowManager.createPublicInputPort(portDTO.getId(), portName);
if (portDTO.getGroupAccessControl() != null) {
((PublicPort) inputPort).setGroupAccessControl(portDTO.getGroupAccessControl());
}
if (portDTO.getUserAccessControl() != null) {
((PublicPort) inputPort).setUserAccessControl(portDTO.getUserAccessControl());
}
} else {
inputPort = flowManager.createLocalInputPort(portDTO.getId(), portDTO.getName());
}
@ -337,12 +330,6 @@ public class StandardFlowSnippet implements FlowSnippet {
if (group.isRootGroup() || Boolean.TRUE.equals(portDTO.getAllowRemoteAccess())) {
final String portName = generatePublicOutputPortName(flowManager, portDTO.getName());
outputPort = flowManager.createPublicOutputPort(portDTO.getId(), portName);
if (portDTO.getGroupAccessControl() != null) {
((PublicPort) outputPort).setGroupAccessControl(portDTO.getGroupAccessControl());
}
if (portDTO.getUserAccessControl() != null) {
((PublicPort) outputPort).setUserAccessControl(portDTO.getUserAccessControl());
}
} else {
outputPort = flowManager.createLocalOutputPort(portDTO.getId(), portDTO.getName());
}

1981
nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/XmlFlowSynchronizer.java
View File

File diff suppressed because it is too large Load Diff

8
nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/inheritance/BundleCompatibilityCheck.java
View File

@ -40,14 +40,6 @@ public class BundleCompatibilityCheck implements FlowInheritabilityCheck {
@Override
public FlowInheritability checkInheritability(final DataFlow existingFlow, final DataFlow proposedFlow, final FlowController flowController) {
if (proposedFlow.isXml()) {
return checkInheritability(proposedFlow.getFlowDocument(), flowController);
} else {
return checkVersionedFlowInheritability(proposedFlow, flowController);
}
}
private FlowInheritability checkVersionedFlowInheritability(final DataFlow proposedFlow, final FlowController flowController) {
return checkBundles(proposedFlow, flowController.getExtensionManager());
}

56
nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/inheritance/ConnectionMissingCheck.java
View File

@ -22,24 +22,17 @@ import org.apache.nifi.controller.FlowController;
import org.apache.nifi.controller.flow.FlowManager;
import org.apache.nifi.controller.flow.VersionedDataflow;
import org.apache.nifi.controller.repository.FlowFileRepository;
import org.apache.nifi.controller.serialization.FlowEncodingVersion;
import org.apache.nifi.controller.serialization.FlowFromDOMFactory;
import org.apache.nifi.controller.serialization.FlowSynchronizationException;
import org.apache.nifi.flow.ComponentType;
import org.apache.nifi.flow.VersionedComponent;
import org.apache.nifi.registry.flow.diff.DifferenceType;
import org.apache.nifi.registry.flow.diff.FlowComparison;
import org.apache.nifi.registry.flow.diff.FlowDifference;
import org.apache.nifi.web.api.dto.ConnectionDTO;
import org.apache.nifi.web.api.dto.ProcessGroupDTO;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.w3c.dom.Document;
import org.w3c.dom.Element;
import java.io.IOException;
import java.util.Collections;
import java.util.HashSet;
import java.util.Set;
/**
@ -57,11 +50,7 @@ public class ConnectionMissingCheck implements FlowInheritabilityCheck {
@Override
public FlowInheritability checkInheritability(final DataFlow existingFlow, final DataFlow proposedFlow, final FlowController flowController) {
if (proposedFlow.isXml()) {
return checkInheritability(proposedFlow.getFlowDocument(), flowController);
} else {
return checkInheritability(existingFlow.getVersionedDataflow(), proposedFlow.getVersionedDataflow(), flowController);
}
return checkInheritability(existingFlow.getVersionedDataflow(), proposedFlow.getVersionedDataflow(), flowController);
}
private FlowInheritability checkInheritability(final VersionedDataflow existingFlow, final VersionedDataflow proposedFlow, final FlowController flowController) {
@ -111,47 +100,4 @@ public class ConnectionMissingCheck implements FlowInheritabilityCheck {
return FlowInheritability.inheritable();
}
private FlowInheritability checkInheritability(final Document flowDocument, final FlowController flowController) {
final Element rootGroupElement = (Element) flowDocument.getDocumentElement().getElementsByTagName("rootGroup").item(0);
final FlowEncodingVersion encodingVersion = FlowEncodingVersion.parse(flowDocument.getDocumentElement());
final ProcessGroupDTO rootGroupDto = FlowFromDOMFactory.getProcessGroup(null, rootGroupElement, null, encodingVersion);
final Set<String> connectionIds = findAllConnectionIds(rootGroupDto);
final FlowFileRepository flowFileRepository = flowController.getRepositoryContextFactory().getFlowFileRepository();
final Set<String> queuesWithFlowFiles;
try {
queuesWithFlowFiles = flowFileRepository.findQueuesWithFlowFiles(flowController.createSwapManager());
} catch (final IOException ioe) {
throw new FlowSynchronizationException("Failed to determine which connections have FlowFiles queued", ioe);
}
logger.debug("The following {} Connections/Queues have data queued up currently: {}", queuesWithFlowFiles.size(), queuesWithFlowFiles);
for (final String queueId : queuesWithFlowFiles) {
if (!connectionIds.contains(queueId)) {
return FlowInheritability.notInheritable("Proposed Flow does not contain a Connection with ID " + queueId + " but this instance has data queued in that connection");
}
}
return FlowInheritability.inheritable();
}
private Set<String> findAllConnectionIds(final ProcessGroupDTO group) {
final Set<String> connectionIds = new HashSet<>();
findAllConnectionIds(group, connectionIds);
return connectionIds;
}
private void findAllConnectionIds(final ProcessGroupDTO group, final Set<String> ids) {
for (final ConnectionDTO connectionDTO : group.getContents().getConnections()) {
ids.add(connectionDTO.getId());
}
for (final ProcessGroupDTO childGroup : group.getContents().getProcessGroups()) {
findAllConnectionIds(childGroup, ids);
}
}
}

99
nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/inheritance/FlowFingerprintCheck.java
View File

@ -1,99 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.nifi.controller.inheritance;
import org.apache.nifi.cluster.protocol.DataFlow;
import org.apache.nifi.controller.FlowController;
import org.apache.nifi.encrypt.PropertyEncryptor;
import org.apache.nifi.encrypt.SensitiveValueEncoder;
import org.apache.nifi.fingerprint.FingerprintFactory;
import org.apache.nifi.nar.ExtensionManager;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
public class FlowFingerprintCheck implements FlowInheritabilityCheck {
private static final Logger logger = LoggerFactory.getLogger(FlowFingerprintCheck.class);
@Override
public FlowInheritability checkInheritability(final DataFlow existingFlow, final DataFlow proposedFlow, final FlowController flowController) {
if (existingFlow == null) {
return FlowInheritability.inheritable();
}
final byte[] existingFlowBytes = existingFlow.getFlow();
final byte[] proposedFlowBytes = proposedFlow.getFlow();
final PropertyEncryptor encryptor = flowController.getEncryptor();
final ExtensionManager extensionManager = flowController.getExtensionManager();
final SensitiveValueEncoder sensitiveValueEncoder = flowController.getSensitiveValueEncoder();
final FingerprintFactory fingerprintFactory = new FingerprintFactory(encryptor, extensionManager, sensitiveValueEncoder);
final String existingFlowFingerprintBeforeHash = fingerprintFactory.createFingerprint(existingFlowBytes, flowController);
if (existingFlowFingerprintBeforeHash.trim().isEmpty()) {
return null; // no existing flow, so equivalent to proposed flow
}
if (proposedFlow == null || proposedFlowBytes.length == 0) {
return FlowInheritability.notInheritable("Proposed Flow was empty but Current Flow is not"); // existing flow is not empty and proposed flow is empty (we could orphan flowfiles)
}
final String proposedFlowFingerprintBeforeHash = fingerprintFactory.createFingerprint(proposedFlow.getFlowDocument(), flowController);
if (proposedFlowFingerprintBeforeHash.trim().isEmpty()) {
return FlowInheritability.notInheritable("Proposed Flow was empty but Current Flow is not"); // existing flow is not empty and proposed flow is empty (we could orphan flowfiles)
}
if (logger.isTraceEnabled()) {
logger.trace("Local Fingerprint Before Hash = {}", new Object[] {existingFlowFingerprintBeforeHash});
logger.trace("Proposed Fingerprint Before Hash = {}", new Object[] {proposedFlowFingerprintBeforeHash});
}
final boolean inheritable = existingFlowFingerprintBeforeHash.equals(proposedFlowFingerprintBeforeHash);
if (!inheritable) {
final String discrepancy = findFirstDiscrepancy(existingFlowFingerprintBeforeHash, proposedFlowFingerprintBeforeHash, "Flows");
return FlowInheritability.notInheritable(discrepancy);
}
return FlowInheritability.inheritable();
}
private String findFirstDiscrepancy(final String existing, final String proposed, final String comparisonDescription) {
final int shortestFileLength = Math.min(existing.length(), proposed.length());
for (int i = 0; i < shortestFileLength; i++) {
if (existing.charAt(i) != proposed.charAt(i)) {
final String formattedExistingDelta = formatFlowDiscrepancy(existing, i, 100);
final String formattedProposedDelta = formatFlowDiscrepancy(proposed, i, 100);
return String.format("Found difference in %s:\nLocal Fingerprint: %s\nCluster Fingerprint: %s", comparisonDescription, formattedExistingDelta, formattedProposedDelta);
}
}
// existing must startWith proposed or proposed must startWith existing
if (existing.length() > proposed.length()) {
final String formattedExistingDelta = existing.substring(proposed.length(), Math.min(existing.length(), proposed.length() + 200));
return String.format("Found difference in %s:\nLocal Fingerprint contains additional configuration from Cluster Fingerprint: %s", comparisonDescription, formattedExistingDelta);
} else if (proposed.length() > existing.length()) {
final String formattedProposedDelta = proposed.substring(existing.length(), Math.min(proposed.length(), existing.length() + 200));
return String.format("Found difference in %s:\nCluster Fingerprint contains additional configuration from Local Fingerprint: %s", comparisonDescription, formattedProposedDelta);
}
return "Unable to find any discrepancies between fingerprints. Please contact the NiFi support team";
}
private String formatFlowDiscrepancy(final String flowFingerprint, final int deltaIndex, final int deltaPad) {
return flowFingerprint.substring(Math.max(0, deltaIndex - deltaPad), Math.min(flowFingerprint.length(), deltaIndex + deltaPad));
}
}

783
nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/serialization/FlowFromDOMFactory.java
View File

@ -1,783 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.nifi.controller.serialization;
import org.apache.nifi.connectable.Size;
import org.apache.nifi.controller.ScheduledState;
import org.apache.nifi.controller.service.ControllerServiceState;
import org.apache.nifi.encrypt.EncryptionException;
import org.apache.nifi.encrypt.PropertyEncryptor;
import org.apache.nifi.groups.RemoteProcessGroupPortDescriptor;
import org.apache.nifi.parameter.ExpressionLanguageAwareParameterParser;
import org.apache.nifi.parameter.ParameterParser;
import org.apache.nifi.parameter.ParameterTokenList;
import org.apache.nifi.remote.StandardRemoteProcessGroupPortDescriptor;
import org.apache.nifi.scheduling.ExecutionNode;
import org.apache.nifi.scheduling.SchedulingStrategy;
import org.apache.nifi.util.DomUtils;
import org.apache.nifi.web.api.dto.BundleDTO;
import org.apache.nifi.web.api.dto.ConnectableDTO;
import org.apache.nifi.web.api.dto.ConnectionDTO;
import org.apache.nifi.web.api.dto.ControllerServiceDTO;
import org.apache.nifi.web.api.dto.FlowAnalysisRuleDTO;
import org.apache.nifi.web.api.dto.FlowRegistryClientDTO;
import org.apache.nifi.web.api.dto.FlowSnippetDTO;
import org.apache.nifi.web.api.dto.FunnelDTO;
import org.apache.nifi.web.api.dto.LabelDTO;
import org.apache.nifi.web.api.dto.ParameterContextDTO;
import org.apache.nifi.web.api.dto.ParameterDTO;
import org.apache.nifi.web.api.dto.ParameterProviderConfigurationDTO;
import org.apache.nifi.web.api.dto.ParameterProviderDTO;
import org.apache.nifi.web.api.dto.PortDTO;
import org.apache.nifi.web.api.dto.PositionDTO;
import org.apache.nifi.web.api.dto.ProcessGroupDTO;
import org.apache.nifi.web.api.dto.ProcessorConfigDTO;
import org.apache.nifi.web.api.dto.ProcessorDTO;
import org.apache.nifi.web.api.dto.RemoteProcessGroupDTO;
import org.apache.nifi.web.api.dto.ReportingTaskDTO;
import org.apache.nifi.web.api.dto.VersionControlInformationDTO;
import org.apache.nifi.web.api.entity.ParameterContextReferenceEntity;
import org.apache.nifi.web.api.entity.ParameterEntity;
import org.apache.nifi.web.api.entity.ParameterProviderConfigurationEntity;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.w3c.dom.Element;
import org.w3c.dom.NodeList;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
import java.util.LinkedHashMap;
import java.util.LinkedHashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.TimeUnit;
public class FlowFromDOMFactory {
private static final Logger logger = LoggerFactory.getLogger(FlowFromDOMFactory.class);
private static final String DEPRECATED_FLOW_REGISTRY_CLIENT_TYPE = "org.apache.nifi.registry.flow.NifiRegistryFlowRegistryClient";
public static BundleDTO getBundle(final Element bundleElement) {
if (bundleElement == null) {
return null;
}
final Element groupElement = DomUtils.getChild(bundleElement, "group");
final Element artifactElement = DomUtils.getChild(bundleElement, "artifact");
final Element versionElement = DomUtils.getChild(bundleElement, "version");
return new BundleDTO(groupElement.getTextContent(), artifactElement.getTextContent(), versionElement.getTextContent());
}
public static PositionDTO getPosition(final Element positionElement) {
if (positionElement == null) {
throw new IllegalArgumentException("Invalid Flow: Found no 'position' element");
}
return new PositionDTO(Double.parseDouble(positionElement.getAttribute("x")), Double.parseDouble(positionElement.getAttribute("y")));
}
public static Size getSize(final Element sizeElement) {
if (sizeElement == null) {
throw new IllegalArgumentException("Invalid Flow: Found no 'size' element");
}
return new Size(Double.parseDouble(sizeElement.getAttribute("width")), Double.parseDouble(sizeElement.getAttribute("height")));
}
public static Map<String, String> getStyle(final Element stylesElement) {
final Map<String, String> styles = new HashMap<>();
if (stylesElement == null) {
return styles;
}
for (final Element styleElement : getChildrenByTagName(stylesElement, "style")) {
final String styleName = styleElement.getAttribute("name");
final String styleValue = styleElement.getTextContent();
styles.put(styleName, styleValue);
}
return styles;
}
public static ControllerServiceDTO getControllerService(final Element element, final PropertyEncryptor encryptor, final FlowEncodingVersion flowEncodingVersion) {
final ControllerServiceDTO dto = new ControllerServiceDTO();
dto.setId(getString(element, "id"));
dto.setVersionedComponentId(getString(element, "versionedComponentId"));
dto.setName(getString(element, "name"));
dto.setComments(getString(element, "comment"));
dto.setBulletinLevel(getString(element, "bulletinLevel"));
dto.setType(getString(element, "class"));
dto.setBundle(getBundle(DomUtils.getChild(element, "bundle")));
final boolean enabled = getBoolean(element, "enabled");
dto.setState(enabled ? ControllerServiceState.ENABLED.name() : ControllerServiceState.DISABLED.name());
dto.setSensitiveDynamicPropertyNames(getSensitivePropertyNames(element));
dto.setProperties(getProperties(element, encryptor, flowEncodingVersion));
dto.setAnnotationData(getString(element, "annotationData"));
return dto;
}
public static ReportingTaskDTO getReportingTask(final Element element, final PropertyEncryptor encryptor, final FlowEncodingVersion flowEncodingVersion) {
final ReportingTaskDTO dto = new ReportingTaskDTO();
dto.setId(getString(element, "id"));
dto.setName(getString(element, "name"));
dto.setComments(getString(element, "comment"));
dto.setType(getString(element, "class"));
dto.setBundle(getBundle(DomUtils.getChild(element, "bundle")));
dto.setSchedulingPeriod(getString(element, "schedulingPeriod"));
dto.setState(getString(element, "scheduledState"));
dto.setSchedulingStrategy(getString(element, "schedulingStrategy"));
dto.setSensitiveDynamicPropertyNames(getSensitivePropertyNames(element));
dto.setProperties(getProperties(element, encryptor, flowEncodingVersion));
dto.setAnnotationData(getString(element, "annotationData"));
return dto;
}
public static FlowAnalysisRuleDTO getFlowAnalysisRule(Element element, PropertyEncryptor encryptor, FlowEncodingVersion flowEncodingVersion) {
final FlowAnalysisRuleDTO dto = new FlowAnalysisRuleDTO();
dto.setId(getString(element, "id"));
dto.setName(getString(element, "name"));
dto.setComments(getString(element, "comment"));
dto.setType(getString(element, "class"));
dto.setBundle(getBundle(DomUtils.getChild(element, "bundle")));
dto.setEnforcementPolicy(getString(element, "enforcementPolicy"));
dto.setState(getString(element, "state"));
dto.setSensitiveDynamicPropertyNames(getSensitivePropertyNames(element));
dto.setProperties(getProperties(element, encryptor, flowEncodingVersion));
return dto;
}
public static FlowRegistryClientDTO getFlowRegistryClient(final Element element, final PropertyEncryptor encryptor, final FlowEncodingVersion flowEncodingVersion) {
final FlowRegistryClientDTO dto = new FlowRegistryClientDTO();
if (isOldStyleRegistryClient(element)) {
return getFlowRegistryClientFromOldStyleConfig(element);
}
dto.setId(getString(element, "id"));
dto.setName(getString(element, "name"));
dto.setDescription(getString(element, "description"));
dto.setUri(getString(element, "uri"));
dto.setType(getString(element, "class"));
dto.setBundle(getBundle(DomUtils.getChild(element, "bundle")));
dto.setSensitiveDynamicPropertyNames(getSensitivePropertyNames(element));
dto.setProperties(getProperties(element, encryptor, flowEncodingVersion));
dto.setAnnotationData(getString(element, "annotationData"));
return dto;
}
private static FlowRegistryClientDTO getFlowRegistryClientFromOldStyleConfig(final Element element) {
final String id = getString(element, "id");
final String name = getString(element, "name");
final String url = getString(element, "url");
final String description = getString(element, "description");
final FlowRegistryClientDTO dto = new FlowRegistryClientDTO();
dto.setId(id);
dto.setName(name);
dto.setDescription(description);
dto.setUri(url);
dto.setType(DEPRECATED_FLOW_REGISTRY_CLIENT_TYPE);
dto.setBundle(new BundleDTO("org.apache.nifi", "nifi-flow-registry-client-nar", "1.18.0"));
dto.setSensitiveDynamicPropertyNames(Collections.emptySet());
dto.setProperties(Collections.singletonMap("url", url));
dto.setAnnotationData(null);
return dto;
}
private static boolean isOldStyleRegistryClient(final Element element) {
final String id = getString(element, "id");
final String identifier = getString(element, "identifier");
final List<Element> bundleElements = getChildrenByTagName(element, "bundle");
return id != null && identifier == null && bundleElements.isEmpty();
}
public static ParameterProviderDTO getParameterProvider(final Element element, final PropertyEncryptor encryptor, final FlowEncodingVersion flowEncodingVersion) {
final ParameterProviderDTO dto = new ParameterProviderDTO();
dto.setId(getString(element, "id"));
dto.setName(getString(element, "name"));
dto.setComments(getString(element, "comment"));
dto.setType(getString(element, "class"));
dto.setBundle(getBundle(DomUtils.getChild(element, "bundle")));
dto.setProperties(getProperties(element, encryptor, flowEncodingVersion));
dto.setAnnotationData(getString(element, "annotationData"));
return dto;
}
public static ParameterContextDTO getParameterContext(final Element element, final PropertyEncryptor encryptor) {
final ParameterContextDTO dto = new ParameterContextDTO();
dto.setId(getString(element, "id"));
dto.setName(getString(element, "name"));
dto.setDescription(getString(element, "description"));
final Set<ParameterEntity> parameterDtos = new LinkedHashSet<>();
final List<Element> parameterElements = FlowFromDOMFactory.getChildrenByTagName(element, "parameter");
for (final Element parameterElement : parameterElements) {
final ParameterDTO parameterDto = new ParameterDTO();
parameterDto.setName(getString(parameterElement, "name"));
parameterDto.setDescription(getString(parameterElement, "description"));
parameterDto.setSensitive(getBoolean(parameterElement, "sensitive"));
parameterDto.setProvided(getBoolean(parameterElement, "provided"));
final String value = decrypt(getString(parameterElement, "value"), encryptor);
parameterDto.setValue(value);
final ParameterEntity parameterEntity = new ParameterEntity();
parameterEntity.setParameter(parameterDto);
parameterDtos.add(parameterEntity);
}
final List<Element> inheritedParameterContextIds = FlowFromDOMFactory.getChildrenByTagName(element, "inheritedParameterContextId");
final List<ParameterContextReferenceEntity> parameterContexts = new ArrayList<>();
for (final Element inheritedParameterContextElement : inheritedParameterContextIds) {
final ParameterContextReferenceEntity parameterContextReference = new ParameterContextReferenceEntity();
parameterContextReference.setId(inheritedParameterContextElement.getTextContent());
parameterContexts.add(parameterContextReference);
}
dto.setInheritedParameterContexts(parameterContexts);
final ParameterProviderConfigurationEntity parameterProviderConfiguration = getParameterProviderConfiguration(element);
if (parameterProviderConfiguration != null) {
dto.setParameterProviderConfiguration(parameterProviderConfiguration);
}
dto.setParameters(parameterDtos);
return dto;
}
private static ParameterProviderConfigurationEntity getParameterProviderConfiguration(final Element parameterContextElement) {
final String parameterProviderId = getString(parameterContextElement, "parameterProviderId");
if (parameterProviderId != null) {
final ParameterProviderConfigurationEntity entity = new ParameterProviderConfigurationEntity();
entity.setId(parameterProviderId);
final ParameterProviderConfigurationDTO dto = new ParameterProviderConfigurationDTO();
final String parameterGroupName = getString(parameterContextElement, "parameterGroupName");
final Boolean isSynchronized = getBoolean(parameterContextElement, "isSynchronized");
dto.setParameterProviderId(parameterProviderId);
dto.setParameterGroupName(parameterGroupName);
dto.setSynchronized(isSynchronized);
entity.setComponent(dto);
return entity;
}
return null;
}
public static ProcessGroupDTO getProcessGroup(final String parentId, final Element element, final PropertyEncryptor encryptor, final FlowEncodingVersion encodingVersion) {
final ProcessGroupDTO dto = new ProcessGroupDTO();
final String groupId = getString(element, "id");
dto.setId(groupId);
dto.setVersionedComponentId(getString(element, "versionedComponentId"));
dto.setParentGroupId(parentId);
dto.setName(getString(element, "name"));
dto.setPosition(getPosition(DomUtils.getChild(element, "position")));
dto.setComments(getString(element, "comment"));
dto.setFlowfileConcurrency(getString(element, "flowfileConcurrency"));
dto.setFlowfileOutboundPolicy(getString(element, "flowfileOutboundPolicy"));
dto.setDefaultFlowFileExpiration(getString(element, "defaultFlowFileExpiration"));
dto.setDefaultBackPressureObjectThreshold(getLong(element, "defaultBackPressureObjectThreshold"));
dto.setDefaultBackPressureDataSizeThreshold(getString(element, "defaultBackPressureDataSizeThreshold"));
dto.setLogFileSuffix(getString(element, "logFileSuffix"));
final Element versionControlInfoElement = DomUtils.getChild(element, "versionControlInformation");
dto.setVersionControlInformation(getVersionControlInformation(versionControlInfoElement));
final String parameterContextId = getString(element, "parameterContextId");
final ParameterContextReferenceEntity parameterContextReference = new ParameterContextReferenceEntity();
parameterContextReference.setId(parameterContextId);
dto.setParameterContext(parameterContextReference);
final Set<ProcessorDTO> processors = new HashSet<>();
final Set<ConnectionDTO> connections = new HashSet<>();
final Set<FunnelDTO> funnels = new HashSet<>();
final Set<PortDTO> inputPorts = new HashSet<>();
final Set<PortDTO> outputPorts = new HashSet<>();
final Set<LabelDTO> labels = new HashSet<>();
final Set<ProcessGroupDTO> processGroups = new HashSet<>();
final Set<RemoteProcessGroupDTO> remoteProcessGroups = new HashSet<>();
final Set<ControllerServiceDTO> controllerServices = new HashSet<>();
NodeList nodeList = DomUtils.getChildNodesByTagName(element, "processor");
for (int i = 0; i < nodeList.getLength(); i++) {
processors.add(getProcessor((Element) nodeList.item(i), encryptor, encodingVersion));
}
nodeList = DomUtils.getChildNodesByTagName(element, "funnel");
for (int i = 0; i < nodeList.getLength(); i++) {
funnels.add(getFunnel((Element) nodeList.item(i)));
}
nodeList = DomUtils.getChildNodesByTagName(element, "inputPort");
for (int i = 0; i < nodeList.getLength(); i++) {
inputPorts.add(getPort((Element) nodeList.item(i)));
}
nodeList = DomUtils.getChildNodesByTagName(element, "outputPort");
for (int i = 0; i < nodeList.getLength(); i++) {
outputPorts.add(getPort((Element) nodeList.item(i)));
}
nodeList = DomUtils.getChildNodesByTagName(element, "label");
for (int i = 0; i < nodeList.getLength(); i++) {
labels.add(getLabel((Element) nodeList.item(i)));
}
nodeList = DomUtils.getChildNodesByTagName(element, "processGroup");
for (int i = 0; i < nodeList.getLength(); i++) {
processGroups.add(getProcessGroup(groupId, (Element) nodeList.item(i), encryptor, encodingVersion));
}
nodeList = DomUtils.getChildNodesByTagName(element, "remoteProcessGroup");
for (int i = 0; i < nodeList.getLength(); i++) {
remoteProcessGroups.add(getRemoteProcessGroup((Element) nodeList.item(i), encryptor));
}
nodeList = DomUtils.getChildNodesByTagName(element, "connection");
for (int i = 0; i < nodeList.getLength(); i++) {
connections.add(getConnection((Element) nodeList.item(i)));
}
nodeList = DomUtils.getChildNodesByTagName(element, "controllerService");
for (int i=0; i < nodeList.getLength(); i++) {
controllerServices.add(getControllerService((Element) nodeList.item(i), encryptor, encodingVersion));
}
final FlowSnippetDTO groupContents = new FlowSnippetDTO();
groupContents.setConnections(connections);
groupContents.setFunnels(funnels);
groupContents.setInputPorts(inputPorts);
groupContents.setLabels(labels);
groupContents.setOutputPorts(outputPorts);
groupContents.setProcessGroups(processGroups);
groupContents.setProcessors(processors);
groupContents.setRemoteProcessGroups(remoteProcessGroups);
groupContents.setControllerServices(controllerServices);
dto.setContents(groupContents);
return dto;
}
private static VersionControlInformationDTO getVersionControlInformation(final Element versionControlInfoElement) {
if (versionControlInfoElement == null) {
return null;
}
final VersionControlInformationDTO dto = new VersionControlInformationDTO();
dto.setRegistryId(getString(versionControlInfoElement, "registryId"));
dto.setBucketId(getString(versionControlInfoElement, "bucketId"));
dto.setBucketName(getString(versionControlInfoElement, "bucketName"));
dto.setFlowId(getString(versionControlInfoElement, "flowId"));
dto.setFlowName(getString(versionControlInfoElement, "flowName"));
dto.setFlowDescription(getString(versionControlInfoElement, "flowDescription"));
dto.setVersion(getInt(versionControlInfoElement, "version"));
dto.setStorageLocation(getString(versionControlInfoElement, "storageLocation"));
return dto;
}
public static ConnectionDTO getConnection(final Element element) {
final ConnectionDTO dto = new ConnectionDTO();
dto.setId(getString(element, "id"));
dto.setName(getString(element, "name"));
dto.setLabelIndex(getOptionalInt(element, "labelIndex"));
dto.setzIndex(getOptionalLong(element, "zIndex"));
dto.setVersionedComponentId(getString(element, "versionedComponentId"));
final List<PositionDTO> bends = new ArrayList<>();
final Element bendPointsElement = DomUtils.getChild(element, "bendPoints");
if (bendPointsElement != null) {
for (final Element bendPointElement : getChildrenByTagName(bendPointsElement, "bendPoint")) {
final PositionDTO bend = getPosition(bendPointElement);
bends.add(bend);
}
}
dto.setBends(bends);
final ConnectableDTO sourceConnectable = new ConnectableDTO();
dto.setSource(sourceConnectable);
sourceConnectable.setId(getString(element, "sourceId"));
sourceConnectable.setGroupId(getString(element, "sourceGroupId"));
sourceConnectable.setType(getString(element, "sourceType"));
final ConnectableDTO destConnectable = new ConnectableDTO();
dto.setDestination(destConnectable);
destConnectable.setId(getString(element, "destinationId"));
destConnectable.setGroupId(getString(element, "destinationGroupId"));
destConnectable.setType(getString(element, "destinationType"));
final Set<String> relationships = new HashSet<>();
final List<Element> relationshipNodeList = getChildrenByTagName(element, "relationship");
for (final Element relationshipElem : relationshipNodeList) {
relationships.add(relationshipElem.getTextContent());
}
dto.setSelectedRelationships(relationships);
dto.setBackPressureObjectThreshold(getLong(element, "maxWorkQueueSize"));
final String maxDataSize = getString(element, "maxWorkQueueDataSize");
if (maxDataSize != null && !maxDataSize.trim().isEmpty()) {
dto.setBackPressureDataSizeThreshold(maxDataSize);
}
String expiration = getString(element, "flowFileExpiration");
if (expiration == null) {
expiration = "0 sec";
}
dto.setFlowFileExpiration(expiration);
final List<String> prioritizerClasses = new ArrayList<>();
final List<Element> prioritizerNodeList = getChildrenByTagName(element, "queuePrioritizerClass");
for (final Element prioritizerElement : prioritizerNodeList) {
prioritizerClasses.add(prioritizerElement.getTextContent().trim());
}
dto.setPrioritizers(prioritizerClasses);
dto.setLoadBalanceStrategy(getString(element, "loadBalanceStrategy"));
dto.setLoadBalancePartitionAttribute(getString(element, "partitioningAttribute"));
dto.setLoadBalanceCompression(getString(element, "loadBalanceCompression"));
return dto;
}
public static RemoteProcessGroupDTO getRemoteProcessGroup(final Element element, final PropertyEncryptor encryptor) {
final RemoteProcessGroupDTO dto = new RemoteProcessGroupDTO();
dto.setId(getString(element, "id"));
dto.setVersionedComponentId(getString(element, "versionedComponentId"));
dto.setName(getString(element, "name"));
dto.setTargetUri(getString(element, "url"));
dto.setTargetUris(getString(element, "urls"));
dto.setTransmitting(getBoolean(element, "transmitting"));
dto.setPosition(getPosition(DomUtils.getChild(element, "position")));
dto.setCommunicationsTimeout(getString(element, "timeout"));
dto.setComments(getString(element, "comment"));
dto.setYieldDuration(getString(element, "yieldPeriod"));
dto.setTransportProtocol(getString(element, "transportProtocol"));
dto.setProxyHost(getString(element, "proxyHost"));
dto.setProxyPort(getOptionalInt(element, "proxyPort"));
dto.setProxyUser(getString(element, "proxyUser"));
dto.setLocalNetworkInterface(getString(element, "networkInterface"));
final String rawPassword = getString(element, "proxyPassword");
final String proxyPassword = encryptor == null ? rawPassword : decrypt(rawPassword, encryptor);
dto.setProxyPassword(proxyPassword);
return dto;
}
public static LabelDTO getLabel(final Element element) {
final LabelDTO dto = new LabelDTO();
dto.setId(getString(element, "id"));
dto.setVersionedComponentId(getString(element, "versionedComponentId"));
dto.setLabel(getString(element, "value"));
dto.setPosition(getPosition(DomUtils.getChild(element, "position")));
final Size size = getSize(DomUtils.getChild(element, "size"));
dto.setWidth(size.getWidth());
dto.setHeight(size.getHeight());
dto.setzIndex(getLong(element, "zIndex"));
dto.setStyle(getStyle(DomUtils.getChild(element, "styles")));
return dto;
}
public static FunnelDTO getFunnel(final Element element) {
final FunnelDTO dto = new FunnelDTO();
dto.setId(getString(element, "id"));
dto.setVersionedComponentId(getString(element, "versionedComponentId"));
dto.setPosition(getPosition(DomUtils.getChild(element, "position")));
return dto;
}
public static PortDTO getPort(final Element element) {
final PortDTO portDTO = new PortDTO();
portDTO.setId(getString(element, "id"));
portDTO.setVersionedComponentId(getString(element, "versionedComponentId"));
portDTO.setPosition(getPosition(DomUtils.getChild(element, "position")));
portDTO.setName(getString(element, "name"));
portDTO.setComments(getString(element, "comments"));
portDTO.setAllowRemoteAccess(getBoolean(element, "allowRemoteAccess"));
final ScheduledState scheduledState = getScheduledState(element);
portDTO.setState(scheduledState.toString());
final List<Element> maxTasksElements = getChildrenByTagName(element, "maxConcurrentTasks");
if (!maxTasksElements.isEmpty()) {
portDTO.setConcurrentlySchedulableTaskCount(Integer.parseInt(maxTasksElements.get(0).getTextContent()));
}
final List<Element> userAccessControls = getChildrenByTagName(element, "userAccessControl");
if (userAccessControls != null && !userAccessControls.isEmpty()) {
final Set<String> users = new HashSet<>();
portDTO.setUserAccessControl(users);
for (final Element userElement : userAccessControls) {
users.add(userElement.getTextContent());
}
}
final List<Element> groupAccessControls = getChildrenByTagName(element, "groupAccessControl");
if (groupAccessControls != null && !groupAccessControls.isEmpty()) {
final Set<String> groups = new HashSet<>();
portDTO.setGroupAccessControl(groups);
for (final Element groupElement : groupAccessControls) {
groups.add(groupElement.getTextContent());
}
}
return portDTO;
}
public static RemoteProcessGroupPortDescriptor getRemoteProcessGroupPort(final Element element) {
final StandardRemoteProcessGroupPortDescriptor descriptor = new StandardRemoteProcessGroupPortDescriptor();
// What we have serialized is the ID of the Remote Process Group, followed by a dash ('-'), followed by
// the actual ID of the port; we want to get rid of the remote process group id.
String id = getString(element, "id");
if (id.length() > 37) {
id = id.substring(37);
}
descriptor.setId(id);
final String targetId = getString(element, "targetId");
descriptor.setTargetId(targetId == null ? id : targetId);
descriptor.setVersionedComponentId(getString(element, "versionedComponentId"));
descriptor.setName(getString(element, "name"));
descriptor.setComments(getString(element, "comments"));
descriptor.setConcurrentlySchedulableTaskCount(getInt(element, "maxConcurrentTasks"));
descriptor.setUseCompression(getBoolean(element, "useCompression"));
descriptor.setBatchCount(getOptionalInt(element, "batchCount"));
descriptor.setBatchSize(getString(element, "batchSize"));
descriptor.setBatchDuration(getString(element, "batchDuration"));
descriptor.setTransmitting("RUNNING".equalsIgnoreCase(getString(element, "scheduledState")));
return descriptor;
}
public static ProcessorDTO getProcessor(final Element element, final PropertyEncryptor encryptor, final FlowEncodingVersion flowEncodingVersion) {
final ProcessorDTO dto = new ProcessorDTO();
dto.setId(getString(element, "id"));
dto.setVersionedComponentId(getString(element, "versionedComponentId"));
dto.setName(getString(element, "name"));
dto.setType(getString(element, "class"));
dto.setBundle(getBundle(DomUtils.getChild(element, "bundle")));
dto.setPosition(getPosition(DomUtils.getChild(element, "position")));
dto.setStyle(getStyle(DomUtils.getChild(element, "styles")));
final ProcessorConfigDTO configDto = new ProcessorConfigDTO();
dto.setConfig(configDto);
configDto.setComments(getString(element, "comment"));
configDto.setConcurrentlySchedulableTaskCount(getInt(element, "maxConcurrentTasks"));
final String schedulingPeriod = getString(element, "schedulingPeriod");
configDto.setSchedulingPeriod(schedulingPeriod);
configDto.setPenaltyDuration(getString(element, "penalizationPeriod"));
configDto.setYieldDuration(getString(element, "yieldPeriod"));
configDto.setBulletinLevel(getString(element, "bulletinLevel"));
configDto.setLossTolerant(getBoolean(element, "lossTolerant"));
if (getString(element, "retryCount") != null) {
configDto.setRetryCount(getInt(element, "retryCount"));
} else {
configDto.setRetryCount(10);
}
configDto.setMaxBackoffPeriod(getString(element, "maxBackoffPeriod"));
configDto.setBackoffMechanism(getString(element, "backoffMechanism"));
final Set<String> retriedRelationships = new HashSet<>();
final List<Element> retriedRelationshipList = getChildrenByTagName(element, "retriedRelationship");
for (final Element retriedRelationship : retriedRelationshipList) {
retriedRelationships.add(retriedRelationship.getTextContent());
}
configDto.setRetriedRelationships(retriedRelationships);
final ScheduledState scheduledState = getScheduledState(element);
dto.setState(scheduledState.toString());
// handle scheduling strategy
final String schedulingStrategyName = getString(element, "schedulingStrategy");
if (schedulingStrategyName == null || schedulingStrategyName.trim().isEmpty()) {
configDto.setSchedulingStrategy(SchedulingStrategy.TIMER_DRIVEN.name());
} else {
configDto.setSchedulingStrategy(schedulingStrategyName.trim());
}
// handle execution node
final String executionNode = getString(element, "executionNode");
if (executionNode == null || executionNode.trim().isEmpty()) {
configDto.setExecutionNode(ExecutionNode.ALL.name());
} else {
configDto.setExecutionNode(executionNode.trim());
}
final Long runDurationNanos = getOptionalLong(element, "runDurationNanos");
if (runDurationNanos != null) {
configDto.setRunDurationMillis(TimeUnit.NANOSECONDS.toMillis(runDurationNanos));
}
configDto.setSensitiveDynamicPropertyNames(getSensitivePropertyNames(element));
configDto.setProperties(getProperties(element, encryptor, flowEncodingVersion));
configDto.setAnnotationData(getString(element, "annotationData"));
final Set<String> autoTerminatedRelationships = new HashSet<>();
final List<Element> autoTerminateList = getChildrenByTagName(element, "autoTerminatedRelationship");
for (final Element autoTerminateElement : autoTerminateList) {
autoTerminatedRelationships.add(autoTerminateElement.getTextContent());
}
configDto.setAutoTerminatedRelationships(autoTerminatedRelationships);
return dto;
}
private static Set<String> getSensitivePropertyNames(final Element element) {
final Set<String> sensitivePropertyNames = new LinkedHashSet<>();
final List<Element> propertyElements = getChildrenByTagName(element, "property");
for (final Element propertyElement : propertyElements) {
final String rawPropertyValue = getString(propertyElement, "value");
if (isValueSensitive(rawPropertyValue)) {
final String name = getString(propertyElement, "name");
sensitivePropertyNames.add(name);
}
}
return sensitivePropertyNames;
}
private static LinkedHashMap<String, String> getProperties(final Element element, final PropertyEncryptor encryptor, final FlowEncodingVersion flowEncodingVersion) {
final LinkedHashMap<String, String> properties = new LinkedHashMap<>();
final List<Element> propertyNodeList = getChildrenByTagName(element, "property");
final ParameterParser parameterParser = new ExpressionLanguageAwareParameterParser();
for (final Element propertyElement : propertyNodeList) {
final String name = getString(propertyElement, "name");
final String rawPropertyValue = getString(propertyElement, "value");
final String value = encryptor == null ? rawPropertyValue : decrypt(rawPropertyValue, encryptor);
if (flowEncodingVersion == null || (flowEncodingVersion.getMajorVersion() <= 1 && flowEncodingVersion.getMinorVersion() < 4)) {
// Version 1.4 introduced the #{paramName} syntax for referencing parameters. If the version is less than 1.4, we must escpae any
// #{...} reference that we find.
final ParameterTokenList parameterTokenList = parameterParser.parseTokens(value);
final String escaped = parameterTokenList.escape();
properties.put(name, escaped);
} else {
properties.put(name, value);
}
}
return properties;
}
private static String getString(final Element element, final String childElementName) {
final List<Element> nodeList = getChildrenByTagName(element, childElementName);
if (nodeList == null || nodeList.isEmpty()) {
return null;
}
final Element childElement = nodeList.get(0);
return childElement.getTextContent();
}
private static Integer getOptionalInt(final Element element, final String childElementName) {
final List<Element> nodeList = getChildrenByTagName(element, childElementName);
if (nodeList == null || nodeList.isEmpty()) {
return null;
}
final Element childElement = nodeList.get(0);
final String val = childElement.getTextContent();
if (val == null) {
return null;
}
return Integer.parseInt(val);
}
private static Long getOptionalLong(final Element element, final String childElementName) {
final List<Element> nodeList = getChildrenByTagName(element, childElementName);
if (nodeList == null || nodeList.isEmpty()) {
return null;
}
final Element childElement = nodeList.get(0);
final String val = childElement.getTextContent();
if (val == null) {
return null;
}
return Long.parseLong(val);
}
private static int getInt(final Element element, final String childElementName) {
return Integer.parseInt(getString(element, childElementName));
}
private static Long getLong(final Element element, final String childElementName) {
// missing element must be handled gracefully, e.g. flow definition from a previous version without this element
String longString = getString(element, childElementName);
return longString == null ? null : Long.parseLong(longString);
}
private static boolean getBoolean(final Element element, final String childElementName) {
return Boolean.parseBoolean(getString(element, childElementName));
}
private static ScheduledState getScheduledState(final Element element) {
return ScheduledState.valueOf(getString(element, "scheduledState"));
}
private static List<Element> getChildrenByTagName(final Element element, final String childElementName) {
return DomUtils.getChildElementsByTagName(element, childElementName);
}
private static String decrypt(final String value, final PropertyEncryptor encryptor) {
if (isValueSensitive(value)) {
try {
return encryptor.decrypt(value.substring(FlowSerializer.ENC_PREFIX.length(), value.length() - FlowSerializer.ENC_SUFFIX.length()));
} catch (EncryptionException e) {
final String moreDescriptiveMessage = "There was a problem decrypting a sensitive flow configuration value. " +
"Check that the nifi.sensitive.props.key value in nifi.properties matches the value used to encrypt the flow.xml.gz file";
logger.error(moreDescriptiveMessage, e);
throw new EncryptionException(moreDescriptiveMessage, e);
}
} else {
return value;
}
}
private static boolean isValueSensitive(final String value) {
return value != null && value.startsWith(FlowSerializer.ENC_PREFIX) && value.endsWith(FlowSerializer.ENC_SUFFIX);
}
}

4
nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/serialization/RunningComponentSetFilter.java
View File

@ -53,7 +53,9 @@ public class RunningComponentSetFilter implements ComponentSetFilter {
public RunningComponentSetFilter(final VersionedDataflow dataflow) {
dataflow.getControllerServices().forEach(service -> controllerServices.put(service.getInstanceIdentifier(), service));
dataflow.getReportingTasks().forEach(task -> reportingTasks.put(task.getInstanceIdentifier(), task));
dataflow.getFlowAnalysisRules().forEach(rule -> flowAnalysisRules.put(rule.getInstanceIdentifier(), rule));
if (dataflow.getFlowAnalysisRules() != null) {
dataflow.getFlowAnalysisRules().forEach(rule -> flowAnalysisRules.put(rule.getInstanceIdentifier(), rule));
}
flatten(dataflow.getRootGroup());
}

716
nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/serialization/StandardFlowSerializer.java
View File

@ -1,716 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.nifi.controller.serialization;
import org.apache.nifi.bundle.BundleCoordinate;
import org.apache.nifi.components.PropertyDescriptor;
import org.apache.nifi.connectable.ConnectableType;
import org.apache.nifi.connectable.Connection;
import org.apache.nifi.connectable.Funnel;
import org.apache.nifi.connectable.Port;
import org.apache.nifi.connectable.Position;
import org.apache.nifi.connectable.Size;
import org.apache.nifi.controller.FlowAnalysisRuleNode;
import org.apache.nifi.controller.FlowController;
import org.apache.nifi.controller.ParameterProviderNode;
import org.apache.nifi.controller.ProcessorNode;
import org.apache.nifi.controller.ReportingTaskNode;
import org.apache.nifi.controller.flow.FlowManager;
import org.apache.nifi.controller.label.Label;
import org.apache.nifi.controller.service.ControllerServiceNode;
import org.apache.nifi.controller.service.ControllerServiceState;
import org.apache.nifi.encrypt.PropertyEncryptor;
import org.apache.nifi.flowfile.FlowFilePrioritizer;
import org.apache.nifi.groups.ProcessGroup;
import org.apache.nifi.groups.RemoteProcessGroup;
import org.apache.nifi.parameter.Parameter;
import org.apache.nifi.parameter.ParameterContext;
import org.apache.nifi.parameter.ParameterContextManager;
import org.apache.nifi.parameter.ParameterDescriptor;
import org.apache.nifi.parameter.ParameterProviderConfiguration;
import org.apache.nifi.processor.Relationship;
import org.apache.nifi.registry.flow.FlowRegistryClientNode;
import org.apache.nifi.registry.flow.VersionControlInformation;
import org.apache.nifi.remote.PublicPort;
import org.apache.nifi.remote.RemoteGroupPort;
import org.apache.nifi.util.CharacterFilterUtils;
import org.apache.nifi.util.StringUtils;
import org.apache.nifi.xml.processing.ProcessingException;
import org.apache.nifi.xml.processing.parsers.StandardDocumentProvider;
import org.apache.nifi.xml.processing.transform.StandardTransformProvider;
import org.w3c.dom.DOMException;
import org.w3c.dom.Document;
import org.w3c.dom.Element;
import javax.xml.transform.dom.DOMSource;
import javax.xml.transform.stream.StreamResult;
import java.io.BufferedOutputStream;
import java.io.OutputStream;
import java.util.Map;
import java.util.Optional;
import java.util.concurrent.TimeUnit;
import java.util.stream.Collectors;
/**
* Serializes a Flow Controller as XML to an output stream.
*
* NOT THREAD-SAFE.
*/
public class StandardFlowSerializer implements FlowSerializer<Document> {
private static final String MAX_ENCODING_VERSION = "1.4";
public StandardFlowSerializer() {
}
@Override
public Document transform(final FlowController controller, final ScheduledStateLookup scheduledStateLookup) throws FlowSerializationException {
final PropertyEncryptor encryptor = controller.getEncryptor();
try {
// create a new, empty document
final StandardDocumentProvider documentProvider = new StandardDocumentProvider();
documentProvider.setNamespaceAware(true);
final Document doc = documentProvider.newDocument();
// populate document with controller state
final Element rootNode = doc.createElement("flowController");
rootNode.setAttribute("encoding-version", MAX_ENCODING_VERSION);
doc.appendChild(rootNode);
addTextElement(rootNode, "maxTimerDrivenThreadCount", controller.getMaxTimerDrivenThreadCount());
final Element registriesElement = doc.createElement("registries");
rootNode.appendChild(registriesElement);
addFlowRegistryClients(registriesElement, controller.getFlowManager(), encryptor);
final Element parameterContextsElement = doc.createElement("parameterContexts");
rootNode.appendChild(parameterContextsElement);
addParameterContexts(parameterContextsElement, controller.getFlowManager().getParameterContextManager(), encryptor);
addProcessGroup(rootNode, controller.getFlowManager().getRootGroup(), "rootGroup", scheduledStateLookup, encryptor);
// Add root-level controller services
final Element controllerServicesNode = doc.createElement("controllerServices");
rootNode.appendChild(controllerServicesNode);
for (final ControllerServiceNode serviceNode : controller.getFlowManager().getRootControllerServices()) {
addControllerService(controllerServicesNode, serviceNode, encryptor);
}
final Element reportingTasksNode = doc.createElement("reportingTasks");
rootNode.appendChild(reportingTasksNode);
for (final ReportingTaskNode taskNode : controller.getAllReportingTasks()) {
addReportingTask(reportingTasksNode, taskNode, encryptor);
}
final Element flowAnalysisRulesNode = doc.createElement("flowAnalysisRules");
rootNode.appendChild(flowAnalysisRulesNode);
for (final FlowAnalysisRuleNode flowAnalysisRuleNode : controller.getAllFlowAnalysisRules()) {
addFlowAnalysisRule(flowAnalysisRulesNode, flowAnalysisRuleNode, encryptor);
}
final Element parameterProvidersNode = doc.createElement("parameterProviders");
rootNode.appendChild(parameterProvidersNode);
for (final ParameterProviderNode providerNode : controller.getFlowManager().getAllParameterProviders()) {
addParameterProvider(parameterProvidersNode, providerNode, encryptor);
}
return doc;
} catch (final ProcessingException | DOMException | IllegalArgumentException e) {
throw new FlowSerializationException(e);
}
}
@Override
public void serialize(final Document flowConfiguration, final OutputStream os) throws FlowSerializationException {
try {
final DOMSource domSource = new DOMSource(flowConfiguration);
final StreamResult streamResult = new StreamResult(new BufferedOutputStream(os));
// configure the transformer and convert the DOM
final StandardTransformProvider transformProvider = new StandardTransformProvider();
transformProvider.setIndent(true);
// transform the document to byte stream
transformProvider.transform(domSource, streamResult);
} catch (final DOMException | IllegalArgumentException | ProcessingException e) {
throw new FlowSerializationException(e);
}
}
private void addParameterContexts(final Element parentElement, final ParameterContextManager parameterContextManager,
final PropertyEncryptor encryptor) {
for (final ParameterContext parameterContext : parameterContextManager.getParameterContexts()) {
final Element parameterContextElement = parentElement.getOwnerDocument().createElement("parameterContext");
parentElement.appendChild(parameterContextElement);
addStringElement(parameterContextElement, "id", parameterContext.getIdentifier());
addStringElement(parameterContextElement, "name", parameterContext.getName());
addStringElement(parameterContextElement, "description", parameterContext.getDescription());
for(final ParameterContext childContext : parameterContext.getInheritedParameterContexts()) {
addStringElement(parameterContextElement, "inheritedParameterContextId", childContext.getIdentifier());
}
if (parameterContext.getParameterProviderConfiguration() != null) {
final ParameterProviderConfiguration parameterProviderConfiguration = parameterContext.getParameterProviderConfiguration();
addStringElement(parameterContextElement, "parameterProviderId", parameterProviderConfiguration.getParameterProviderId());
if (parameterProviderConfiguration.getParameterGroupName() != null) {
addStringElement(parameterContextElement, "parameterGroupName", parameterProviderConfiguration.getParameterGroupName());
}
addStringElement(parameterContextElement, "isSynchronized", String.valueOf(parameterProviderConfiguration.isSynchronized()));
}
for (final Parameter parameter : parameterContext.getParameters().values()) {
addParameter(parameterContextElement, parameter, encryptor);
}
}
}
private void addParameter(final Element parentElement, final Parameter parameter, final PropertyEncryptor encryptor) {
final Element parameterElement = parentElement.getOwnerDocument().createElement("parameter");
parentElement.appendChild(parameterElement);
final ParameterDescriptor descriptor = parameter.getDescriptor();
addStringElement(parameterElement, "name", descriptor.getName());
addStringElement(parameterElement, "description", descriptor.getDescription());
addStringElement(parameterElement, "sensitive", String.valueOf(descriptor.isSensitive()));
addStringElement(parameterElement, "provided", String.valueOf(parameter.isProvided()));
if (parameter.getValue() != null) {
if (descriptor.isSensitive()) {
final String parameterValue = parameter.getValue();
addStringElement(parameterElement, "value", parameterValue == null ? null : ENC_PREFIX + encryptor.encrypt(parameterValue) + ENC_SUFFIX);
} else {
addStringElement(parameterElement, "value", parameter.getValue());
}
}
}
private void addFlowRegistryClients(final Element parentElement, final FlowManager flowManager, final PropertyEncryptor encryptor) {
for (final String registryId : flowManager.getAllFlowRegistryClients().stream().map(FlowRegistryClientNode::getIdentifier).collect(Collectors.toSet())) {
final FlowRegistryClientNode flowRegistry = flowManager.getFlowRegistryClient(registryId);
final Element registryElement = parentElement.getOwnerDocument().createElement("flowRegistry");
parentElement.appendChild(registryElement);
addTextElement(registryElement, "id", flowRegistry.getIdentifier());
addTextElement(registryElement, "name", flowRegistry.getName());
addTextElement(registryElement, "description", flowRegistry.getDescription());
addTextElement(registryElement, "class", flowRegistry.getCanonicalClassName());
addBundle(registryElement, flowRegistry.getBundleCoordinate());
addConfiguration(registryElement, flowRegistry.getRawPropertyValues(), flowRegistry.getAnnotationData(), encryptor);
}
}
private void addStringElement(final Element parentElement, final String elementName, final String value) {
final Element childElement = parentElement.getOwnerDocument().createElement(elementName);
childElement.setTextContent(CharacterFilterUtils.filterInvalidXmlCharacters(value));
parentElement.appendChild(childElement);
}
private void addSize(final Element parentElement, final Size size) {
final Element element = parentElement.getOwnerDocument().createElement("size");
element.setAttribute("width", String.valueOf(size.getWidth()));
element.setAttribute("height", String.valueOf(size.getHeight()));
parentElement.appendChild(element);
}
private void addPosition(final Element parentElement, final Position position) {
addPosition(parentElement, position, "position");
}
private void addPosition(final Element parentElement, final Position position, final String elementName) {
final Element element = parentElement.getOwnerDocument().createElement(elementName);
element.setAttribute("x", String.valueOf(position.getX()));
element.setAttribute("y", String.valueOf(position.getY()));
parentElement.appendChild(element);
}
private void addProcessGroup(final Element parentElement, final ProcessGroup group, final String elementName, final ScheduledStateLookup scheduledStateLookup,
final PropertyEncryptor encryptor) {
final Document doc = parentElement.getOwnerDocument();
final Element element = doc.createElement(elementName);
parentElement.appendChild(element);
addTextElement(element, "id", group.getIdentifier());
addTextElement(element, "versionedComponentId", group.getVersionedComponentId());
addTextElement(element, "name", group.getName());
addPosition(element, group.getPosition());
addTextElement(element, "comment", group.getComments());
addTextElement(element, "flowfileConcurrency", group.getFlowFileConcurrency().name());
addTextElement(element, "flowfileOutboundPolicy", group.getFlowFileOutboundPolicy().name());
addTextElement(element, "defaultFlowFileExpiration", group.getDefaultFlowFileExpiration());
addTextElement(element, "defaultBackPressureObjectThreshold", group.getDefaultBackPressureObjectThreshold());
addTextElement(element, "defaultBackPressureDataSizeThreshold", group.getDefaultBackPressureDataSizeThreshold());
addTextElement(element, "logFileSuffix", group.getLogFileSuffix());
final VersionControlInformation versionControlInfo = group.getVersionControlInformation();
if (versionControlInfo != null) {
final Element versionControlInfoElement = doc.createElement("versionControlInformation");
addTextElement(versionControlInfoElement, "registryId", versionControlInfo.getRegistryIdentifier());
addTextElement(versionControlInfoElement, "bucketId", versionControlInfo.getBucketIdentifier());
addTextElement(versionControlInfoElement, "bucketName", versionControlInfo.getBucketName());
addTextElement(versionControlInfoElement, "flowId", versionControlInfo.getFlowIdentifier());
addTextElement(versionControlInfoElement, "flowName", versionControlInfo.getFlowName());
addTextElement(versionControlInfoElement, "flowDescription", versionControlInfo.getFlowDescription());
addTextElement(versionControlInfoElement, "version", versionControlInfo.getVersion());
addTextElement(versionControlInfoElement, "storageLocation", versionControlInfo.getStorageLocation());
element.appendChild(versionControlInfoElement);
}
for (final ProcessorNode processor : group.getProcessors()) {
addProcessor(element, processor, scheduledStateLookup, encryptor);
}
for (final Port port : group.getInputPorts()) {
if (port instanceof PublicPort) {
addPublicPort(element, (PublicPort) port, "inputPort", scheduledStateLookup);
} else {
addPort(element, port, "inputPort", scheduledStateLookup);
}
}
for (final Port port : group.getOutputPorts()) {
if (port instanceof PublicPort) {
addPublicPort(element, (PublicPort) port, "outputPort", scheduledStateLookup);
} else {
addPort(element, port, "outputPort", scheduledStateLookup);
}
}
for (final Label label : group.getLabels()) {
addLabel(element, label);
}
for (final Funnel funnel : group.getFunnels()) {
addFunnel(element, funnel);
}
for (final ProcessGroup childGroup : group.getProcessGroups()) {
addProcessGroup(element, childGroup, "processGroup", scheduledStateLookup, encryptor);
}
for (final RemoteProcessGroup remoteRef : group.getRemoteProcessGroups()) {
addRemoteProcessGroup(element, remoteRef, scheduledStateLookup, encryptor);
}
for (final Connection connection : group.getConnections()) {
addConnection(element, connection);
}
for (final ControllerServiceNode service : group.getControllerServices(false)) {
addControllerService(element, service, encryptor);
}
final ParameterContext parameterContext = group.getParameterContext();
if (parameterContext != null) {
addTextElement(element, "parameterContextId", parameterContext.getIdentifier());
}
}
private static void addBundle(final Element parentElement, final BundleCoordinate coordinate) {
// group
final Element groupElement = parentElement.getOwnerDocument().createElement("group");
groupElement.setTextContent(CharacterFilterUtils.filterInvalidXmlCharacters(coordinate.getGroup()));
// artifact
final Element artifactElement = parentElement.getOwnerDocument().createElement("artifact");
artifactElement.setTextContent(CharacterFilterUtils.filterInvalidXmlCharacters(coordinate.getId()));
// version
final Element versionElement = parentElement.getOwnerDocument().createElement("version");
versionElement.setTextContent(CharacterFilterUtils.filterInvalidXmlCharacters(coordinate.getVersion()));
// bundle
final Element bundleElement = parentElement.getOwnerDocument().createElement("bundle");
bundleElement.appendChild(groupElement);
bundleElement.appendChild(artifactElement);
bundleElement.appendChild(versionElement);
parentElement.appendChild(bundleElement);
}
private void addStyle(final Element parentElement, final Map<String, String> style) {
final Element element = parentElement.getOwnerDocument().createElement("styles");
for (final Map.Entry<String, String> entry : style.entrySet()) {
final Element styleElement = parentElement.getOwnerDocument().createElement("style");
styleElement.setAttribute("name", entry.getKey());
styleElement.setTextContent(entry.getValue());
element.appendChild(styleElement);
}
parentElement.appendChild(element);
}
private void addLabel(final Element parentElement, final Label label) {
final Document doc = parentElement.getOwnerDocument();
final Element element = doc.createElement("label");
parentElement.appendChild(element);
addTextElement(element, "id", label.getIdentifier());
addTextElement(element, "versionedComponentId", label.getVersionedComponentId());
addTextElement(element, "zIndex", label.getZIndex());
addPosition(element, label.getPosition());
addSize(element, label.getSize());
addStyle(element, label.getStyle());
addTextElement(element, "value", label.getValue());
parentElement.appendChild(element);
}
private void addFunnel(final Element parentElement, final Funnel funnel) {
final Document doc = parentElement.getOwnerDocument();
final Element element = doc.createElement("funnel");
parentElement.appendChild(element);
addTextElement(element, "id", funnel.getIdentifier());
addTextElement(element, "versionedComponentId", funnel.getVersionedComponentId());
addPosition(element, funnel.getPosition());
}
private void addRemoteProcessGroup(final Element parentElement, final RemoteProcessGroup remoteRef, final ScheduledStateLookup scheduledStateLookup,
final PropertyEncryptor encryptor) {
final Document doc = parentElement.getOwnerDocument();
final Element element = doc.createElement("remoteProcessGroup");
parentElement.appendChild(element);
addTextElement(element, "id", remoteRef.getIdentifier());
addTextElement(element, "versionedComponentId", remoteRef.getVersionedComponentId());
addTextElement(element, "name", remoteRef.getName());
addPosition(element, remoteRef.getPosition());
addTextElement(element, "comment", remoteRef.getComments());
addTextElement(element, "url", remoteRef.getTargetUri());
addTextElement(element, "urls", remoteRef.getTargetUris());
addTextElement(element, "timeout", remoteRef.getCommunicationsTimeout());
addTextElement(element, "yieldPeriod", remoteRef.getYieldDuration());
addTextElement(element, "transmitting", String.valueOf(remoteRef.isTransmitting()));
addTextElement(element, "transportProtocol", remoteRef.getTransportProtocol().name());
addTextElement(element, "proxyHost", remoteRef.getProxyHost());
if (remoteRef.getProxyPort() != null) {
addTextElement(element, "proxyPort", remoteRef.getProxyPort());
}
addTextElement(element, "proxyUser", remoteRef.getProxyUser());
if (!StringUtils.isEmpty(remoteRef.getProxyPassword())) {
final String value = ENC_PREFIX + encryptor.encrypt(remoteRef.getProxyPassword()) + ENC_SUFFIX;
addTextElement(element, "proxyPassword", value);
}
if (remoteRef.getNetworkInterface() != null) {
addTextElement(element, "networkInterface", remoteRef.getNetworkInterface());
}
for (final RemoteGroupPort port : remoteRef.getInputPorts()) {
if (port.hasIncomingConnection()) {
addRemoteGroupPort(element, port, "inputPort", scheduledStateLookup);
}
}
for (final RemoteGroupPort port : remoteRef.getOutputPorts()) {
if (!port.getConnections().isEmpty()) {
addRemoteGroupPort(element, port, "outputPort", scheduledStateLookup);
}
}
parentElement.appendChild(element);
}
private void addRemoteGroupPort(final Element parentElement, final RemoteGroupPort port, final String elementName, final ScheduledStateLookup scheduledStateLookup) {
final Document doc = parentElement.getOwnerDocument();
final Element element = doc.createElement(elementName);
parentElement.appendChild(element);
addTextElement(element, "id", port.getIdentifier());
addTextElement(element, "versionedComponentId", port.getVersionedComponentId());
addTextElement(element, "name", port.getName());
addPosition(element, port.getPosition());
addTextElement(element, "comments", port.getComments());
addTextElement(element, "scheduledState", scheduledStateLookup.getScheduledState(port).name());
addTextElement(element, "targetId", port.getTargetIdentifier());
addTextElement(element, "maxConcurrentTasks", port.getMaxConcurrentTasks());
addTextElement(element, "useCompression", String.valueOf(port.isUseCompression()));
final Integer batchCount = port.getBatchCount();
if (batchCount != null && batchCount > 0) {
addTextElement(element, "batchCount", batchCount);
}
final String batchSize = port.getBatchSize();
if (batchSize != null && batchSize.length() > 0) {
addTextElement(element, "batchSize", batchSize);
}
final String batchDuration = port.getBatchDuration();
if (batchDuration != null && batchDuration.length() > 0) {
addTextElement(element, "batchDuration", batchDuration);
}
parentElement.appendChild(element);
}
private void addPort(final Element parentElement, final Port port, final String elementName, final ScheduledStateLookup scheduledStateLookup) {
final Document doc = parentElement.getOwnerDocument();
final Element element = doc.createElement(elementName);
parentElement.appendChild(element);
addTextElement(element, "id", port.getIdentifier());
addTextElement(element, "versionedComponentId", port.getVersionedComponentId());
addTextElement(element, "name", port.getName());
addPosition(element, port.getPosition());
addTextElement(element, "comments", port.getComments());
addTextElement(element, "scheduledState", scheduledStateLookup.getScheduledState(port).name());
parentElement.appendChild(element);
}
private void addPublicPort(final Element parentElement, final PublicPort port, final String elementName, final ScheduledStateLookup scheduledStateLookup) {
final Document doc = parentElement.getOwnerDocument();
final Element element = doc.createElement(elementName);
parentElement.appendChild(element);
addTextElement(element, "id", port.getIdentifier());
addTextElement(element, "versionedComponentId", port.getVersionedComponentId());
addTextElement(element, "name", port.getName());
addPosition(element, port.getPosition());
addTextElement(element, "comments", port.getComments());
addTextElement(element, "scheduledState", scheduledStateLookup.getScheduledState(port).name());
addTextElement(element, "maxConcurrentTasks", String.valueOf(port.getMaxConcurrentTasks()));
addTextElement(element, "allowRemoteAccess", Boolean.TRUE.toString());
for (final String user : port.getUserAccessControl()) {
addTextElement(element, "userAccessControl", user);
}
for (final String group : port.getGroupAccessControl()) {
addTextElement(element, "groupAccessControl", group);
}
parentElement.appendChild(element);
}
private void addProcessor(final Element parentElement, final ProcessorNode processor, final ScheduledStateLookup scheduledStateLookup,
final PropertyEncryptor encryptor) {
final Document doc = parentElement.getOwnerDocument();
final Element element = doc.createElement("processor");
parentElement.appendChild(element);
addTextElement(element, "id", processor.getIdentifier());
addTextElement(element, "versionedComponentId", processor.getVersionedComponentId());
addTextElement(element, "name", processor.getName());
addPosition(element, processor.getPosition());
addStyle(element, processor.getStyle());
addTextElement(element, "comment", processor.getComments());
addTextElement(element, "class", processor.getCanonicalClassName());
addBundle(element, processor.getBundleCoordinate());
addTextElement(element, "maxConcurrentTasks", processor.getMaxConcurrentTasks());
addTextElement(element, "schedulingPeriod", processor.getSchedulingPeriod());
addTextElement(element, "penalizationPeriod", processor.getPenalizationPeriod());
addTextElement(element, "yieldPeriod", processor.getYieldPeriod());
addTextElement(element, "bulletinLevel", processor.getBulletinLevel().toString());
addTextElement(element, "lossTolerant", String.valueOf(processor.isLossTolerant()));
addTextElement(element, "scheduledState", scheduledStateLookup.getScheduledState(processor).name());
addTextElement(element, "schedulingStrategy", processor.getSchedulingStrategy().name());
addTextElement(element, "executionNode", processor.getExecutionNode().name());
addTextElement(element, "runDurationNanos", processor.getRunDuration(TimeUnit.NANOSECONDS));
addTextElement(element, "retryCount", processor.getRetryCount());
addTextElement(element, "backoffMechanism", processor.getBackoffMechanism().name());
addTextElement(element, "maxBackoffPeriod", processor.getMaxBackoffPeriod());
for (final String relationship : processor.getRetriedRelationships()) {
addTextElement(element, "retriedRelationship", relationship);
}
addConfiguration(element, processor.getRawPropertyValues(), processor.getAnnotationData(), encryptor);
for (final Relationship rel : processor.getAutoTerminatedRelationships()) {
addTextElement(element, "autoTerminatedRelationship", rel.getName());
}
}
private static void addConfiguration(final Element element, final Map<PropertyDescriptor, String> properties, final String annotationData, final PropertyEncryptor encryptor) {
final Document doc = element.getOwnerDocument();
for (final Map.Entry<PropertyDescriptor, String> entry : properties.entrySet()) {
final PropertyDescriptor descriptor = entry.getKey();
String value = entry.getValue();
if (value == null) {
value = descriptor.getDefaultValue();
}
if (value != null && descriptor.isSensitive()) {
value = ENC_PREFIX + encryptor.encrypt(value) + ENC_SUFFIX;
}
final Element propElement = doc.createElement("property");
addTextElement(propElement, "name", descriptor.getName());
if (value != null) {
addTextElement(propElement, "value", value);
}
element.appendChild(propElement);
}
if (annotationData != null) {
addTextElement(element, "annotationData", annotationData);
}
}
private void addConnection(final Element parentElement, final Connection connection) {
final Document doc = parentElement.getOwnerDocument();
final Element element = doc.createElement("connection");
parentElement.appendChild(element);
addTextElement(element, "id", connection.getIdentifier());
addTextElement(element, "versionedComponentId", connection.getVersionedComponentId());
addTextElement(element, "name", connection.getName());
final Element bendPointsElement = doc.createElement("bendPoints");
element.appendChild(bendPointsElement);
for (final Position bendPoint : connection.getBendPoints()) {
addPosition(bendPointsElement, bendPoint, "bendPoint");
}
addTextElement(element, "labelIndex", connection.getLabelIndex());
addTextElement(element, "zIndex", connection.getZIndex());
final String sourceId = connection.getSource().getIdentifier();
final ConnectableType sourceType = connection.getSource().getConnectableType();
final String sourceGroupId;
if (sourceType == ConnectableType.REMOTE_OUTPUT_PORT) {
sourceGroupId = ((RemoteGroupPort) connection.getSource()).getRemoteProcessGroup().getIdentifier();
} else {
sourceGroupId = connection.getSource().getProcessGroup().getIdentifier();
}
final ConnectableType destinationType = connection.getDestination().getConnectableType();
final String destinationId = connection.getDestination().getIdentifier();
final String destinationGroupId;
if (destinationType == ConnectableType.REMOTE_INPUT_PORT) {
destinationGroupId = ((RemoteGroupPort) connection.getDestination()).getRemoteProcessGroup().getIdentifier();
} else {
destinationGroupId = connection.getDestination().getProcessGroup().getIdentifier();
}
addTextElement(element, "sourceId", sourceId);
addTextElement(element, "sourceGroupId", sourceGroupId);
addTextElement(element, "sourceType", sourceType.toString());
addTextElement(element, "destinationId", destinationId);
addTextElement(element, "destinationGroupId", destinationGroupId);
addTextElement(element, "destinationType", destinationType.toString());
for (final Relationship relationship : connection.getRelationships()) {
addTextElement(element, "relationship", relationship.getName());
}
addTextElement(element, "maxWorkQueueSize", connection.getFlowFileQueue().getBackPressureObjectThreshold());
addTextElement(element, "maxWorkQueueDataSize", connection.getFlowFileQueue().getBackPressureDataSizeThreshold());
addTextElement(element, "flowFileExpiration", connection.getFlowFileQueue().getFlowFileExpiration());
for (final FlowFilePrioritizer comparator : connection.getFlowFileQueue().getPriorities()) {
final String className = comparator.getClass().getCanonicalName();
addTextElement(element, "queuePrioritizerClass", className);
}
addTextElement(element, "loadBalanceStrategy", connection.getFlowFileQueue().getLoadBalanceStrategy().name());
addTextElement(element, "partitioningAttribute", connection.getFlowFileQueue().getPartitioningAttribute());
addTextElement(element, "loadBalanceCompression", connection.getFlowFileQueue().getLoadBalanceCompression().name());
parentElement.appendChild(element);
}
public void addControllerService(final Element element, final ControllerServiceNode serviceNode, final PropertyEncryptor encryptor) {
final Element serviceElement = element.getOwnerDocument().createElement("controllerService");
addTextElement(serviceElement, "id", serviceNode.getIdentifier());
addTextElement(serviceElement, "versionedComponentId", serviceNode.getVersionedComponentId());
addTextElement(serviceElement, "name", serviceNode.getName());
addTextElement(serviceElement, "comment", serviceNode.getComments());
addTextElement(serviceElement, "bulletinLevel", serviceNode.getBulletinLevel().toString());
addTextElement(serviceElement, "class", serviceNode.getCanonicalClassName());
addBundle(serviceElement, serviceNode.getBundleCoordinate());
final ControllerServiceState state = serviceNode.getState();
final boolean enabled = (state == ControllerServiceState.ENABLED || state == ControllerServiceState.ENABLING);
addTextElement(serviceElement, "enabled", String.valueOf(enabled));
addConfiguration(serviceElement, serviceNode.getRawPropertyValues(), serviceNode.getAnnotationData(), encryptor);
element.appendChild(serviceElement);
}
public static void addReportingTask(final Element element, final ReportingTaskNode taskNode, final PropertyEncryptor encryptor) {
final Element taskElement = element.getOwnerDocument().createElement("reportingTask");
addTextElement(taskElement, "id", taskNode.getIdentifier());
addTextElement(taskElement, "name", taskNode.getName());
addTextElement(taskElement, "comment", taskNode.getComments());
addTextElement(taskElement, "class", taskNode.getCanonicalClassName());
addBundle(taskElement, taskNode.getBundleCoordinate());
addTextElement(taskElement, "schedulingPeriod", taskNode.getSchedulingPeriod());
addTextElement(taskElement, "scheduledState", taskNode.getScheduledState().name());
addTextElement(taskElement, "schedulingStrategy", taskNode.getSchedulingStrategy().name());
addConfiguration(taskElement, taskNode.getRawPropertyValues(), taskNode.getAnnotationData(), encryptor);
element.appendChild(taskElement);
}
private void addFlowAnalysisRule(Element flowAnalysisRulesNode, FlowAnalysisRuleNode flowAnalysisRuleNode, final PropertyEncryptor encryptor) {
final Element taskElement = flowAnalysisRulesNode.getOwnerDocument().createElement("flowAnalysisRule");
addTextElement(taskElement, "id", flowAnalysisRuleNode.getIdentifier());
addTextElement(taskElement, "name", flowAnalysisRuleNode.getName());
addTextElement(taskElement, "comment", flowAnalysisRuleNode.getComments());
addTextElement(taskElement, "class", flowAnalysisRuleNode.getCanonicalClassName());
addBundle(taskElement, flowAnalysisRuleNode.getBundleCoordinate());
addTextElement(taskElement, "enforcementPolicy", flowAnalysisRuleNode.getEnforcementPolicy().name());
addTextElement(taskElement, "state", flowAnalysisRuleNode.getState().name());
addConfiguration(taskElement, flowAnalysisRuleNode.getRawPropertyValues(), flowAnalysisRuleNode.getAnnotationData(), encryptor);
flowAnalysisRulesNode.appendChild(taskElement);
}
public static void addParameterProvider(final Element element, final ParameterProviderNode providerNode, final PropertyEncryptor encryptor) {
final Element taskElement = element.getOwnerDocument().createElement("parameterProvider");
addTextElement(taskElement, "id", providerNode.getIdentifier());
addTextElement(taskElement, "name", providerNode.getName());
addTextElement(taskElement, "comment", providerNode.getComments());
addTextElement(taskElement, "class", providerNode.getCanonicalClassName());
addBundle(taskElement, providerNode.getBundleCoordinate());
addConfiguration(taskElement, providerNode.getRawPropertyValues(), providerNode.getAnnotationData(), encryptor);
element.appendChild(taskElement);
}
private static void addTextElement(final Element element, final String name, final long value) {
addTextElement(element, name, String.valueOf(value));
}
private static void addTextElement(final Element element, final String name, final String value) {
final Document doc = element.getOwnerDocument();
final Element toAdd = doc.createElement(name);
toAdd.setTextContent(CharacterFilterUtils.filterInvalidXmlCharacters(value)); // value should already be filtered, but just in case ensure there are no invalid xml characters
element.appendChild(toAdd);
}
private static void addTextElement(final Element element, final String name, final Optional<String> value) {
if (!value.isPresent()) {
return;
}
final Document doc = element.getOwnerDocument();
final Element toAdd = doc.createElement(name);
toAdd.setTextContent(CharacterFilterUtils.filterInvalidXmlCharacters(value.get())); // value should already be filtered, but just in case ensure there are no invalid xml characters
element.appendChild(toAdd);
}
}

64
nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/serialization/StandardFlowSynchronizer.java
View File

@ -1,64 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.nifi.controller.serialization;
import org.apache.nifi.cluster.protocol.DataFlow;
import org.apache.nifi.controller.FlowController;
import org.apache.nifi.controller.MissingBundleException;
import org.apache.nifi.controller.UninheritableFlowException;
import org.apache.nifi.controller.XmlFlowSynchronizer;
import org.apache.nifi.groups.BundleUpdateStrategy;
import org.apache.nifi.services.FlowService;
public class StandardFlowSynchronizer implements FlowSynchronizer {
private final XmlFlowSynchronizer xmlFlowSynchronizer;
private final VersionedFlowSynchronizer versionedFlowSynchronizer;
public StandardFlowSynchronizer(final XmlFlowSynchronizer xmlFlowSynchronizer, final VersionedFlowSynchronizer versionedFlowSynchronizer) {
this.xmlFlowSynchronizer = xmlFlowSynchronizer;
this.versionedFlowSynchronizer = versionedFlowSynchronizer;
}
@Override
public void sync(final FlowController controller, final DataFlow dataFlow, final FlowService flowService, final BundleUpdateStrategy bundleUpdateStrategy)
throws FlowSerializationException, UninheritableFlowException, FlowSynchronizationException, MissingBundleException {
final FlowSynchronizer synchronizer = isXml(dataFlow) ? xmlFlowSynchronizer : versionedFlowSynchronizer;
synchronizer.sync(controller, dataFlow, flowService, bundleUpdateStrategy);
}
public static boolean isFlowEmpty(final DataFlow dataFlow) {
if (dataFlow == null || dataFlow.getFlow() == null || dataFlow.getFlow().length == 0) {
return true;
}
if (isXml(dataFlow)) {
return XmlFlowSynchronizer.isFlowEmpty(dataFlow.getFlowDocument());
} else {
return VersionedFlowSynchronizer.isFlowEmpty(dataFlow);
}
}
private static boolean isXml(final DataFlow dataFlow) {
if (dataFlow == null || dataFlow.getFlow() == null || dataFlow.getFlow().length == 0) {
return true;
}
return dataFlow.isXml();
}
}

90
nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/serialization/VersionedFlowSynchronizer.java
View File

@ -66,6 +66,7 @@ import org.apache.nifi.controller.inheritance.BundleCompatibilityCheck;
import org.apache.nifi.controller.inheritance.ConnectionMissingCheck;
import org.apache.nifi.controller.inheritance.FlowInheritability;
import org.apache.nifi.controller.inheritance.FlowInheritabilityCheck;
import org.apache.nifi.controller.inheritance.MissingComponentsCheck;
import org.apache.nifi.controller.reporting.ReportingTaskInstantiationException;
import org.apache.nifi.controller.service.ControllerServiceNode;
import org.apache.nifi.encrypt.EncryptionException;
@ -163,56 +164,78 @@ public class VersionedFlowSynchronizer implements FlowSynchronizer {
final boolean flowAlreadySynchronized = controller.isFlowSynchronized();
logger.info("Synchronizing FlowController with proposed flow: Controller Already Synchronized = {}", flowAlreadySynchronized);
final DataFlow existingDataFlow = getExistingDataFlow(controller);
boolean existingFlowEmpty = isFlowEmpty(existingDataFlow);
// If bundle update strategy is configured to allow for compatible bundles, update any components to use compatible bundles if
// the exact bundle does not exist.
if (bundleUpdateStrategy == BundleUpdateStrategy.USE_SPECIFIED_OR_COMPATIBLE_OR_GHOST) {
if (!existingFlowEmpty && bundleUpdateStrategy == BundleUpdateStrategy.USE_SPECIFIED_OR_COMPATIBLE_OR_GHOST) {
mapCompatibleBundles(proposedFlow, controller.getExtensionManager());
}
// serialize controller state to bytes
final DataFlow existingDataFlow = getExistingDataFlow(controller);
checkFlowInheritability(existingDataFlow, proposedFlow, controller, bundleUpdateStrategy);
final FlowComparison flowComparison = compareFlows(existingDataFlow, proposedFlow, controller.getEncryptor());
final Set<FlowDifference> flowDifferences = flowComparison.getDifferences();
if (flowDifferences.isEmpty()) {
logger.debug("No differences between current flow and proposed flow. Will not create backup of existing flow.");
} else if (isExistingFlowEmpty(controller)) {
logger.debug("Currently loaded dataflow is empty. Will not create backup of existing flow.");
} else {
backupExistingFlow();
logger.debug("Checking missing component inheritability");
final FlowInheritabilityCheck missingComponentsCheck = new MissingComponentsCheck();
final FlowInheritability componentInheritability = missingComponentsCheck.checkInheritability(existingDataFlow, proposedFlow, controller);
if (!componentInheritability.isInheritable()) {
throw new UninheritableFlowException("Proposed Flow is not inheritable by the flow controller because of differences in missing components: " + componentInheritability.getExplanation());
}
final AffectedComponentSet affectedComponents = determineAffectedComponents(flowComparison, controller);
final AffectedComponentSet activeSet = affectedComponents.toActiveSet();
FlowComparison flowComparison = null;
AffectedComponentSet affectedComponents = null;
AffectedComponentSet activeSet = null;
// Stop the active components, and then wait for all components to be stopped.
logger.info("In order to inherit proposed dataflow, will stop any components that will be affected by the update");
if (logger.isDebugEnabled()) {
logger.debug("Will stop the following components:");
logger.debug(activeSet.toString());
final String differencesToString = flowDifferences.stream()
.map(FlowDifference::toString)
.collect(Collectors.joining("\n"));
logger.debug("This Active Set was determined from the following Flow Differences:\n{}", differencesToString);
if (!existingFlowEmpty) {
flowComparison = compareFlows(existingDataFlow, proposedFlow, controller.getEncryptor());
final Set<FlowDifference> flowDifferences = flowComparison.getDifferences();
if (flowDifferences.isEmpty()) {
logger.debug("No differences between current flow and proposed flow. Will not create backup of existing flow.");
} else if (isExistingFlowEmpty(controller)) {
logger.debug("Currently loaded dataflow is empty. Will not create backup of existing flow.");
} else {
backupExistingFlow();
}
affectedComponents = determineAffectedComponents(flowComparison, controller);
activeSet = affectedComponents.toActiveSet();
// Stop the active components, and then wait for all components to be stopped.
logger.info("In order to inherit proposed dataflow, will stop any components that will be affected by the update");
if (logger.isDebugEnabled()) {
logger.debug("Will stop the following components:");
logger.debug(activeSet.toString());
final String differencesToString = flowDifferences.stream()
.map(FlowDifference::toString)
.collect(Collectors.joining("\n"));
logger.debug("This Active Set was determined from the following Flow Differences:\n{}",
differencesToString);
}
activeSet.stop();
}
activeSet.stop();
try {
// Ensure that the proposed flow doesn't remove any Connections for which there is currently data queued
verifyNoConnectionsWithDataRemoved(existingDataFlow, proposedFlow, controller, flowComparison);
if (!existingFlowEmpty) {
verifyNoConnectionsWithDataRemoved(existingDataFlow, proposedFlow, controller, flowComparison);
}
synchronizeFlow(controller, existingDataFlow, proposedFlow, affectedComponents);
} finally {
// We have to call toExistingSet() here because some of the components that existed in the active set may no longer exist,
// so attempting to start them will fail.
final AffectedComponentSet startable = activeSet.toExistingSet().toStartableSet();
final ComponentSetFilter runningComponentFilter = new RunningComponentSetFilter(proposedFlow.getVersionedDataflow());
final ComponentSetFilter stoppedComponentFilter = runningComponentFilter.reverse();
startable.removeComponents(stoppedComponentFilter);
startable.start();
if (!existingFlowEmpty) {
final AffectedComponentSet startable = activeSet.toExistingSet().toStartableSet();
final ComponentSetFilter runningComponentFilter = new RunningComponentSetFilter(proposedFlow.getVersionedDataflow());
final ComponentSetFilter stoppedComponentFilter = runningComponentFilter.reverse();
startable.removeComponents(stoppedComponentFilter);
startable.start();
}
}
final long millis = System.currentTimeMillis() - start;
@ -661,6 +684,11 @@ public class VersionedFlowSynchronizer implements FlowSynchronizer {
final VersionedDataflow dataflow,
final AffectedComponentSet affectedComponentSet
) throws FlowAnalysisRuleInstantiationException {
// Guard state in order to be able to read flow.json from before adding the flow analysis rules
if (dataflow.getFlowAnalysisRules() == null) {
return;
}
for (final VersionedFlowAnalysisRule versionedFlowAnalysisRule : dataflow.getFlowAnalysisRules()) {
final FlowAnalysisRuleNode existing = controller.getFlowAnalysisRuleNode(versionedFlowAnalysisRule.getInstanceIdentifier());
if (existing == null) {
@ -1201,13 +1229,15 @@ public class VersionedFlowSynchronizer implements FlowSynchronizer {
private DataFlow getExistingDataFlow(final FlowController controller) {
final FlowManager flowManager = controller.getFlowManager();
final ProcessGroup root = flowManager.getRootGroup();
// Determine missing components
final Set<String> missingComponents = new HashSet<>();
flowManager.getAllControllerServices().stream().filter(ComponentNode::isExtensionMissing).forEach(cs -> missingComponents.add(cs.getIdentifier()));
flowManager.getAllReportingTasks().stream().filter(ComponentNode::isExtensionMissing).forEach(r -> missingComponents.add(r.getIdentifier()));
flowManager.getAllParameterProviders().stream().filter(ComponentNode::isExtensionMissing).forEach(r -> missingComponents.add(r.getIdentifier()));
flowManager.findAllProcessors(AbstractComponentNode::isExtensionMissing).forEach(p -> missingComponents.add(p.getIdentifier()));
flowManager.getAllFlowRegistryClients().stream().filter(ComponentNode::isExtensionMissing).forEach(c -> missingComponents.add(c.getIdentifier()));
root.findAllProcessors().stream().filter(AbstractComponentNode::isExtensionMissing).forEach(p -> missingComponents.add(p.getIdentifier()));
logger.trace("Exporting snippets from controller");
final byte[] existingSnippets = controller.getSnippetManager().export();

188
nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/service/ControllerServiceLoader.java
View File

@ -1,188 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.nifi.controller.service;
import org.apache.nifi.bundle.BundleCoordinate;
import org.apache.nifi.components.PropertyDescriptor;
import org.apache.nifi.controller.FlowController;
import org.apache.nifi.controller.serialization.FlowEncodingVersion;
import org.apache.nifi.controller.serialization.FlowFromDOMFactory;
import org.apache.nifi.encrypt.PropertyEncryptor;
import org.apache.nifi.groups.ProcessGroup;
import org.apache.nifi.logging.LogLevel;
import org.apache.nifi.util.BundleUtils;
import org.apache.nifi.web.api.dto.BundleDTO;
import org.apache.nifi.web.api.dto.ControllerServiceDTO;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.w3c.dom.Element;
import java.nio.charset.StandardCharsets;
import java.util.Collection;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.UUID;
import java.util.stream.Collectors;
public class ControllerServiceLoader {
private static final Logger logger = LoggerFactory.getLogger(ControllerServiceLoader.class);
public static Map<ControllerServiceNode, Element> loadControllerServices(final List<Element> serviceElements, final FlowController controller,
final ProcessGroup parentGroup, final PropertyEncryptor encryptor, final FlowEncodingVersion encodingVersion) {
final Map<ControllerServiceNode, Element> nodeMap = new HashMap<>();
for (final Element serviceElement : serviceElements) {
final ControllerServiceNode serviceNode = createControllerService(controller, serviceElement, encryptor, encodingVersion);
if (parentGroup == null) {
controller.getFlowManager().addRootControllerService(serviceNode);
} else {
parentGroup.addControllerService(serviceNode);
}
// We need to clone the node because it will be used in a separate thread below, and
// Element is not thread-safe.
nodeMap.put(serviceNode, (Element) serviceElement.cloneNode(true));
}
for (final Map.Entry<ControllerServiceNode, Element> entry : nodeMap.entrySet()) {
configureControllerService(entry.getKey(), entry.getValue(), encryptor, encodingVersion);
}
return nodeMap;
}
public static void enableControllerServices(final Map<ControllerServiceNode, Element> nodeMap, final FlowController controller,
final PropertyEncryptor encryptor, final boolean autoResumeState, final FlowEncodingVersion encodingVersion) {
// Start services
if (autoResumeState) {
final Set<ControllerServiceNode> nodesToEnable = new HashSet<>();
for (final ControllerServiceNode node : nodeMap.keySet()) {
final Element controllerServiceElement = nodeMap.get(node);
final ControllerServiceDTO dto;
synchronized (controllerServiceElement.getOwnerDocument()) {
dto = FlowFromDOMFactory.getControllerService(controllerServiceElement, encryptor, encodingVersion);
}
final ControllerServiceState state = ControllerServiceState.valueOf(dto.getState());
if (state == ControllerServiceState.ENABLED) {
nodesToEnable.add(node);
logger.debug("Will enable Controller Service {}", node);
} else {
logger.debug("Will not enable Controller Service {} because its state is set to {}", node, state);
}
}
enableControllerServices(nodesToEnable, controller, autoResumeState);
} else {
logger.debug("Will not enable the following Controller Services because 'auto-resume state' flag is false: {}", nodeMap.keySet());
}
}
public static void enableControllerServices(final Collection<ControllerServiceNode> nodesToEnable, final FlowController controller, final boolean autoResumeState) {
// Start services
if (autoResumeState) {
logger.debug("Enabling Controller Services {}", nodesToEnable);
nodesToEnable.forEach(ControllerServiceNode::performValidation); // validate services before attempting to enable them
controller.getControllerServiceProvider().enableControllerServices(nodesToEnable);
} else {
logger.debug("Will not enable the following Controller Services because 'auto-resume state' flag is false: {}", nodesToEnable);
}
}
public static ControllerServiceNode cloneControllerService(final FlowController flowController, final ControllerServiceNode controllerService) {
// create a new id for the clone seeded from the original id so that it is consistent in a cluster
final UUID id = UUID.nameUUIDFromBytes(controllerService.getIdentifier().getBytes(StandardCharsets.UTF_8));
final ControllerServiceNode clone = flowController.getFlowManager().createControllerService(controllerService.getCanonicalClassName(), id.toString(),
controllerService.getBundleCoordinate(), Collections.emptySet(), false, true, null);
clone.setName(controllerService.getName());
clone.setComments(controllerService.getComments());
clone.setBulletinLevel(controllerService.getBulletinLevel());
if (controllerService.getProperties() != null) {
Map<String,String> properties = new HashMap<>();
for (Map.Entry<PropertyDescriptor, String> propEntry : controllerService.getRawPropertyValues().entrySet()) {
properties.put(propEntry.getKey().getName(), propEntry.getValue());
}
clone.setProperties(properties);
}
return clone;
}
private static ControllerServiceNode createControllerService(final FlowController flowController, final Element controllerServiceElement, final PropertyEncryptor encryptor,
final FlowEncodingVersion encodingVersion) {
final ControllerServiceDTO dto = FlowFromDOMFactory.getControllerService(controllerServiceElement, encryptor, encodingVersion);
BundleCoordinate coordinate;
try {
coordinate = BundleUtils.getCompatibleBundle(flowController.getExtensionManager(), dto.getType(), dto.getBundle());
} catch (final IllegalStateException e) {
final BundleDTO bundleDTO = dto.getBundle();
if (bundleDTO == null) {
coordinate = BundleCoordinate.UNKNOWN_COORDINATE;
} else {
coordinate = new BundleCoordinate(bundleDTO.getGroup(), bundleDTO.getArtifact(), bundleDTO.getVersion());
}
}
final ControllerServiceNode node = flowController.getFlowManager().createControllerService(dto.getType(), dto.getId(), coordinate, Collections.emptySet(), false, true, null);
node.setName(dto.getName());
node.setComments(dto.getComments());
if (dto.getBulletinLevel() != null) {
node.setBulletinLevel(LogLevel.valueOf(dto.getBulletinLevel()));
} else {
// this situation exists for backward compatibility with nifi 1.16 and earlier where controller services do not have bulletinLevels set in flow.xml/flow.json
// and bulletinLevels are at the WARN level by default
node.setBulletinLevel(LogLevel.WARN);
}
node.setVersionedComponentId(dto.getVersionedComponentId());
return node;
}
private static void configureControllerService(final ControllerServiceNode node, final Element controllerServiceElement, final PropertyEncryptor encryptor,
final FlowEncodingVersion encodingVersion) {
final ControllerServiceDTO dto = FlowFromDOMFactory.getControllerService(controllerServiceElement, encryptor, encodingVersion);
node.pauseValidationTrigger();
try {
node.setAnnotationData(dto.getAnnotationData());
final Set<String> sensitiveDynamicPropertyNames = getSensitiveDynamicPropertyNames(dto.getSensitiveDynamicPropertyNames(), node);
node.setProperties(dto.getProperties(), false, sensitiveDynamicPropertyNames);
} finally {
node.resumeValidationTrigger();
}
}
private static Set<String> getSensitiveDynamicPropertyNames(final Set<String> parsedSensitivePropertyNames, final ControllerServiceNode controllerServiceNode) {
final Set<String> sensitivePropertyNames = parsedSensitivePropertyNames == null ? Collections.emptySet() : parsedSensitivePropertyNames;
return sensitivePropertyNames.stream().filter(
propertyName -> {
final PropertyDescriptor propertyDescriptor = controllerServiceNode.getPropertyDescriptor(propertyName);
return propertyDescriptor.isDynamic();
}
).collect(Collectors.toSet());
}
}

40
nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/fingerprint/FingerprintException.java
View File

@ -1,40 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.nifi.fingerprint;
/**
*/
public class FingerprintException extends RuntimeException {
private static final long serialVersionUID = 189234798327894327L;
public FingerprintException(Throwable cause) {
super(cause);
}
public FingerprintException(String message, Throwable cause) {
super(message, cause);
}
public FingerprintException(String message) {
super(message);
}
public FingerprintException() {
}
}

1050
nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/fingerprint/FingerprintFactory.java
View File

File diff suppressed because it is too large Load Diff

6
nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/persistence/FlowConfigurationArchiveManager.java
View File

@ -78,11 +78,7 @@ public class FlowConfigurationArchiveManager {
final String archiveDirVal = properties.getFlowConfigurationArchiveDir();
final Path archiveDirectory;
if (archiveDirVal == null || archiveDirVal.trim().isEmpty()) {
File persistenceFile = properties.getFlowConfigurationJsonFile();
if (persistenceFile == null) {
persistenceFile = properties.getFlowConfigurationFile();
}
File persistenceFile = properties.getFlowConfigurationFile();
archiveDirectory = (archiveDirVal == null || archiveDirVal.equals(""))
? persistenceFile.toPath().getParent().resolve("archive") : new File(archiveDirVal).toPath();
} else {

71
nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/persistence/StandardFlowConfigurationDAO.java
View File

@ -16,18 +16,23 @@
*/
package org.apache.nifi.persistence;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.nio.file.Files;
import java.util.zip.GZIPInputStream;
import java.util.zip.GZIPOutputStream;
import org.apache.nifi.cluster.protocol.DataFlow;
import org.apache.nifi.controller.FlowController;
import org.apache.nifi.controller.FlowSerializationStrategy;
import org.apache.nifi.controller.MissingBundleException;
import org.apache.nifi.controller.UninheritableFlowException;
import org.apache.nifi.controller.XmlFlowSynchronizer;
import org.apache.nifi.controller.serialization.FlowSerializationException;
import org.apache.nifi.controller.serialization.FlowSerializer;
import org.apache.nifi.controller.serialization.FlowSynchronizationException;
import org.apache.nifi.controller.serialization.FlowSynchronizer;
import org.apache.nifi.controller.serialization.StandardFlowSerializer;
import org.apache.nifi.controller.serialization.StandardFlowSynchronizer;
import org.apache.nifi.controller.serialization.VersionedFlowSerializer;
import org.apache.nifi.controller.serialization.VersionedFlowSynchronizer;
import org.apache.nifi.groups.BundleUpdateStrategy;
@ -38,22 +43,11 @@ import org.apache.nifi.util.file.FileUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.nio.file.Files;
import java.util.zip.GZIPInputStream;
import java.util.zip.GZIPOutputStream;
public final class StandardFlowConfigurationDAO implements FlowConfigurationDAO {
private static final Logger LOG = LoggerFactory.getLogger(StandardFlowConfigurationDAO.class);
private static final String CLUSTER_FLOW_SERIALIZATION_FORMAT = "nifi.cluster.flow.serialization.format";
private static final String FLOW_SERIALIZATION_FORMAT_XML = "XML";
private static final String FLOW_SERIALIZATION_FORMAT_JSON = "JSON";
private final File xmlFile;
private final File jsonFile;
private final FlowConfigurationArchiveManager archiveManager;
private final NiFiProperties nifiProperties;
@ -61,16 +55,12 @@ public final class StandardFlowConfigurationDAO implements FlowConfigurationDAO
private volatile boolean jsonFileExists;
private final String clusterFlowSerializationFormat;
private final FlowSerializationStrategy serializationStrategy;
public StandardFlowConfigurationDAO(final NiFiProperties nifiProperties, final ExtensionManager extensionManager,
final FlowSerializationStrategy serializationStrategy) throws IOException {
public StandardFlowConfigurationDAO(final NiFiProperties nifiProperties, final ExtensionManager extensionManager) throws IOException {
this.nifiProperties = nifiProperties;
this.clusterFlowSerializationFormat = nifiProperties.getProperty(CLUSTER_FLOW_SERIALIZATION_FORMAT);
this.serializationStrategy = serializationStrategy;
this.clusterFlowSerializationFormat = nifiProperties.getProperty(CLUSTER_FLOW_SERIALIZATION_FORMAT, FLOW_SERIALIZATION_FORMAT_JSON);
xmlFile = nifiProperties.getFlowConfigurationFile();
jsonFile = nifiProperties.getFlowConfigurationJsonFile();
jsonFile = nifiProperties.getFlowConfigurationFile();
jsonFileExists = jsonFile.length() > 0L;
@ -97,13 +87,10 @@ public final class StandardFlowConfigurationDAO implements FlowConfigurationDAO
public synchronized void load(final FlowController controller, final DataFlow dataFlow, final FlowService flowService, final BundleUpdateStrategy bundleUpdateStrategy)
throws IOException, FlowSerializationException, FlowSynchronizationException, UninheritableFlowException, MissingBundleException {
final VersionedFlowSynchronizer versionedFlowSynchronizer = new VersionedFlowSynchronizer(extensionManager, nifiProperties.getFlowConfigurationJsonFile(), archiveManager);
final XmlFlowSynchronizer xmlFlowSynchronizer = new XmlFlowSynchronizer(nifiProperties, extensionManager);
final FlowSynchronizer standardFlowSynchronizer = new StandardFlowSynchronizer(xmlFlowSynchronizer, versionedFlowSynchronizer);
final FlowSynchronizer standardFlowSynchronizer = new VersionedFlowSynchronizer(extensionManager, nifiProperties.getFlowConfigurationFile(), archiveManager);
controller.synchronize(standardFlowSynchronizer, dataFlow, flowService, bundleUpdateStrategy);
if (StandardFlowSynchronizer.isFlowEmpty(dataFlow)) {
if (VersionedFlowSynchronizer.isFlowEmpty(dataFlow)) {
// If the dataflow is empty, we want to save it. We do this because when we start up a brand new cluster with no
// dataflow, we need to ensure that the flow is consistent across all nodes in the cluster and that upon restart
// of NiFi, the root group ID does not change. However, we don't always want to save it, because if the flow is
@ -116,14 +103,7 @@ public final class StandardFlowConfigurationDAO implements FlowConfigurationDAO
}
private File getReadableFile() {
if (jsonFileExists) {
return jsonFile;
}
if (xmlFile.length() > 0) {
return xmlFile;
}
return null;
return jsonFileExists ? jsonFile : null;
}
@Override
@ -166,10 +146,10 @@ public final class StandardFlowConfigurationDAO implements FlowConfigurationDAO
try {
// Serialize based on the serialization format configured for cluster communications. If not configured, use JSON.
final FlowSerializer<?> serializer;
if (FLOW_SERIALIZATION_FORMAT_XML.equalsIgnoreCase(clusterFlowSerializationFormat)) {
serializer = new StandardFlowSerializer();
} else {
if (FLOW_SERIALIZATION_FORMAT_JSON.equalsIgnoreCase(clusterFlowSerializationFormat)) {
serializer = new VersionedFlowSerializer(extensionManager);
} else {
throw new IllegalArgumentException("Unknown serialization format");
}
controller.serialize(serializer, os);
@ -184,12 +164,8 @@ public final class StandardFlowConfigurationDAO implements FlowConfigurationDAO
throw new NullPointerException();
}
if (serializationStrategy.writesJson()) {
saveJson(controller, archive);
}
if (serializationStrategy.writesXml()) {
saveXml(controller, archive);
}
saveJson(controller, archive);
}
private void saveJson(final FlowController controller, final boolean archive) throws IOException {
@ -198,11 +174,6 @@ public final class StandardFlowConfigurationDAO implements FlowConfigurationDAO
jsonFileExists = true;
}
private void saveXml(final FlowController controller, final boolean archive) throws IOException {
final FlowSerializer<?> serializer = new StandardFlowSerializer();
saveFlow(controller, serializer, xmlFile, archive);
}
private void saveFlow(final FlowController controller, final FlowSerializer<?> serializer, final File file, final boolean archive) throws IOException {
final File tempFile = new File(file.getParentFile(), file.getName() + ".temp.gz");

4
nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/spring/StandardFlowServiceFactoryBean.java
View File

@ -20,7 +20,6 @@ import org.apache.nifi.authorization.Authorizer;
import org.apache.nifi.cluster.coordination.ClusterCoordinator;
import org.apache.nifi.cluster.protocol.impl.NodeProtocolSenderListener;
import org.apache.nifi.controller.FlowController;
import org.apache.nifi.controller.FlowSerializationStrategy;
import org.apache.nifi.controller.StandardFlowService;
import org.apache.nifi.encrypt.PropertyEncryptor;
import org.apache.nifi.services.FlowService;
@ -64,8 +63,7 @@ public class StandardFlowServiceFactoryBean implements FactoryBean, ApplicationC
flowController,
properties,
revisionManager,
authorizer,
FlowSerializationStrategy.WRITE_XML_AND_JSON);
authorizer);
}
}

194
nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/util/FlowParser.java
View File

@ -19,39 +19,16 @@ package org.apache.nifi.util;
import org.apache.commons.io.IOUtils;
import org.apache.nifi.cluster.protocol.StandardDataFlow;
import org.apache.nifi.controller.flow.VersionedDataflow;
import org.apache.nifi.controller.serialization.FlowFromDOMFactory;
import org.apache.nifi.flow.VersionedPort;
import org.apache.nifi.flow.VersionedProcessGroup;
import org.apache.nifi.util.file.FileUtils;
import org.apache.nifi.web.api.dto.PortDTO;
import org.apache.nifi.web.api.dto.PositionDTO;
import org.apache.nifi.xml.processing.ProcessingException;
import org.apache.nifi.xml.processing.parsers.StandardDocumentProvider;
import org.apache.nifi.xml.processing.transform.StandardTransformProvider;
import org.apache.nifi.xml.processing.transform.TransformProvider;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.w3c.dom.Document;
import org.w3c.dom.Element;
import org.w3c.dom.Node;
import org.w3c.dom.NodeList;
import org.xml.sax.ErrorHandler;
import org.xml.sax.SAXException;
import javax.xml.XMLConstants;
import javax.xml.parsers.ParserConfigurationException;
import javax.xml.transform.Result;
import javax.xml.transform.Source;
import javax.xml.transform.dom.DOMSource;
import javax.xml.transform.stream.StreamResult;
import javax.xml.validation.Schema;
import javax.xml.validation.SchemaFactory;
import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.File;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.StandardOpenOption;
@ -60,7 +37,6 @@ import java.util.Collections;
import java.util.List;
import java.util.Set;
import java.util.zip.GZIPInputStream;
import java.util.zip.GZIPOutputStream;
/**
* Parses a flow and returns the root group id and root group ports.
@ -69,16 +45,6 @@ public class FlowParser {
private static final Logger logger = LoggerFactory.getLogger(FlowParser.class);
private static final String FLOW_XSD = "/FlowConfiguration.xsd";
private static final byte XML_FIRST_CHARACTER = '<';
private final Schema flowSchema;
public FlowParser() throws SAXException {
SchemaFactory schemaFactory = SchemaFactory.newInstance(XMLConstants.W3C_XML_SCHEMA_NS_URI);
flowSchema = schemaFactory.newSchema(FlowParser.class.getResource(FLOW_XSD));
}
/**
* Extracts the root group id from the flow configuration file provided in nifi.properties, and extracts
* the root group input ports and output ports, and their access controls.
@ -112,61 +78,13 @@ public class FlowParser {
return null;
}
if (isXml(flowBytes)) {
return parseXml(flowBytes);
}
return parseJson(flowBytes);
} catch (final SAXException | ParserConfigurationException | IOException ex) {
} catch (final IOException ex) {
logger.error("Unable to parse flow {}", flowPath.toAbsolutePath(), ex);
return null;
}
}
private boolean isXml(final byte[] contents) {
if (contents == null || contents.length == 0) {
return false;
}
final byte firstByte = contents[0];
return firstByte == XML_FIRST_CHARACTER;
}
private FlowInfo parseXml(final byte[] flowBytes) throws ParserConfigurationException, IOException, SAXException {
// create validating document builder
final ErrorHandler errorHandler = new LoggingXmlParserErrorHandler("Flow Configuration", logger);
final StandardDocumentProvider documentProvider = new StandardDocumentProvider();
documentProvider.setSchema(flowSchema);
documentProvider.setNamespaceAware(true);
documentProvider.setErrorHandler(errorHandler);
// parse the flow
final Document document = documentProvider.parse(new ByteArrayInputStream(flowBytes));
// extract the root group id
final Element rootElement = document.getDocumentElement();
final Element rootGroupElement = (Element) rootElement.getElementsByTagName("rootGroup").item(0);
if (rootGroupElement == null) {
logger.warn("rootGroup element not found in Flow Configuration file");
return null;
}
final Element rootGroupIdElement = (Element) rootGroupElement.getElementsByTagName("id").item(0);
if (rootGroupIdElement == null) {
logger.warn("id element not found under rootGroup in Flow Configuration file");
return null;
}
final String rootGroupId = rootGroupIdElement.getTextContent();
final List<PortDTO> ports = new ArrayList<>();
ports.addAll(getPorts(rootGroupElement, "inputPort"));
ports.addAll(getPorts(rootGroupElement, "outputPort"));
return new FlowInfo(rootGroupId, ports);
}
private FlowInfo parseJson(final byte[] flowBytes) {
final StandardDataFlow standardDataFlow = new StandardDataFlow(flowBytes, new byte[0], null, Collections.emptySet());
final VersionedDataflow dataflow = standardDataFlow.getVersionedDataflow();
@ -207,114 +125,4 @@ public class FlowParser {
dto.setType(port.getType().name());
return dto;
}
/**
* Generates a {@link Document} from the flow configuration file provided
*/
public Document parseDocument(final File flowConfigurationFile) {
if (flowConfigurationFile == null) {
logger.debug("Flow Configuration file was null");
return null;
}
// if the flow doesn't exist or is 0 bytes, then return null
final Path flowPath = flowConfigurationFile.toPath();
try {
if (!Files.exists(flowPath) || Files.size(flowPath) == 0) {
logger.warn("Flow Configuration does not exist or was empty");
return null;
}
} catch (IOException e) {
logger.error("An error occurred determining the size of the Flow Configuration file");
return null;
}
// otherwise create the appropriate input streams to read the file
try (final InputStream in = Files.newInputStream(flowPath, StandardOpenOption.READ);
final InputStream gzipIn = new GZIPInputStream(in)) {
final byte[] flowBytes = IOUtils.toByteArray(gzipIn);
if (flowBytes == null || flowBytes.length == 0) {
logger.warn("Could not extract root group id because Flow Configuration File was empty");
return null;
}
// create validating document builder
final StandardDocumentProvider documentProvider = new StandardDocumentProvider();
documentProvider.setErrorHandler(new LoggingXmlParserErrorHandler("Flow Configuration", logger));
documentProvider.setSchema(flowSchema);
documentProvider.setNamespaceAware(true);
return documentProvider.parse(new ByteArrayInputStream(flowBytes));
} catch (final ProcessingException | IOException e) {
logger.error("Unable to parse flow {}", flowPath.toAbsolutePath(), e);
return null;
}
}
/**
* Gets the ports that are direct children of the given element.
*
* @param element the element containing ports
* @param type the type of port to find (inputPort or outputPort)
* @return a list of PortDTOs representing the found ports
*/
private List<PortDTO> getPorts(final Element element, final String type) {
final List<PortDTO> ports = new ArrayList<>();
// add input ports
final List<Element> portNodeList = getChildrenByTagName(element, type);
for (final Element portElement : portNodeList) {
final PortDTO portDTO = FlowFromDOMFactory.getPort(portElement);
portDTO.setType(type);
ports.add(portDTO);
}
return ports;
}
/**
* Writes a given XML Flow out to the specified path.
*
* @param flowDocument flowDocument of the associated XML content to write to disk
* @param flowXmlPath path on disk to write the flow
* @throws IOException if there are issues in accessing the target destination for the flow
*/
public void writeFlow(final Document flowDocument, final Path flowXmlPath) throws IOException {
final ByteArrayOutputStream outputStream = new ByteArrayOutputStream();
final Source xmlSource = new DOMSource(flowDocument);
final Result outputTarget = new StreamResult(outputStream);
final TransformProvider transformProvider = new StandardTransformProvider();
transformProvider.transform(xmlSource, outputTarget);
final InputStream is = new ByteArrayInputStream(outputStream.toByteArray());
try (final OutputStream output = Files.newOutputStream(flowXmlPath, StandardOpenOption.WRITE, StandardOpenOption.CREATE);
final OutputStream gzipOut = new GZIPOutputStream(output)) {
FileUtils.copy(is, gzipOut);
}
}
/**
* Finds child elements with the given tagName.
*
* @param element the parent element
* @param tagName the child element name to find
* @return a list of matching child elements
*/
public static List<Element> getChildrenByTagName(final Element element, final String tagName) {
final List<Element> matches = new ArrayList<>();
final NodeList nodeList = element.getChildNodes();
for (int i = 0; i < nodeList.getLength(); i++) {
final Node node = nodeList.item(i);
if (!(node instanceof Element)) {
continue;
}
final Element child = (Element) nodeList.item(i);
if (child.getNodeName().equals(tagName)) {
matches.add(child);
}
}
return matches;
}
}

43
nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/util/LoggingXmlParserErrorHandler.java
View File

@ -1,43 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.nifi.util;
import org.slf4j.Logger;
import org.xml.sax.SAXParseException;
import org.xml.sax.helpers.DefaultHandler;
/**
* ErrorHandler implementation for Logging XML schema validation errors
*/
public class LoggingXmlParserErrorHandler extends DefaultHandler {
private final Logger logger;
private final String xmlDocTitle;
private static final String MESSAGE_FORMAT = "Schema validation %s parsing %s at line %d, col %d: %s";
public LoggingXmlParserErrorHandler(String xmlDocTitle, Logger logger) {
this.logger = logger;
this.xmlDocTitle = xmlDocTitle;
}
@Override
public void error(final SAXParseException err) throws SAXParseException {
String message = String.format(MESSAGE_FORMAT, "error", xmlDocTitle, err.getLineNumber(),
err.getColumnNumber(), err.getMessage());
logger.warn(message);
}
}

607
nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/resources/FlowConfiguration.xsd
View File

@ -1,607 +0,0 @@
<?xml version="1.0" encoding="UTF-8"?>
<!--
Licensed to the Apache Software Foundation (ASF) under one or more
contributor license agreements. See the NOTICE file distributed with
this work for additional information regarding copyright ownership.
The ASF licenses this file to You under the Apache License, Version 2.0
(the "License"); you may not use this file except in compliance with
the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
-->
<xs:schema xmlns:xs="http://www.w3.org/2001/XMLSchema" version="1.0">
<xs:element name="flowController" type="FlowControllerType" />
<xs:complexType name="FlowControllerType">
<xs:sequence>
<xs:choice>
<xs:element name="maxThreadCount" type="xs:positiveInteger"/>
<xs:sequence>
<xs:element name="maxTimerDrivenThreadCount" type="xs:positiveInteger"/>
<xs:element name="maxEventDrivenThreadCount" type="xs:positiveInteger"/>
</xs:sequence>
</xs:choice>
<xs:element name="registries" type="RegistriesType" minOccurs="0" maxOccurs="1" />
<xs:element name="parameterContexts" type="ParameterContextsType" minOccurs="0" maxOccurs="1" />
<!-- Groupings of Processors/Ports -->
<xs:element name="rootGroup" type="RootProcessGroupType" />
<!-- This exists for backward compatibility between NiFi 1.x and NiFi 0.x. Any Controller Service that is listed
here is assigned to the root group -->
<xs:element name="controllerServices" type="ControllerServicesType" minOccurs="0" maxOccurs="1" />
<xs:element name="reportingTasks" type="ReportingTasksType" minOccurs="0" maxOccurs="1" />
<xs:element name="flowAnalysisRules" type="FlowAnalysisRulesType" minOccurs="0" maxOccurs="1" />
<xs:element name="parameterProviders" type="ParameterProvidersType" minOccurs="0" maxOccurs="1" />
</xs:sequence>
<xs:attribute name="encoding-version" type="xs:string"/>
</xs:complexType>
<xs:complexType name="RegistriesType">
<xs:sequence>
<xs:element name="flowRegistry" type="FlowRegistryType" minOccurs="0" maxOccurs="unbounded" />
</xs:sequence>
</xs:complexType>
<xs:complexType name="FlowRegistryType">
<xs:sequence>
<xs:element name="id" type="NonEmptyStringType" />
<xs:element name="name" type="NonEmptyStringType" />
<xs:element name="url" type="xs:string" />
<xs:element name="description" type="xs:string" />
<!-- "class" is the actual Java class that performs the type of processing desired-->
<xs:element name="class" type="NonEmptyStringType"/>
<xs:element name="bundle" type="BundleType" />
<xs:element name="property" type="PropertyType" minOccurs="0" maxOccurs="unbounded"/>
<xs:element name="annotationData" type="xs:string" minOccurs="0" maxOccurs="1" />
</xs:sequence>
</xs:complexType>
<xs:complexType name="ParameterContextsType">
<xs:sequence>
<xs:element name="parameterContext" type="ParameterContextType" minOccurs="0" maxOccurs="unbounded" />
</xs:sequence>
</xs:complexType>
<xs:complexType name="ParameterContextType">
<xs:sequence>
<xs:element name="id" type="NonEmptyStringType" />
<xs:element name="name" type="NonEmptyStringType" />
<xs:element name="description" type="xs:string" minOccurs="0" maxOccurs="1" />
<xs:element name="inheritedParameterContextId" type="xs:string" minOccurs="0" maxOccurs="unbounded" />
<xs:element name="parameterProviderId" type="xs:string" minOccurs="0" maxOccurs="1" />
<xs:element name="parameterGroupName" type="xs:string" minOccurs="0" maxOccurs="1" />
<xs:element name="isSynchronized" type="xs:boolean" minOccurs="0" maxOccurs="1" />
<xs:element name="parameter" type="ParameterType" minOccurs="0" maxOccurs="unbounded" />
</xs:sequence>
</xs:complexType>
<xs:complexType name="ParameterType">
<xs:sequence>
<xs:element name="name" type="NonEmptyStringType" />
<xs:element name="description" type="xs:string" />
<xs:element name="sensitive" type="xs:boolean" />
<xs:element name="provided" type="xs:boolean" />
<xs:element name="value" type="xs:string" minOccurs="0" maxOccurs="1" />
</xs:sequence>
</xs:complexType>
<!-- the processor "id" is a key that should be valid within each flowController-->
<xs:complexType name="ProcessorType">
<xs:sequence>
<!-- The "id" is a name that is tied to the ability for a FlowFile to be recovered in the event
of unexpected process termination or execution failure. The "id" should rarely change and if it
must then the database directory should be deleted to be safe.-->
<xs:element name="id" type="NonEmptyStringType"/>
<xs:element name="versionedComponentId" type="NonEmptyStringType" minOccurs="0" maxOccurs="1" />
<!-- The "name" is a nicely displayable description of the processor's duty-->
<xs:element name="name" type="NonEmptyStringType"/>
<xs:element name="position" type="PositionType" />
<xs:element name="styles" type="Styles" />
<!-- The "comment" section allows users to store important context information
for this processor. This information may display in GUI or in logs or
whatever presentation the application might have.-->
<xs:element name="comment" type="xs:string" minOccurs="0" maxOccurs="1"/>
<!-- "class" is the actual Java class that performs the type of processing desired-->
<xs:element name="class" type="NonEmptyStringType"/>
<xs:element name="bundle" type="BundleType" />
<!-- the number of concurrent tasks for this configured
processor that can be executed at any one time. This value can be 0
IFF schedulingStrategy is EVENT_DRIVEN -->
<xs:element name="maxConcurrentTasks" type="xs:nonNegativeInteger"/>
<xs:element name="schedulingPeriod" type="NonEmptyStringType"/>
<xs:element name="penalizationPeriod" type="TimePeriod" />
<xs:element name="yieldPeriod" type="TimePeriod" />
<xs:element name="bulletinLevel" type="LogLevel" />
<!-- whether or not this processor is loss-tolerant -->
<xs:element name="lossTolerant" type="xs:boolean" />
<xs:element name="scheduledState" type="ScheduledState" />
<!-- "isolated" is deprecated.
Was used to determine whether or not this processor runs on the primary node
in a clustered environment. If 'true', then the processor runs on only the
primary node. If 'false' or not present, then the processor runs on all the
nodes. -->
<xs:element name="isolated" type="xs:boolean" maxOccurs="1" minOccurs="0" />
<xs:element name="schedulingStrategy" type="SchedulingStrategy" minOccurs="0" maxOccurs="1" />
<xs:element name="executionNode" type="ExecutionNode" minOccurs="0" maxOccurs="1" />
<xs:element name="runDurationNanos" type="xs:long" minOccurs="0" maxOccurs="1" />
<!-- properties that must be valid for the processor to execute.
The valid required properties can be read by looking at this processor's Javadocs-->
<xs:element name="property" type="PropertyType" minOccurs="0" maxOccurs="unbounded"/>
<!-- Annotation data used for more advanced configuration -->
<xs:element name="annotationData" type="xs:string" minOccurs="0" maxOccurs="1" />
<!-- Indicates that a relationship with the given name can be auto-terminated -->
<xs:element name="autoTerminatedRelationship" type="xs:string" minOccurs="0" maxOccurs="unbounded" />
<xs:element name="retryCount" type="xs:nonNegativeInteger" minOccurs="0" maxOccurs="1" />
<xs:element name="retriedRelationships" type="xs:string" minOccurs="0" maxOccurs="unbounded" />
<xs:element name="backoffMechanism" type="BackoffMechanism" minOccurs="0" maxOccurs="1" />
<xs:element name="maxBackoffPeriod" type="NonEmptyStringType" minOccurs="0" maxOccurs="1" />
</xs:sequence>
</xs:complexType>
<!-- The "name" should be a key within each processor-->
<xs:complexType name="PropertyType">
<xs:sequence>
<xs:element name="name" type="NonEmptyStringType"/>
<!-- Not present if the value has not been set. -->
<xs:element name="value" type="xs:string" minOccurs="0" maxOccurs="1"/>
</xs:sequence>
</xs:complexType>
<xs:simpleType name="NonEmptyStringType">
<xs:restriction base="xs:string">
<xs:minLength value="1"/>
</xs:restriction>
</xs:simpleType>
<xs:simpleType name="ScheduledState">
<xs:restriction base="xs:string">
<xs:enumeration value="DISABLED"></xs:enumeration>
<xs:enumeration value="RUNNING"></xs:enumeration>
<xs:enumeration value="STOPPED"></xs:enumeration>
</xs:restriction>
</xs:simpleType>
<xs:simpleType name="FlowAnalysisRuleEnforcementPolicy">
<xs:restriction base="xs:string">
<xs:enumeration value="ENFORCE"></xs:enumeration>
<xs:enumeration value="WARN"></xs:enumeration>
</xs:restriction>
</xs:simpleType>
<xs:simpleType name="FlowAnalysisState">
<xs:restriction base="xs:string">
<xs:enumeration value="ENABLED"></xs:enumeration>
<xs:enumeration value="DISABLED"></xs:enumeration>
</xs:restriction>
</xs:simpleType>
<xs:complexType name="ProcessGroupType">
<xs:sequence>
<xs:element name="id" type="NonEmptyStringType" />
<xs:element name="versionedComponentId" type="NonEmptyStringType" minOccurs="0" maxOccurs="1" />
<xs:element name="name" type="NonEmptyStringType" />
<xs:element name="position" type="PositionType" />
<xs:element name="comment" type="xs:string" />
<xs:element name="flowfileConcurrency" type="FlowFileConcurrencyType" minOccurs="0" maxOccurs="1" />
<xs:element name="flowfileOutboundPolicy" type="FlowFileOutboundPolicyType" minOccurs="0" maxOccurs="1" />
<xs:element name="defaultFlowFileExpiration" type="xs:string" minOccurs="0" maxOccurs="1" />
<xs:element name="defaultBackPressureObjectThreshold" type="xs:long" minOccurs="0" maxOccurs="1" />
<xs:element name="defaultBackPressureDataSizeThreshold" type="xs:string" minOccurs="0" maxOccurs="1" />
<xs:element name="versionControlInformation" type="VersionControlInformation" minOccurs="0" maxOccurs="1" />
<xs:element name="logFileSuffix" type="xs:string" minOccurs="0" maxOccurs="1"/>
<!-- Each "processor" defines the actual dataflow work horses that make dataflow happen-->
<xs:element name="processor" type="ProcessorType" minOccurs="0" maxOccurs="unbounded"/>
<xs:element name="inputPort" type="PublicPortType" minOccurs="0" maxOccurs="unbounded"/>
<xs:element name="outputPort" type="PublicPortType" minOccurs="0" maxOccurs="unbounded"/>
<xs:element name="label" type="LabelType" minOccurs="0" maxOccurs="unbounded" />
<xs:element name="funnel" type="FunnelType" minOccurs="0" maxOccurs="unbounded" />
<xs:element name="processGroup" type="ProcessGroupType" minOccurs="0" maxOccurs="unbounded" />
<xs:element name="remoteProcessGroup" type="RemoteProcessGroupType" minOccurs="0" maxOccurs="unbounded" />
<xs:element name="connection" type="ConnectionType" minOccurs="0" maxOccurs="unbounded" />
<xs:element name="controllerService" type="ControllerServiceType" minOccurs="0" maxOccurs="unbounded" />
<xs:element name="parameterContextId" type="xs:string" minOccurs="0" maxOccurs="1" />
</xs:sequence>
</xs:complexType>
<xs:simpleType name="FlowFileConcurrencyType">
<xs:restriction base="xs:string">
<xs:enumeration value="SINGLE_FLOWFILE_PER_NODE" />
<xs:enumeration value="SINGLE_BATCH_PER_NODE" />
<xs:enumeration value="UNBOUNDED" />
</xs:restriction>
</xs:simpleType>
<xs:simpleType name="FlowFileOutboundPolicyType">
<xs:restriction base="xs:string">
<xs:enumeration value="STREAM_WHEN_AVAILABLE" />
<xs:enumeration value="BATCH_OUTPUT" />
</xs:restriction>
</xs:simpleType>
<xs:complexType name="VersionControlInformation">
<xs:sequence>
<xs:element name="registryId" type="NonEmptyStringType" />
<xs:element name="bucketId" type="NonEmptyStringType" />
<xs:element name="bucketName" type="NonEmptyStringType" />
<xs:element name="flowId" type="NonEmptyStringType" />
<xs:element name="flowName" type="NonEmptyStringType" />
<xs:element name="flowDescription" type="xs:string" />
<xs:element name="version" type="NonEmptyStringType" />
</xs:sequence>
</xs:complexType>
<!-- Same as ProcessGroupType except:
- RootProcessGroupType doesn't have versionControlInformation -->
<xs:complexType name="RootProcessGroupType">
<xs:sequence>
<xs:element name="id" type="NonEmptyStringType" />
<xs:element name="versionedComponentId" type="NonEmptyStringType" minOccurs="0" maxOccurs="1" />
<xs:element name="name" type="NonEmptyStringType" />
<xs:element name="position" type="PositionType" />
<xs:element name="comment" type="xs:string" />
<xs:element name="flowfileConcurrency" type="FlowFileConcurrencyType" minOccurs="0" maxOccurs="1" />
<xs:element name="flowfileOutboundPolicy" type="FlowFileOutboundPolicyType" minOccurs="0" maxOccurs="1" />
<xs:element name="defaultFlowFileExpiration" type="xs:string" minOccurs="0" maxOccurs="1" />
<xs:element name="defaultBackPressureObjectThreshold" type="xs:long" minOccurs="0" maxOccurs="1" />
<xs:element name="defaultBackPressureDataSizeThreshold" type="xs:string" minOccurs="0" maxOccurs="1" />
<xs:element name="logFileSuffix" type="xs:string" minOccurs="0" maxOccurs="1"/>
<!-- Each "processor" defines the actual dataflow work horses that make dataflow happen-->
<xs:element name="processor" type="ProcessorType" minOccurs="0" maxOccurs="unbounded"/>
<xs:element name="inputPort" type="PublicPortType" minOccurs="0" maxOccurs="unbounded"/>
<xs:element name="outputPort" type="PublicPortType" minOccurs="0" maxOccurs="unbounded"/>
<xs:element name="label" type="LabelType" minOccurs="0" maxOccurs="unbounded" />
<xs:element name="funnel" type="FunnelType" minOccurs="0" maxOccurs="unbounded" />
<xs:element name="processGroup" type="ProcessGroupType" minOccurs="0" maxOccurs="unbounded" />
<xs:element name="remoteProcessGroup" type="RemoteProcessGroupType" minOccurs="0" maxOccurs="unbounded" />
<xs:element name="connection" type="ConnectionType" minOccurs="0" maxOccurs="unbounded" />
<xs:element name="controllerService" type="ControllerServiceType" minOccurs="0" maxOccurs="unbounded" />
<xs:element name="parameterContextId" type="xs:string" minOccurs="0" maxOccurs="1" />
</xs:sequence>
</xs:complexType>
<xs:complexType name="FunnelType">
<xs:sequence>
<xs:element name="id" type="NonEmptyStringType" />
<xs:element name="versionedComponentId" type="NonEmptyStringType" minOccurs="0" maxOccurs="1" />
<xs:element name="position" type="PositionType" />
</xs:sequence>
</xs:complexType>
<xs:complexType name="LabelType">
<xs:sequence>
<xs:element name="id" type="NonEmptyStringType" />
<xs:element name="versionedComponentId" type="NonEmptyStringType" minOccurs="0" maxOccurs="1" />
<xs:element name="position" type="PositionType" />
<xs:element name="size" type="SizeType" />
<xs:element name="styles" type="Styles" />
<xs:element name="value" type="xs:string" />
</xs:sequence>
</xs:complexType>
<xs:complexType name="RemoteProcessGroupType">
<xs:sequence>
<xs:element name="id" type="NonEmptyStringType" />
<xs:element name="versionedComponentId" type="NonEmptyStringType" minOccurs="0" maxOccurs="1" />
<xs:element name="name" type="xs:string" />
<xs:element name="position" type="PositionType" />
<xs:element name="comment" type="xs:string" />
<xs:element name="url" type="xs:anyURI" />
<xs:element name="urls" type="xs:string" minOccurs="0" maxOccurs="1" />
<xs:element name="timeout" type="xs:string" />
<xs:element name="yieldPeriod" type="TimePeriod" minOccurs="0" maxOccurs="1" />
<xs:element name="transmitting" type="xs:boolean" />
<xs:element name="transportProtocol" type="xs:string" minOccurs="0" maxOccurs="1" />
<xs:element name="proxyHost" type="xs:string" minOccurs="0" maxOccurs="1" />
<xs:element name="proxyPort" type="xs:int" minOccurs="0" maxOccurs="1" />
<xs:element name="proxyUser" type="xs:string" minOccurs="0" maxOccurs="1" />
<xs:element name="proxyPassword" type="xs:string" minOccurs="0" maxOccurs="1" />
<xs:element name="networkInterface" type="xs:string" minOccurs="0" maxOccurs="1" />
<!-- The input ports and output ports of the remote group may change without our knowledge; however,
they are persisted here because on a restart of NiFi, we need to have the Input & Output Ports' IDs
and associated names persisted so that we can attempt to connect to these ports -->
<xs:element name="inputPort" type="RemoteGroupPortType" minOccurs="0" maxOccurs="unbounded" />
<xs:element name="outputPort" type="RemoteGroupPortType" minOccurs="0" maxOccurs="unbounded" />
</xs:sequence>
</xs:complexType>
<xs:complexType name="ConnectionType">
<xs:sequence>
<xs:element name="id" type="NonEmptyStringType" />
<xs:element name="versionedComponentId" type="NonEmptyStringType" minOccurs="0" maxOccurs="1" />
<xs:element name="name" type="xs:string" />
<xs:element name="bendPoints" type="BendPointsType" minOccurs="0" maxOccurs="1" />
<xs:element name="labelIndex" type="xs:int" minOccurs="0" maxOccurs="1" />
<xs:element name="zIndex" type="xs:int" minOccurs="0" maxOccurs="1" />
<xs:element name="sourceId" type="NonEmptyStringType" />
<xs:element name="sourceGroupId" type="NonEmptyStringType" />
<xs:element name="sourceType" type="NonEmptyStringType" />
<xs:element name="destinationId" type="NonEmptyStringType" />
<xs:element name="destinationGroupId" type="NonEmptyStringType" />
<xs:element name="destinationType" type="NonEmptyStringType" />
<!-- relationship will be an empty string for Ports. -->
<xs:element name="relationship" type="xs:string" minOccurs="1" maxOccurs="unbounded" />
<!-- "maxWorkQueueSize" is the maximum size this processors work queue should be
before other processors are expected (not required) to stop loading new files onto it.-->
<xs:element name="maxWorkQueueSize" type="xs:nonNegativeInteger"/>
<xs:element name="maxWorkQueueDataSize" type="DataSize" minOccurs="0" maxOccurs="1" />
<!-- "flowFileExpirationMinutes" is the maximum time that a flow file may remain
passing through the flow as configured. If a flow file reaches the
configured maximum age the next processor to see this flow file
will immediately remove it from the flow. A value of zero (the default)
means that no maximum age will be enforced.-->
<xs:element name="flowFileExpiration" type="TimePeriod" minOccurs="0" maxOccurs="1"/>
<!-- "queuePrioritizerClass" are Java classes that can be used to prioritize the work queues for this
processor. The order of the prioritizers is important.-->
<xs:element name="queuePrioritizerClass" type="xs:string" minOccurs="0" maxOccurs="unbounded"/>
<xs:element name="loadBalanceStrategy" type="xs:string" minOccurs="0" maxOccurs="1" />
<xs:element name="partitioningAttribute" type="xs:string" minOccurs="0" maxOccurs="1" />
<xs:element name="loadBalanceCompression" type="xs:string" minOccurs="0" maxOccurs="1" />
</xs:sequence>
</xs:complexType>
<xs:complexType name="PortType">
<xs:sequence>
<xs:element name="id" type="NonEmptyStringType" />
<xs:element name="versionedComponentId" type="NonEmptyStringType" minOccurs="0" maxOccurs="1" />
<xs:element name="name" type="NonEmptyStringType" />
<xs:element name="position" type="PositionType" />
<xs:element name="comments" type="xs:string" />
<xs:element name="scheduledState" type="ScheduledState" />
</xs:sequence>
</xs:complexType>
<xs:complexType name="PublicPortType">
<xs:complexContent>
<xs:extension base="PortType">
<xs:sequence>
<xs:element name="maxConcurrentTasks" type="xs:positiveInteger" minOccurs="0" ></xs:element>
<xs:element name="userAccessControl" type="xs:string" minOccurs="0" maxOccurs="unbounded" />
<xs:element name="groupAccessControl" type="xs:string" minOccurs="0" maxOccurs="unbounded" />
<xs:element name="allowRemoteAccess" type="xs:boolean" minOccurs="0" maxOccurs="1" />
</xs:sequence>
</xs:extension>
</xs:complexContent>
</xs:complexType>
<xs:complexType name="RemoteGroupPortType">
<xs:complexContent>
<xs:extension base="PortType">
<xs:sequence>
<xs:element name="targetId" type="NonEmptyStringType" minOccurs="0" maxOccurs="1" />
<xs:element name="maxConcurrentTasks" type="xs:positiveInteger"></xs:element>
<xs:element name="useCompression" type="xs:boolean"></xs:element>
<xs:element name="batchCount" type="xs:positiveInteger" minOccurs="0" maxOccurs="1" />
<xs:element name="batchSize" type="xs:string" minOccurs="0" maxOccurs="1" />
<xs:element name="batchDuration" type="xs:string" minOccurs="0" maxOccurs="1" />
</xs:sequence>
</xs:extension>
</xs:complexContent>
</xs:complexType>
<xs:complexType name="BundleType">
<xs:sequence>
<xs:element name="group" type="NonEmptyStringType" />
<xs:element name="artifact" type="NonEmptyStringType" />
<xs:element name="version" type="NonEmptyStringType" />
</xs:sequence>
</xs:complexType>
<xs:complexType name="PositionType">
<xs:attribute name="x" type="xs:double" use="required" />
<xs:attribute name="y" type="xs:double" use="required" />
</xs:complexType>
<xs:complexType name="SizeType">
<xs:attribute name="width" type="xs:double" use="required" />
<xs:attribute name="height" type="xs:double" use="required" />
</xs:complexType>
<xs:complexType name="BendPointsType">
<xs:sequence>
<xs:element name="bendPoint" type="PositionType" minOccurs="0" maxOccurs="unbounded" />
</xs:sequence>
</xs:complexType>
<xs:complexType name="Styles">
<xs:sequence>
<xs:element name="style" type="Style" minOccurs="0" maxOccurs="unbounded" />
</xs:sequence>
</xs:complexType>
<xs:complexType name="Style">
<xs:simpleContent>
<xs:extension base="xs:string">
<xs:attribute name="name" use="required"></xs:attribute>
</xs:extension>
</xs:simpleContent>
</xs:complexType>
<xs:simpleType name="TimePeriod">
<xs:restriction base="xs:string">
<xs:pattern value="\d+\s*(ns|nano|nanos|nanosecond|nanoseconds|ms|milli|millis|millisecond|milliseconds|s|sec|secs|second|seconds|m|min|mins|minute|minutes|h|hr|hrs|hour|hours|d|day|days|w|wk|wks|week|weeks)"></xs:pattern>
</xs:restriction>
</xs:simpleType>
<xs:simpleType name="DataSize">
<xs:restriction base="xs:string">
<xs:pattern value="\d+\s*(B|KB|MB|GB|TB|b|kb|mb|gb|tb)" />
</xs:restriction>
</xs:simpleType>
<xs:simpleType name="LogLevel">
<xs:restriction base="xs:string">
<xs:enumeration value="TRACE"></xs:enumeration>
<xs:enumeration value="DEBUG"></xs:enumeration>
<xs:enumeration value="INFO"></xs:enumeration>
<xs:enumeration value="WARN"></xs:enumeration>
<xs:enumeration value="ERROR"></xs:enumeration>
<xs:enumeration value="FATAL"></xs:enumeration>
<xs:enumeration value="NONE"></xs:enumeration>
</xs:restriction>
</xs:simpleType>
<xs:simpleType name="SchedulingStrategy">
<xs:restriction base="xs:string">
<xs:enumeration value="EVENT_DRIVEN"></xs:enumeration>
<xs:enumeration value="TIMER_DRIVEN"></xs:enumeration>
<xs:enumeration value="PRIMARY_NODE_ONLY"></xs:enumeration>
<xs:enumeration value="CRON_DRIVEN"></xs:enumeration>
</xs:restriction>
</xs:simpleType>
<xs:simpleType name="BackoffMechanism">
<xs:restriction base="xs:string">
<xs:enumeration value="PENALIZE_FLOWFILE"></xs:enumeration>
<xs:enumeration value="YIELD_PROCESSOR"></xs:enumeration>
</xs:restriction>
</xs:simpleType>
<xs:simpleType name="ExecutionNode">
<xs:restriction base="xs:string">
<xs:enumeration value="ALL"></xs:enumeration>
<xs:enumeration value="PRIMARY"></xs:enumeration>
</xs:restriction>
</xs:simpleType>
<xs:complexType name="ControllerServicesType">
<xs:sequence>
<xs:element name="controllerService" type="ControllerServiceType" minOccurs="0" maxOccurs="unbounded" />
</xs:sequence>
</xs:complexType>
<xs:complexType name="ControllerServiceType">
<xs:sequence>
<xs:element name="id" type="NonEmptyStringType" />
<xs:element name="versionedComponentId" type="NonEmptyStringType" minOccurs="0" maxOccurs="1" />
<xs:element name="name" type="NonEmptyStringType" />
<xs:element name="comment" type="xs:string" />
<xs:element name="bulletinLevel" type="LogLevel" minOccurs="0" maxOccurs="1" />
<xs:element name="class" type="NonEmptyStringType" />
<xs:element name="bundle" type="BundleType" />
<xs:element name="enabled" type="xs:boolean" />
<xs:element name="property" type="PropertyType" minOccurs="0" maxOccurs="unbounded"/>
<xs:element name="annotationData" type="xs:string" minOccurs="0" maxOccurs="1" />
</xs:sequence>
</xs:complexType>
<xs:complexType name="ReportingTasksType">
<xs:sequence>
<xs:element name="reportingTask" type="ReportingTaskType" minOccurs="0" maxOccurs="unbounded" />
</xs:sequence>
</xs:complexType>
<xs:complexType name="ReportingTaskType">
<xs:sequence>
<xs:element name="id" type="NonEmptyStringType" />
<xs:element name="name" type="NonEmptyStringType" />
<xs:element name="comment" type="xs:string" />
<xs:element name="class" type="NonEmptyStringType" />
<xs:element name="bundle" type="BundleType" />
<xs:element name="schedulingPeriod" type="NonEmptyStringType"/>
<xs:element name="scheduledState" type="ScheduledState" />
<xs:element name="schedulingStrategy" type="SchedulingStrategy" />
<xs:element name="property" type="PropertyType" minOccurs="0" maxOccurs="unbounded"/>
<xs:element name="annotationData" type="xs:string" minOccurs="0" maxOccurs="1" />
</xs:sequence>
</xs:complexType>
<xs:complexType name="FlowAnalysisRulesType">
<xs:sequence>
<xs:element name="flowAnalysisRule" type="FlowAnalysisRuleType" minOccurs="0" maxOccurs="unbounded" />
</xs:sequence>
</xs:complexType>
<xs:complexType name="FlowAnalysisRuleType">
<xs:sequence>
<xs:element name="id" type="NonEmptyStringType" />
<xs:element name="name" type="NonEmptyStringType" />
<xs:element name="comment" type="xs:string" />
<xs:element name="class" type="NonEmptyStringType" />
<xs:element name="bundle" type="BundleType" />
<xs:element name="enforcementPolicy" type="FlowAnalysisRuleEnforcementPolicy"/>
<xs:element name="state" type="FlowAnalysisState" />
<xs:element name="property" type="PropertyType" minOccurs="0" maxOccurs="unbounded"/>
<xs:element name="annotationData" type="xs:string" minOccurs="0" maxOccurs="1" />
</xs:sequence>
</xs:complexType>
<xs:complexType name="ParameterProvidersType">
<xs:sequence>
<xs:element name="parameterProvider" type="ParameterProviderType" minOccurs="0" maxOccurs="unbounded" />
</xs:sequence>
</xs:complexType>
<xs:complexType name="ParameterProviderType">
<xs:sequence>
<xs:element name="id" type="NonEmptyStringType" />
<xs:element name="name" type="NonEmptyStringType" />
<xs:element name="comment" type="xs:string" />
<xs:element name="class" type="NonEmptyStringType" />
<xs:element name="bundle" type="BundleType" />
<xs:element name="property" type="PropertyType" minOccurs="0" maxOccurs="unbounded"/>
<xs:element name="annotationData" type="xs:string" minOccurs="0" maxOccurs="1" />
</xs:sequence>
</xs:complexType>
<xs:complexType name="SnippetType">
<xs:sequence>
<xs:any processContents="skip" minOccurs="0" maxOccurs="unbounded" />
</xs:sequence>
<xs:anyAttribute processContents="skip" />
</xs:complexType>
</xs:schema>

38
nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/controller/StandardFlowServiceTest.java
View File

@ -20,10 +20,12 @@ import org.apache.commons.io.IOUtils;
import org.apache.nifi.admin.service.AuditService;
import org.apache.nifi.authorization.Authorizer;
import org.apache.nifi.cluster.protocol.StandardDataFlow;
import org.apache.nifi.controller.flow.VersionedDataflow;
import org.apache.nifi.controller.repository.FlowFileEventRepository;
import org.apache.nifi.controller.serialization.FlowSerializationException;
import org.apache.nifi.controller.serialization.FlowSerializer;
import org.apache.nifi.controller.serialization.ScheduledStateLookup;
import org.apache.nifi.controller.serialization.StandardFlowSerializer;
import org.apache.nifi.controller.serialization.VersionedFlowSerializer;
import org.apache.nifi.controller.status.history.StatusHistoryRepository;
import org.apache.nifi.encrypt.PropertyEncryptor;
import org.apache.nifi.events.VolatileBulletinRepository;
@ -39,11 +41,10 @@ import org.apache.nifi.web.api.dto.ProcessorConfigDTO;
import org.apache.nifi.web.api.dto.ProcessorDTO;
import org.apache.nifi.web.revision.RevisionManager;
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.BeforeAll;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Disabled;
import org.junit.jupiter.api.Test;
import org.w3c.dom.Document;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
@ -86,19 +87,18 @@ public class StandardFlowServiceTest {
extensionManager = mock(ExtensionDiscoveringManager.class);
flowController = FlowController.createStandaloneInstance(mockFlowFileEventRepository, properties, authorizer, mockAuditService, mockEncryptor,
new VolatileBulletinRepository(), extensionManager, statusHistoryRepository, null);
flowService = StandardFlowService.createStandaloneInstance(flowController, properties, revisionManager, authorizer,
FlowSerializationStrategy.WRITE_XML_AND_JSON);
flowService = StandardFlowService.createStandaloneInstance(flowController, properties, revisionManager, authorizer);
statusHistoryRepository = mock(StatusHistoryRepository.class);
}
@Test
public void testLoadWithFlow() throws IOException {
byte[] flowBytes = IOUtils.toByteArray(StandardFlowServiceTest.class.getResourceAsStream("/conf/all-flow.xml"));
byte[] flowBytes = IOUtils.toByteArray(StandardFlowServiceTest.class.getResourceAsStream("/conf/all-flow.json"));
flowService.load(new StandardDataFlow(flowBytes, null, null, new HashSet<>()));
StandardFlowSerializer serializer = new StandardFlowSerializer();
final FlowSerializer<VersionedDataflow> serializer = new VersionedFlowSerializer(extensionManager);
ByteArrayOutputStream baos = new ByteArrayOutputStream();
final Document doc = serializer.transform(flowController, ScheduledStateLookup.IDENTITY_LOOKUP);
final VersionedDataflow doc = serializer.transform(flowController, ScheduledStateLookup.IDENTITY_LOOKUP);
serializer.serialize(doc, baos);
String expectedFlow = new String(flowBytes).trim();
@ -118,15 +118,15 @@ public class StandardFlowServiceTest {
@Test
public void testLoadExistingFlow() throws IOException {
byte[] flowBytes = IOUtils.toByteArray(StandardFlowServiceTest.class.getResourceAsStream("/conf/all-flow.xml"));
byte[] flowBytes = IOUtils.toByteArray(StandardFlowServiceTest.class.getResourceAsStream("/conf/all-flow.json"));
flowService.load(new StandardDataFlow(flowBytes, null, null, new HashSet<>()));
flowBytes = IOUtils.toByteArray(StandardFlowServiceTest.class.getResourceAsStream("/conf/all-flow-inheritable.xml"));
flowBytes = IOUtils.toByteArray(StandardFlowServiceTest.class.getResourceAsStream("/conf/all-flow-inheritable.json"));
flowService.load(new StandardDataFlow(flowBytes, null, null, new HashSet<>()));
StandardFlowSerializer serializer = new StandardFlowSerializer();
final FlowSerializer<VersionedDataflow> serializer = new VersionedFlowSerializer(extensionManager);
ByteArrayOutputStream baos = new ByteArrayOutputStream();
final Document doc = serializer.transform(flowController, ScheduledStateLookup.IDENTITY_LOOKUP);
final VersionedDataflow doc = serializer.transform(flowController, ScheduledStateLookup.IDENTITY_LOOKUP);
serializer.serialize(doc, baos);
String expectedFlow = new String(flowBytes).trim();
@ -136,17 +136,17 @@ public class StandardFlowServiceTest {
@Test
public void testLoadExistingFlowWithUninheritableFlow() throws IOException {
byte[] originalBytes = IOUtils.toByteArray(StandardFlowServiceTest.class.getResourceAsStream("/conf/all-flow.xml"));
byte[] originalBytes = IOUtils.toByteArray(StandardFlowServiceTest.class.getResourceAsStream("/conf/all-flow.json"));
flowService.load(new StandardDataFlow(originalBytes, null, null, new HashSet<>()));
try {
byte[] updatedBytes = IOUtils.toByteArray(StandardFlowServiceTest.class.getResourceAsStream("/conf/all-flow-uninheritable.xml"));
byte[] updatedBytes = IOUtils.toByteArray(StandardFlowServiceTest.class.getResourceAsStream("/conf/all-flow-uninheritable.json"));
flowService.load(new StandardDataFlow(updatedBytes, null, null, new HashSet<>()));
fail("should have thrown " + UninheritableFlowException.class);
} catch (UninheritableFlowException ufe) {
StandardFlowSerializer serializer = new StandardFlowSerializer();
final FlowSerializer<VersionedDataflow> serializer = new VersionedFlowSerializer(extensionManager);
ByteArrayOutputStream baos = new ByteArrayOutputStream();
final Document doc = serializer.transform(flowController, ScheduledStateLookup.IDENTITY_LOOKUP);
final VersionedDataflow doc = serializer.transform(flowController, ScheduledStateLookup.IDENTITY_LOOKUP);
serializer.serialize(doc, baos);
String expectedFlow = new String(originalBytes).trim();
@ -158,7 +158,7 @@ public class StandardFlowServiceTest {
@Test
public void testLoadExistingFlowWithCorruptFlow() throws IOException {
byte[] originalBytes = IOUtils.toByteArray(StandardFlowServiceTest.class.getResourceAsStream("/conf/all-flow.xml"));
byte[] originalBytes = IOUtils.toByteArray(StandardFlowServiceTest.class.getResourceAsStream("/conf/all-flow.json"));
flowService.load(new StandardDataFlow(originalBytes, null, null, new HashSet<>()));
try {
@ -166,9 +166,9 @@ public class StandardFlowServiceTest {
flowService.load(new StandardDataFlow(updatedBytes, null, null, new HashSet<>()));
fail("should have thrown " + FlowSerializationException.class);
} catch (FlowSerializationException ufe) {
StandardFlowSerializer serializer = new StandardFlowSerializer();
final FlowSerializer<VersionedDataflow> serializer = new VersionedFlowSerializer(extensionManager);
ByteArrayOutputStream baos = new ByteArrayOutputStream();
final Document doc = serializer.transform(flowController, ScheduledStateLookup.IDENTITY_LOOKUP);
final VersionedDataflow doc = serializer.transform(flowController, ScheduledStateLookup.IDENTITY_LOOKUP);
serializer.serialize(doc, baos);
String expectedFlow = new String(originalBytes).trim();

97
nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/controller/TestFlowController.java
View File

@ -43,7 +43,6 @@ import org.apache.nifi.controller.repository.FlowFileEventRepository;
import org.apache.nifi.controller.scheduling.StandardProcessScheduler;
import org.apache.nifi.controller.serialization.FlowSynchronizationException;
import org.apache.nifi.controller.serialization.FlowSynchronizer;
import org.apache.nifi.controller.serialization.StandardFlowSynchronizer;
import org.apache.nifi.controller.serialization.VersionedFlowSynchronizer;
import org.apache.nifi.controller.service.ControllerServiceNode;
import org.apache.nifi.controller.service.ControllerServiceProvider;
@ -86,8 +85,8 @@ import org.apache.nifi.web.api.dto.PositionDTO;
import org.apache.nifi.web.api.dto.ProcessorConfigDTO;
import org.apache.nifi.web.api.dto.ProcessorDTO;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.BeforeAll;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import java.io.File;
@ -140,7 +139,7 @@ public class TestFlowController {
private BulletinRepository bulletinRepo;
private ExtensionDiscoveringManager extensionManager;
private StatusHistoryRepository statusHistoryRepository;
private FlowSynchronizer standardFlowSynchronizer;
private FlowSynchronizer flowSynchronizer;
private static List<String> allIdentifiers;
@ -212,10 +211,8 @@ public class TestFlowController {
auditService, encryptor, bulletinRepo, extensionManager, statusHistoryRepository,
mock(RuleViolationsManager.class));
final XmlFlowSynchronizer xmlFlowSynchronizer = new XmlFlowSynchronizer(nifiProperties, extensionManager);
final VersionedFlowSynchronizer versionedFlowSynchronizer = new VersionedFlowSynchronizer(extensionManager,
nifiProperties.getFlowConfigurationJsonFile(), new FlowConfigurationArchiveManager(nifiProperties));
standardFlowSynchronizer = new StandardFlowSynchronizer(xmlFlowSynchronizer, versionedFlowSynchronizer);
flowSynchronizer = new VersionedFlowSynchronizer(extensionManager,
nifiProperties.getFlowConfigurationFile(), new FlowConfigurationArchiveManager(nifiProperties));
}
@AfterEach
@ -247,12 +244,12 @@ public class TestFlowController {
public void testSynchronizeFlowWithReportingTaskAndProcessorReferencingControllerService() throws IOException {
// create a mock proposed data flow with the same auth fingerprint as the current authorizer
final String authFingerprint = authorizer.getFingerprint();
final File flowFile = new File("src/test/resources/conf/reporting-task-with-cs-flow-0.7.0.xml");
final File flowFile = new File("src/test/resources/conf/reporting-task-with-cs-flow-0.7.0.json");
final String flow = IOUtils.toString(new FileInputStream(flowFile), StandardCharsets.UTF_8);
final DataFlow proposedDataFlow = new StandardDataFlow(flow.getBytes(StandardCharsets.UTF_8), null, authFingerprint.getBytes(StandardCharsets.UTF_8), Collections.emptySet());
controller.synchronize(standardFlowSynchronizer, proposedDataFlow, mock(FlowService.class), BundleUpdateStrategy.IGNORE_BUNDLE);
controller.synchronize(flowSynchronizer, proposedDataFlow, mock(FlowService.class), BundleUpdateStrategy.IGNORE_BUNDLE);
// should be two controller services
final Set<ControllerServiceNode> controllerServiceNodes = controller.getFlowManager().getAllControllerServices();
@ -306,12 +303,12 @@ public class TestFlowController {
public void testSynchronizeFlowWithParameterProviderReferencingControllerService() throws IOException {
// create a mock proposed data flow with the same auth fingerprint as the current authorizer
final String authFingerprint = authorizer.getFingerprint();
final File flowFile = new File("src/test/resources/conf/parameter-provider-with-cs-flow.xml");
final File flowFile = new File("src/test/resources/conf/parameter-provider-with-cs-flow.json");
final String flow = IOUtils.toString(new FileInputStream(flowFile), StandardCharsets.UTF_8);
final DataFlow proposedDataFlow = new StandardDataFlow(flow.getBytes(StandardCharsets.UTF_8), null, authFingerprint.getBytes(StandardCharsets.UTF_8), Collections.emptySet());
controller.synchronize(standardFlowSynchronizer, proposedDataFlow, mock(FlowService.class), BundleUpdateStrategy.IGNORE_BUNDLE);
controller.synchronize(flowSynchronizer, proposedDataFlow, mock(FlowService.class), BundleUpdateStrategy.IGNORE_BUNDLE);
// should be two controller services
final Set<ControllerServiceNode> controllerServiceNodes = controller.getFlowManager().getAllControllerServices();
@ -365,12 +362,12 @@ public class TestFlowController {
public void testSynchronizeFlowWithProcessorReferencingControllerService() throws IOException {
// create a mock proposed data flow with the same auth fingerprint as the current authorizer
final String authFingerprint = authorizer.getFingerprint();
final File flowFile = new File("src/test/resources/conf/processor-with-cs-flow-0.7.0.xml");
final File flowFile = new File("src/test/resources/conf/processor-with-cs-flow-0.7.0.json");
final String flow = IOUtils.toString(new FileInputStream(flowFile), StandardCharsets.UTF_8);
final DataFlow proposedDataFlow = new StandardDataFlow(flow.getBytes(StandardCharsets.UTF_8), null, authFingerprint.getBytes(StandardCharsets.UTF_8), Collections.emptySet());
controller.synchronize(standardFlowSynchronizer, proposedDataFlow, mock(FlowService.class), BundleUpdateStrategy.IGNORE_BUNDLE);
controller.synchronize(flowSynchronizer, proposedDataFlow, mock(FlowService.class), BundleUpdateStrategy.IGNORE_BUNDLE);
try {
// should be two controller services
@ -407,31 +404,29 @@ public class TestFlowController {
final DataFlow proposedDataFlow = mock(DataFlow.class);
when(proposedDataFlow.getAuthorizerFingerprint()).thenReturn(authFingerprint.getBytes(StandardCharsets.UTF_8));
controller.synchronize(standardFlowSynchronizer, proposedDataFlow, mock(FlowService.class), BundleUpdateStrategy.IGNORE_BUNDLE);
controller.synchronize(flowSynchronizer, proposedDataFlow, mock(FlowService.class), BundleUpdateStrategy.IGNORE_BUNDLE);
assertEquals(authFingerprint, authorizer.getFingerprint());
}
@Test
public void testSynchronizeFlowWhenAuthorizationsAreDifferent() throws IOException {
final File flowFile = new File("src/test/resources/conf/processor-with-cs-flow-0.7.0.xml");
final File flowFile = new File("src/test/resources/conf/processor-with-cs-flow-0.7.0.json");
final String flow = IOUtils.toString(new FileInputStream(flowFile), StandardCharsets.UTF_8);
final String authFingerprint = "<authorizations></authorizations>";
final DataFlow proposedDataFlow = new StandardDataFlow(flow.getBytes(StandardCharsets.UTF_8), null, authFingerprint.getBytes(StandardCharsets.UTF_8), Collections.emptySet());
controller.synchronize(standardFlowSynchronizer, proposedDataFlow, mock(FlowService.class), BundleUpdateStrategy.IGNORE_BUNDLE);
controller.synchronize(flowSynchronizer, proposedDataFlow, mock(FlowService.class), BundleUpdateStrategy.IGNORE_BUNDLE);
controller.initializeFlow();
assertThrows(UninheritableFlowException.class,
() -> controller.synchronize(standardFlowSynchronizer, proposedDataFlow, mock(FlowService.class), BundleUpdateStrategy.IGNORE_BUNDLE));
assertNotEquals(authFingerprint, authorizer.getFingerprint());
purgeFlow();
}
@Test
public void testSynchronizeFlowWithInvalidParameterContextReference() throws IOException {
final File flowFile = new File("src/test/resources/conf/parameter-context-flow-error.xml");
final File flowFile = new File("src/test/resources/conf/parameter-context-flow-error.json");
final String flow = IOUtils.toString(new FileInputStream(flowFile), StandardCharsets.UTF_8);
final String authFingerprint = "<authorizations></authorizations>";
@ -439,7 +434,7 @@ public class TestFlowController {
assertThrows(FlowSynchronizationException.class,
() -> {
controller.synchronize(standardFlowSynchronizer, proposedDataFlow, mock(FlowService.class), BundleUpdateStrategy.IGNORE_BUNDLE);
controller.synchronize(flowSynchronizer, proposedDataFlow, mock(FlowService.class), BundleUpdateStrategy.IGNORE_BUNDLE);
controller.initializeFlow();
});
purgeFlow();
@ -447,14 +442,14 @@ public class TestFlowController {
@Test
public void testSynchronizeFlowWithNestedParameterContexts() throws IOException {
final File flowFile = new File("src/test/resources/conf/parameter-context-flow.xml");
final File flowFile = new File("src/test/resources/conf/parameter-context-flow.json");
final String flow = IOUtils.toString(new FileInputStream(flowFile), StandardCharsets.UTF_8);
final String authFingerprint = "<authorizations></authorizations>";
final DataFlow proposedDataFlow = new StandardDataFlow(flow.getBytes(StandardCharsets.UTF_8), null, authFingerprint.getBytes(StandardCharsets.UTF_8), Collections.emptySet());
try {
controller.synchronize(standardFlowSynchronizer, proposedDataFlow, mock(FlowService.class), BundleUpdateStrategy.IGNORE_BUNDLE);
controller.synchronize(flowSynchronizer, proposedDataFlow, mock(FlowService.class), BundleUpdateStrategy.IGNORE_BUNDLE);
controller.initializeFlow();
ParameterContext parameterContext = controller.getFlowManager().getParameterContextManager().getParameterContext("context");
@ -469,14 +464,14 @@ public class TestFlowController {
@Test
public void testCreateParameterContextWithAndWithoutValidation() throws IOException {
final File flowFile = new File("src/test/resources/conf/parameter-context-flow.xml");
final File flowFile = new File("src/test/resources/conf/parameter-context-flow.json");
final String flow = IOUtils.toString(new FileInputStream(flowFile), StandardCharsets.UTF_8);
final String authFingerprint = "<authorizations></authorizations>";
final DataFlow proposedDataFlow = new StandardDataFlow(flow.getBytes(StandardCharsets.UTF_8), null, authFingerprint.getBytes(StandardCharsets.UTF_8), Collections.emptySet());
try {
controller.synchronize(standardFlowSynchronizer, proposedDataFlow, mock(FlowService.class), BundleUpdateStrategy.IGNORE_BUNDLE);
controller.synchronize(flowSynchronizer, proposedDataFlow, mock(FlowService.class), BundleUpdateStrategy.IGNORE_BUNDLE);
controller.initializeFlow();
final Map<String, Parameter> parameters = new HashMap<>();
@ -520,7 +515,7 @@ public class TestFlowController {
final DataFlow proposedDataFlow = mock(DataFlow.class);
when(proposedDataFlow.getAuthorizerFingerprint()).thenReturn(authFingerprint.getBytes(StandardCharsets.UTF_8));
controller.synchronize(standardFlowSynchronizer, proposedDataFlow, mock(FlowService.class), BundleUpdateStrategy.IGNORE_BUNDLE);
controller.synchronize(flowSynchronizer, proposedDataFlow, mock(FlowService.class), BundleUpdateStrategy.IGNORE_BUNDLE);
assertNotEquals(authFingerprint, authorizer.getFingerprint());
assertTrue(authorizer.getGroups().isEmpty());
@ -530,19 +525,19 @@ public class TestFlowController {
@Test
public void testSynchronizeFlowWhenProposedAuthorizationsAreNull() throws IOException {
final File flowFile = new File("src/test/resources/conf/processor-with-cs-flow-0.7.0.xml");
final File flowFile = new File("src/test/resources/conf/processor-with-cs-flow-0.7.0.json");
final String flow = IOUtils.toString(new FileInputStream(flowFile), StandardCharsets.UTF_8);
final String authFingerprint = "<authorizations></authorizations>";
final DataFlow proposedDataFlow = new StandardDataFlow(flow.getBytes(StandardCharsets.UTF_8), null, authFingerprint.getBytes(StandardCharsets.UTF_8), Collections.emptySet());
controller.synchronize(standardFlowSynchronizer, proposedDataFlow, mock(FlowService.class), BundleUpdateStrategy.IGNORE_BUNDLE);
controller.synchronize(flowSynchronizer, proposedDataFlow, mock(FlowService.class), BundleUpdateStrategy.IGNORE_BUNDLE);
controller.initializeFlow();
final DataFlow dataflowWithNullAuthorizations = new StandardDataFlow(flow.getBytes(StandardCharsets.UTF_8), null, null, Collections.emptySet());
assertThrows(UninheritableFlowException.class,
() -> controller.synchronize(standardFlowSynchronizer, dataflowWithNullAuthorizations, mock(FlowService.class), BundleUpdateStrategy.IGNORE_BUNDLE));
() -> controller.synchronize(flowSynchronizer, dataflowWithNullAuthorizations, mock(FlowService.class), BundleUpdateStrategy.IGNORE_BUNDLE));
purgeFlow();
}
@ -550,8 +545,8 @@ public class TestFlowController {
public void testSynchronizeFlowWhenProposedAuthorizationsAreNullAndEmptyFlow() {
final DataFlow proposedDataFlow = mock(DataFlow.class);
when(proposedDataFlow.getAuthorizerFingerprint()).thenReturn(null);
controller.synchronize(standardFlowSynchronizer, proposedDataFlow, mock(FlowService.class), BundleUpdateStrategy.IGNORE_BUNDLE);
when(proposedDataFlow.getVersionedDataflow()).thenReturn(getVersionedDataflow());
controller.synchronize(flowSynchronizer, proposedDataFlow, mock(FlowService.class), BundleUpdateStrategy.IGNORE_BUNDLE);
assertTrue(authorizer.getGroups().isEmpty());
assertTrue(authorizer.getUsers().isEmpty());
@ -588,7 +583,7 @@ public class TestFlowController {
controller.shutdown(true);
controller = FlowController.createStandaloneInstance(flowFileEventRepo, nifiProperties, authorizer,
auditService, encryptor, bulletinRepo, extensionManager, statusHistoryRepository, null);
controller.synchronize(standardFlowSynchronizer, proposedDataFlow, mock(FlowService.class), BundleUpdateStrategy.IGNORE_BUNDLE);
controller.synchronize(flowSynchronizer, proposedDataFlow, mock(FlowService.class), BundleUpdateStrategy.IGNORE_BUNDLE);
assertEquals(authFingerprint, authorizer.getFingerprint());
}
@ -603,7 +598,7 @@ public class TestFlowController {
UninheritableFlowException uninheritableFlowException =
assertThrows(UninheritableFlowException.class,
() -> controller.synchronize(standardFlowSynchronizer, proposedDataFlow, mock(FlowService.class), BundleUpdateStrategy.IGNORE_BUNDLE));
() -> controller.synchronize(flowSynchronizer, proposedDataFlow, mock(FlowService.class), BundleUpdateStrategy.IGNORE_BUNDLE));
assertTrue(uninheritableFlowException.getMessage().contains("Proposed flow has missing components " +
"that are not considered missing in the current flow (1,2)"), uninheritableFlowException.getMessage());
}
@ -643,7 +638,7 @@ public class TestFlowController {
when(proposedDataFlow.getMissingComponents()).thenReturn(new HashSet<>());
UninheritableFlowException uninheritableFlowException =
assertThrows(UninheritableFlowException.class,
() -> standardFlowSynchronizer.sync(mockFlowController, proposedDataFlow,
() -> flowSynchronizer.sync(mockFlowController, proposedDataFlow,
mock(FlowService.class), BundleUpdateStrategy.IGNORE_BUNDLE));
assertTrue(uninheritableFlowException.getMessage().contains("Current flow has missing components that are not" +
" considered missing in the proposed flow (1,2,3)"), uninheritableFlowException.getMessage());
@ -654,8 +649,8 @@ public class TestFlowController {
final LogRepository logRepository = LogRepositoryFactory.getRepository("d89ada5d-35fb-44ff-83f1-4cc00b48b2df");
logRepository.removeAllObservers();
syncFlow("src/test/resources/nifi/fingerprint/flow4.xml", standardFlowSynchronizer);
syncFlow("src/test/resources/nifi/fingerprint/flow4.xml", standardFlowSynchronizer);
syncFlow("src/test/resources/nifi/fingerprint/flow4.json", flowSynchronizer);
syncFlow("src/test/resources/nifi/fingerprint/flow4.json", flowSynchronizer);
}
@Test
@ -664,14 +659,14 @@ public class TestFlowController {
logRepository.removeAllObservers();
// first sync should work because we are syncing to an empty flow controller
syncFlow("src/test/resources/nifi/fingerprint/flow4.xml", standardFlowSynchronizer);
syncFlow("src/test/resources/nifi/fingerprint/flow4.json", flowSynchronizer);
controller.initializeFlow();
// second sync should fail because the bundle of the processor is different
assertThrows(UninheritableFlowException.class,
() -> syncFlow("src/test/resources/nifi/fingerprint/flow4-with-different-bundle.xml",
standardFlowSynchronizer));
() -> syncFlow("src/test/resources/nifi/fingerprint/flow4-with-different-bundle.json",
flowSynchronizer));
}
private void syncFlow(String flowXmlFile, FlowSynchronizer standardFlowSynchronizer) throws IOException {
@ -686,7 +681,7 @@ public class TestFlowController {
final byte[] authFingerprintBytes = authFingerprint.getBytes(StandardCharsets.UTF_8);
final DataFlow proposedDataFlow1 = new StandardDataFlow(flowBytes, null, authFingerprintBytes, Collections.emptySet());
controller.synchronize(standardFlowSynchronizer, proposedDataFlow1, mock(FlowService.class), BundleUpdateStrategy.IGNORE_BUNDLE);
controller.synchronize(standardFlowSynchronizer, proposedDataFlow1, mock(FlowService.class), BundleUpdateStrategy.USE_SPECIFIED_OR_FAIL);
}
@Test
@ -1303,10 +1298,7 @@ public class TestFlowController {
authFingerprint.getBytes(StandardCharsets.UTF_8),
Collections.emptySet());
// following assertion asserts that VersionedFlowSynchronizer is used
assertFalse(proposedDataFlow.isXml());
controller.synchronize(standardFlowSynchronizer, proposedDataFlow, mock(FlowService.class), BundleUpdateStrategy.IGNORE_BUNDLE);
controller.synchronize(flowSynchronizer, proposedDataFlow, mock(FlowService.class), BundleUpdateStrategy.IGNORE_BUNDLE);
// should be an empty dataflow
final Map<String, Integer> componentCounts = controller.getFlowManager().getComponentCounts();
@ -1334,10 +1326,7 @@ public class TestFlowController {
authFingerprint.getBytes(StandardCharsets.UTF_8),
Collections.emptySet());
// following assertion asserts that VersionedFlowSynchronizer is used
assertFalse(proposedDataFlow.isXml());
controller.synchronize(standardFlowSynchronizer, proposedDataFlow, mock(FlowService.class), BundleUpdateStrategy.IGNORE_BUNDLE);
controller.synchronize(flowSynchronizer, proposedDataFlow, mock(FlowService.class), BundleUpdateStrategy.IGNORE_BUNDLE);
final Map<String, Integer> componentCounts = controller.getFlowManager().getComponentCounts();
@ -1366,6 +1355,14 @@ public class TestFlowController {
}
private String getNewJsonFlow() throws JsonProcessingException {
final VersionedDataflow versionedDataflow = getVersionedDataflow();
final ObjectMapper mapper = new ObjectMapper();
final String jsonString = mapper.writeValueAsString(versionedDataflow);
return jsonString;
}
private static VersionedDataflow getVersionedDataflow() {
final VersionedDataflow versionedDataflow = new VersionedDataflow();
versionedDataflow.setEncodingVersion(new VersionedFlowEncodingVersion(2, 0));
@ -1398,10 +1395,6 @@ public class TestFlowController {
rootGroup.setFlowFileConcurrency("UNBOUNDED");
rootGroup.setComponentType(ComponentType.PROCESS_GROUP);
versionedDataflow.setRootGroup(rootGroup);
final ObjectMapper mapper = new ObjectMapper();
final String jsonString = mapper.writeValueAsString(versionedDataflow);
return jsonString;
return versionedDataflow;
}
}

169
nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/controller/serialization/StandardFlowSerializerTest.java
View File

@ -1,169 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.nifi.controller.serialization;
import org.apache.commons.io.FileUtils;
import org.apache.nifi.admin.service.AuditService;
import org.apache.nifi.authorization.AbstractPolicyBasedAuthorizer;
import org.apache.nifi.authorization.MockPolicyBasedAuthorizer;
import org.apache.nifi.bundle.Bundle;
import org.apache.nifi.controller.DummyScheduledProcessor;
import org.apache.nifi.controller.FlowController;
import org.apache.nifi.controller.ProcessorNode;
import org.apache.nifi.controller.repository.FlowFileEventRepository;
import org.apache.nifi.controller.status.history.StatusHistoryRepository;
import org.apache.nifi.encrypt.PropertyEncryptor;
import org.apache.nifi.nar.ExtensionDiscoveringManager;
import org.apache.nifi.nar.StandardExtensionDiscoveringManager;
import org.apache.nifi.nar.SystemBundle;
import org.apache.nifi.parameter.Parameter;
import org.apache.nifi.parameter.ParameterContext;
import org.apache.nifi.parameter.ParameterDescriptor;
import org.apache.nifi.parameter.ParameterReferenceManager;
import org.apache.nifi.parameter.StandardParameterContext;
import org.apache.nifi.provenance.MockProvenanceRepository;
import org.apache.nifi.reporting.BulletinRepository;
import org.apache.nifi.util.NiFiProperties;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.mockito.Mockito;
import org.w3c.dom.Document;
import java.io.ByteArrayOutputStream;
import java.io.File;
import java.nio.charset.StandardCharsets;
import java.util.Arrays;
import java.util.Collections;
import java.util.HashMap;
import java.util.Map;
import java.util.UUID;
import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertTrue;
public class StandardFlowSerializerTest {
private static final String RAW_COMMENTS
= "<tagName> \"This\" is an ' example with many characters that need to be filtered and escaped \u0002 in it. \u007f \u0086 " + Character.MIN_SURROGATE;
private static final String SERIALIZED_COMMENTS
= "&lt;tagName&gt; \"This\" is an ' example with many characters that need to be filtered and escaped in it. &#127; &#134; ";
private static final String RAW_STRING_WITH_EMOJI = "String with \uD83D\uDCA7 droplet emoji";
private static final String SERIALIZED_STRING_WITH_EMOJI = "String with &#128167; droplet emoji";
private volatile String propsFile = StandardFlowSerializerTest.class.getResource("/standardflowserializertest.nifi.properties").getFile();
private FlowController controller;
private Bundle systemBundle;
private ExtensionDiscoveringManager extensionManager;
private StandardFlowSerializer serializer;
@BeforeEach
public void setUp() throws Exception {
final FlowFileEventRepository flowFileEventRepo = Mockito.mock(FlowFileEventRepository.class);
final AuditService auditService = Mockito.mock(AuditService.class);
final Map<String, String> otherProps = new HashMap<>();
otherProps.put(NiFiProperties.PROVENANCE_REPO_IMPLEMENTATION_CLASS, MockProvenanceRepository.class.getName());
otherProps.put("nifi.remote.input.socket.port", "");
otherProps.put("nifi.remote.input.secure", "");
final NiFiProperties nifiProperties = NiFiProperties.createBasicNiFiProperties(propsFile, otherProps);
final PropertyEncryptor encryptor = Mockito.mock(PropertyEncryptor.class);
// use the system bundle
systemBundle = SystemBundle.create(nifiProperties);
extensionManager = new StandardExtensionDiscoveringManager();
extensionManager.discoverExtensions(systemBundle, Collections.emptySet());
final AbstractPolicyBasedAuthorizer authorizer = new MockPolicyBasedAuthorizer();
final BulletinRepository bulletinRepo = Mockito.mock(BulletinRepository.class);
controller = FlowController.createStandaloneInstance(flowFileEventRepo, nifiProperties, authorizer,
auditService, encryptor, bulletinRepo, extensionManager, Mockito.mock(StatusHistoryRepository.class), null);
serializer = new StandardFlowSerializer();
}
@AfterEach
public void after() throws Exception {
controller.shutdown(true);
FileUtils.deleteDirectory(new File("./target/standardflowserializertest"));
}
private static ParameterContext createParameterContext(final String id, final String name) {
return new StandardParameterContext.Builder()
.id(id)
.name(name)
.parameterReferenceManager(ParameterReferenceManager.EMPTY)
.build();
}
@Test
public void testSerializationEscapingAndFiltering() throws Exception {
final ProcessorNode dummy = controller.getFlowManager().createProcessor(DummyScheduledProcessor.class.getName(),
UUID.randomUUID().toString(), systemBundle.getBundleDetails().getCoordinate());
dummy.setComments(RAW_COMMENTS);
controller.getFlowManager().getRootGroup().addProcessor(dummy);
final ParameterContext parameterContext = createParameterContext("context", "Context");
final ParameterContext referencedContext = createParameterContext("referenced-context", "Referenced Context");
final ParameterContext referencedContext2 = createParameterContext("referenced-context-2", "Referenced Context 2");
final Map<String, Parameter> parameters = new HashMap<>();
final ParameterDescriptor parameterDescriptor = new ParameterDescriptor.Builder().name("foo").sensitive(true).build();
parameters.put("foo", new Parameter(parameterDescriptor, "value"));
parameterContext.setInheritedParameterContexts(Arrays.asList(referencedContext, referencedContext2));
parameterContext.setParameters(parameters);
controller.getFlowManager().getParameterContextManager().addParameterContext(parameterContext);
controller.getFlowManager().getParameterContextManager().addParameterContext(referencedContext);
controller.getFlowManager().getParameterContextManager().addParameterContext(referencedContext2);
controller.getFlowManager().getRootGroup().setParameterContext(parameterContext);
controller.getFlowManager().getRootGroup().setParameterContext(parameterContext);
// serialize the controller
final ByteArrayOutputStream os = new ByteArrayOutputStream();
final Document doc = serializer.transform(controller, ScheduledStateLookup.IDENTITY_LOOKUP);
serializer.serialize(doc, os);
// verify the results contain the serialized string
final String serializedFlow = os.toString(StandardCharsets.UTF_8.name());
assertTrue(serializedFlow.contains(SERIALIZED_COMMENTS));
assertFalse(serializedFlow.contains(RAW_COMMENTS));
assertFalse(serializedFlow.contains("\u0001"));
assertTrue(serializedFlow.contains("<inheritedParameterContextId>referenced-context</inheritedParameterContextId>"));
}
@Test
public void testSerializationEmoji() throws Exception {
final ProcessorNode dummy = controller.getFlowManager().createProcessor(DummyScheduledProcessor.class.getName(),
UUID.randomUUID().toString(), systemBundle.getBundleDetails().getCoordinate());
dummy.setName(RAW_STRING_WITH_EMOJI);
controller.getFlowManager().getRootGroup().addProcessor(dummy);
// serialize the controller
final ByteArrayOutputStream os = new ByteArrayOutputStream();
final Document doc = serializer.transform(controller, ScheduledStateLookup.IDENTITY_LOOKUP);
serializer.serialize(doc, os);
// verify the results contain the serialized string
final String serializedFlow = os.toString(StandardCharsets.UTF_8.name());
assertTrue(serializedFlow.contains(SERIALIZED_STRING_WITH_EMOJI));
assertFalse(serializedFlow.contains(RAW_STRING_WITH_EMOJI));
}
}

405
nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/fingerprint/FingerprintFactoryTest.java
View File

@ -1,405 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.nifi.fingerprint;
import org.apache.commons.codec.DecoderException;
import org.apache.commons.codec.binary.Hex;
import org.apache.commons.io.IOUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.nifi.connectable.Position;
import org.apache.nifi.controller.ScheduledState;
import org.apache.nifi.controller.serialization.FlowEncodingVersion;
import org.apache.nifi.controller.serialization.FlowSerializer;
import org.apache.nifi.controller.serialization.ScheduledStateLookup;
import org.apache.nifi.controller.serialization.StandardFlowSerializer;
import org.apache.nifi.encrypt.PropertyEncryptor;
import org.apache.nifi.encrypt.SensitiveValueEncoder;
import org.apache.nifi.groups.RemoteProcessGroup;
import org.apache.nifi.nar.ExtensionManager;
import org.apache.nifi.nar.StandardExtensionDiscoveringManager;
import org.apache.nifi.remote.RemoteGroupPort;
import org.apache.nifi.remote.protocol.SiteToSiteTransportProtocol;
import org.apache.nifi.xml.processing.parsers.DocumentProvider;
import org.apache.nifi.xml.processing.parsers.StandardDocumentProvider;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.w3c.dom.Document;
import org.w3c.dom.Element;
import java.io.FileInputStream;
import java.io.IOException;
import java.lang.reflect.Method;
import java.nio.charset.StandardCharsets;
import java.util.Collections;
import java.util.Optional;
import static org.apache.nifi.controller.serialization.ScheduledStateLookup.IDENTITY_LOOKUP;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertNotEquals;
import static org.junit.jupiter.api.Assertions.assertNotNull;
import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when;
/**
*/
public class FingerprintFactoryTest {
private PropertyEncryptor encryptor;
private ExtensionManager extensionManager;
private FingerprintFactory fingerprintFactory;
private SensitiveValueEncoder sensitiveValueEncoder;
@BeforeEach
public void setup() {
encryptor = createEncryptor();
sensitiveValueEncoder = createSensitiveValueEncoder();
extensionManager = new StandardExtensionDiscoveringManager();
fingerprintFactory = new FingerprintFactory(encryptor, extensionManager, sensitiveValueEncoder);
}
@Test
public void testSameFingerprint() throws IOException {
final String fp1 = fingerprintFactory.createFingerprint(getResourceBytes("/nifi/fingerprint/flow1a.xml"), null);
final String fp2 = fingerprintFactory.createFingerprint(getResourceBytes("/nifi/fingerprint/flow1b.xml"), null);
assertEquals(fp1, fp2);
}
@Test
public void testDifferentFingerprint() throws IOException {
final String fp1 = fingerprintFactory.createFingerprint(getResourceBytes("/nifi/fingerprint/flow1a.xml"), null);
final String fp2 = fingerprintFactory.createFingerprint(getResourceBytes("/nifi/fingerprint/flow2.xml"), null);
assertNotEquals(fp1, fp2);
}
@Test
public void testInheritedParameterContextsInFingerprint() throws IOException {
final String flowWithParameterContext = new String(getResourceBytes("/nifi/fingerprint/flow-with-parameter-context.xml"));
final String flowWithNoInheritedParameterContexts = flowWithParameterContext.replaceAll("<inheritedParameterContextId>.*?</inheritedParameterContextId>", "");
final String flowWithDifferentInheritedParamContextOrder = flowWithParameterContext
.replaceFirst("153a6266-dcd0-33e9-b5af-b8c282d25bf1", "SWAP")
.replaceFirst("253a6266-dcd0-33e9-b5af-b8c282d25bf2", "153a6266-dcd0-33e9-b5af-b8c282d25bf1")
.replaceFirst("SWAP", "253a6266-dcd0-33e9-b5af-b8c282d25bf2");
final String originalFingerprint = fingerprintFactory.createFingerprint(flowWithParameterContext.getBytes(StandardCharsets.UTF_8), null);
final String fingerprintWithNoInheritedParameterContexts = fingerprintFactory.createFingerprint(flowWithNoInheritedParameterContexts.getBytes(StandardCharsets.UTF_8), null);
final String fingerprintWithDifferentInheritedParamContextOrder = fingerprintFactory.createFingerprint(flowWithDifferentInheritedParamContextOrder.getBytes(StandardCharsets.UTF_8), null);
assertNotEquals(originalFingerprint, fingerprintWithNoInheritedParameterContexts);
assertNotEquals(originalFingerprint, fingerprintWithDifferentInheritedParamContextOrder);
}
@Test
public void testResourceValueInFingerprint() throws IOException {
final String fingerprint = fingerprintFactory.createFingerprint(getResourceBytes("/nifi/fingerprint/flow1a.xml"), null);
assertEquals(3, StringUtils.countMatches(fingerprint, "success"));
assertTrue(fingerprint.contains("In Connection"));
}
@Test
public void testSameFlowWithDifferentBundleShouldHaveDifferentFingerprints() throws IOException {
final String fp1 = fingerprintFactory.createFingerprint(getResourceBytes("/nifi/fingerprint/flow3-with-bundle-1.xml"), null);
assertTrue(fp1.contains("org.apache.nifinifi-standard-nar1.0"));
final String fp2 = fingerprintFactory.createFingerprint(getResourceBytes("/nifi/fingerprint/flow3-with-bundle-2.xml"), null);
assertTrue(fp2.contains("org.apache.nifinifi-standard-nar2.0"));
assertNotEquals(fp1, fp2);
}
@Test
public void testSameFlowAndOneHasNoBundleShouldHaveDifferentFingerprints() throws IOException {
final String fp1 = fingerprintFactory.createFingerprint(getResourceBytes("/nifi/fingerprint/flow3-with-bundle-1.xml"), null);
assertTrue(fp1.contains("org.apache.nifinifi-standard-nar1.0"));
final String fp2 = fingerprintFactory.createFingerprint(getResourceBytes("/nifi/fingerprint/flow3-with-no-bundle.xml"), null);
assertTrue(fp2.contains("MISSING_BUNDLE"));
assertNotEquals(fp1, fp2);
}
@Test
public void testSameFlowAndOneHasMissingBundleShouldHaveDifferentFingerprints() throws IOException {
final String fp1 = fingerprintFactory.createFingerprint(getResourceBytes("/nifi/fingerprint/flow3-with-bundle-1.xml"), null);
assertTrue(fp1.contains("org.apache.nifinifi-standard-nar1.0"));
final String fp2 = fingerprintFactory.createFingerprint(getResourceBytes("/nifi/fingerprint/flow3-with-missing-bundle.xml"), null);
assertTrue(fp2.contains("missingmissingmissing"));
assertNotEquals(fp1, fp2);
}
@Test
public void testConnectionWithMultipleRelationshipsSortedInFingerprint() throws IOException {
final String fingerprint = fingerprintFactory.createFingerprint(getResourceBytes("/nifi/fingerprint/flow-connection-with-multiple-rels.xml"), null);
assertNotNull(fingerprint);
assertTrue(fingerprint.contains("AAABBBCCCDDD"));
}
@Test
public void testPublicPortWithAccessPoliciesFingerprint() throws IOException {
final String fingerprint = fingerprintFactory.createFingerprint(getResourceBytes("/nifi/fingerprint/flow1a.xml"), null);
// user access control
assertTrue(fingerprint.contains("user1"));
// group access control
assertTrue(fingerprint.contains("group1"));
// policies fingerprinted
assertTrue(fingerprint.contains("user1group1"));
}
@Test
public void testPublicPortWithDifferentFingerprintInAccessPolicies() throws IOException {
final String f1 = fingerprintFactory.createFingerprint(getResourceBytes("/nifi/fingerprint/flow1a.xml"), null);
assertEquals(2, StringUtils.countMatches(f1, "user1group1"));
final Document document = getDocument("src/test/resources/nifi/fingerprint/public-port-with-no-policies.xml");
final Element rootProcessGroup = document.getDocumentElement();
final StringBuilder sb = new StringBuilder();
fingerprintFactory.addProcessGroupFingerprint(sb, rootProcessGroup, new FlowEncodingVersion(1, 0));
final String f2 = sb.toString();
assertEquals(2, StringUtils.countMatches(f1, "NO_USER_ACCESS_CONTROLNO_GROUP_ACCESS_CONTROL"));
// actual -> Public-IntrueNO_USER_ACCESS_CONTROLNO_GROUP_ACCESS_CONTROL
// expected -> Public-Intrueuser1group1
assertNotEquals(StringUtils.countMatches(f1, "user1group1"), StringUtils.countMatches(f2, "user1group1"));
}
@Test
public void testPublicPortWithNoAccessPoliciesFingerprint() throws IOException {
final Document document = getDocument("src/test/resources/nifi/fingerprint/public-port-with-no-policies.xml");
final Element rootProcessGroup = document.getDocumentElement();
final StringBuilder sb = new StringBuilder();
fingerprintFactory.addProcessGroupFingerprint(sb, rootProcessGroup, new FlowEncodingVersion(1, 0));
final String fingerprint = sb.toString();
assertTrue(fingerprint.contains("NO_USER_ACCESS_CONTROL"));
assertTrue(fingerprint.contains("NO_GROUP_ACCESS_CONTROL"));
}
private Document getDocument(final String filePath) throws IOException {
try (final FileInputStream inputStream = new FileInputStream(filePath)) {
final DocumentProvider documentProvider = new StandardDocumentProvider();
return documentProvider.parse(inputStream);
}
}
private byte[] getResourceBytes(final String resource) throws IOException {
return IOUtils.toByteArray(FingerprintFactoryTest.class.getResourceAsStream(resource));
}
private <T> Element serializeElement(final PropertyEncryptor encryptor, final Class<T> componentClass, final T component,
final String serializerMethodName, ScheduledStateLookup scheduledStateLookup) throws Exception {
final DocumentProvider documentProvider = new StandardDocumentProvider();
final Document doc = documentProvider.newDocument();
final FlowSerializer flowSerializer = new StandardFlowSerializer();
final Method serializeMethod = StandardFlowSerializer.class.getDeclaredMethod(serializerMethodName,
Element.class, componentClass, ScheduledStateLookup.class, PropertyEncryptor.class);
serializeMethod.setAccessible(true);
final Element rootElement = doc.createElement("root");
serializeMethod.invoke(flowSerializer, rootElement, component, scheduledStateLookup, encryptor);
return rootElement;
}
private <T> String fingerprint(final String methodName, final Class<T> inputClass, final T input) throws Exception {
final Method fingerprintFromComponent = FingerprintFactory.class.getDeclaredMethod(methodName,
StringBuilder.class, inputClass);
fingerprintFromComponent.setAccessible(true);
final StringBuilder fingerprint = new StringBuilder();
fingerprintFromComponent.invoke(fingerprintFactory, fingerprint, input);
return fingerprint.toString();
}
@Test
public void testRemoteProcessGroupFingerprintRaw() throws Exception {
// Fill out every configuration.
final RemoteProcessGroup component = mock(RemoteProcessGroup.class);
when(component.getName()).thenReturn("name");
when(component.getIdentifier()).thenReturn("id");
when(component.getPosition()).thenReturn(new Position(10.5, 20.3));
when(component.getTargetUri()).thenReturn("http://node1:8080/nifi");
when(component.getTargetUris()).thenReturn("http://node1:8080/nifi, http://node2:8080/nifi");
when(component.getNetworkInterface()).thenReturn("eth0");
when(component.getComments()).thenReturn("comment");
when(component.getCommunicationsTimeout()).thenReturn("10 sec");
when(component.getYieldDuration()).thenReturn("30 sec");
when(component.getTransportProtocol()).thenReturn(SiteToSiteTransportProtocol.RAW);
when(component.getProxyHost()).thenReturn(null);
when(component.getProxyPort()).thenReturn(null);
when(component.getProxyUser()).thenReturn(null);
when(component.getProxyPassword()).thenReturn(null);
when(component.getVersionedComponentId()).thenReturn(Optional.empty());
// Assert fingerprints with expected one.
final String expected = "id" +
"NO_VALUE" +
"http://node1:8080/nifi, http://node2:8080/nifi" +
"eth0" +
"10 sec" +
"30 sec" +
"RAW" +
"NO_VALUE" +
"NO_VALUE" +
"NO_VALUE" +
"NO_VALUE";
final Element rootElement = serializeElement(encryptor, RemoteProcessGroup.class, component, "addRemoteProcessGroup", IDENTITY_LOOKUP);
final Element componentElement = (Element) rootElement.getElementsByTagName("remoteProcessGroup").item(0);
assertEquals(expected, fingerprint("addRemoteProcessGroupFingerprint", Element.class, componentElement));
}
@Test
public void testRemoteProcessGroupFingerprintWithProxy() throws Exception {
final String proxyPassword = "proxy-pass";
// Fill out every configuration.
final RemoteProcessGroup component = mock(RemoteProcessGroup.class);
when(component.getName()).thenReturn("name");
when(component.getIdentifier()).thenReturn("id");
when(component.getPosition()).thenReturn(new Position(10.5, 20.3));
when(component.getTargetUri()).thenReturn("http://node1:8080/nifi");
when(component.getTargetUris()).thenReturn("http://node1:8080/nifi, http://node2:8080/nifi");
when(component.getComments()).thenReturn("comment");
when(component.getCommunicationsTimeout()).thenReturn("10 sec");
when(component.getYieldDuration()).thenReturn("30 sec");
when(component.getTransportProtocol()).thenReturn(SiteToSiteTransportProtocol.HTTP);
when(component.getProxyHost()).thenReturn("proxy-host");
when(component.getProxyPort()).thenReturn(3128);
when(component.getProxyUser()).thenReturn("proxy-user");
when(component.getProxyPassword()).thenReturn(proxyPassword);
when(component.getVersionedComponentId()).thenReturn(Optional.empty());
final String hashedProxyPassword = sensitiveValueEncoder.getEncoded(proxyPassword);
// Assert fingerprints with expected one.
final String expected = "id" +
"NO_VALUE" +
"http://node1:8080/nifi, http://node2:8080/nifi" +
"NO_VALUE" +
"10 sec" +
"30 sec" +
"HTTP" +
"proxy-host" +
"3128" +
"proxy-user" +
hashedProxyPassword;
final Element rootElement = serializeElement(encryptor, RemoteProcessGroup.class, component, "addRemoteProcessGroup", IDENTITY_LOOKUP);
final Element componentElement = (Element) rootElement.getElementsByTagName("remoteProcessGroup").item(0);
assertEquals(expected, fingerprint("addRemoteProcessGroupFingerprint", Element.class, componentElement));
}
@Test
public void testRemotePortFingerprint() throws Exception {
// Fill out every configuration.
final RemoteProcessGroup groupComponent = mock(RemoteProcessGroup.class);
when(groupComponent.getName()).thenReturn("name");
when(groupComponent.getIdentifier()).thenReturn("id");
when(groupComponent.getPosition()).thenReturn(new Position(10.5, 20.3));
when(groupComponent.getTargetUri()).thenReturn("http://node1:8080/nifi");
when(groupComponent.getTransportProtocol()).thenReturn(SiteToSiteTransportProtocol.RAW);
when(groupComponent.getVersionedComponentId()).thenReturn(Optional.empty());
final RemoteGroupPort portComponent = mock(RemoteGroupPort.class);
when(groupComponent.getInputPorts()).thenReturn(Collections.singleton(portComponent));
when(portComponent.getName()).thenReturn("portName");
when(portComponent.getIdentifier()).thenReturn("portId");
when(portComponent.getPosition()).thenReturn(new Position(10.5, 20.3));
when(portComponent.getComments()).thenReturn("portComment");
when(portComponent.getScheduledState()).thenReturn(ScheduledState.RUNNING);
when(portComponent.getMaxConcurrentTasks()).thenReturn(3);
when(portComponent.isUseCompression()).thenReturn(true);
when(portComponent.getBatchCount()).thenReturn(1234);
when(portComponent.getBatchSize()).thenReturn("64KB");
when(portComponent.getBatchDuration()).thenReturn("10sec");
// Serializer doesn't serialize if a port doesn't have any connection.
when(portComponent.hasIncomingConnection()).thenReturn(true);
when(portComponent.getVersionedComponentId()).thenReturn(Optional.empty());
// Assert fingerprints with expected one.
final String expected = "portId" +
"NO_VALUE" +
"NO_VALUE" +
"3" +
"true" +
"1234" +
"64KB" +
"10sec";
final Element rootElement = serializeElement(encryptor, RemoteProcessGroup.class, groupComponent, "addRemoteProcessGroup", IDENTITY_LOOKUP);
final Element componentElement = (Element) rootElement.getElementsByTagName("inputPort").item(0);
assertEquals(expected, fingerprint("addRemoteGroupPortFingerprint", Element.class, componentElement));
}
@Test
public void testControllerServicesIncludedInGroupFingerprint() throws IOException {
final Document document = getDocument("src/test/resources/nifi/fingerprint/group-with-controller-services.xml");
final Element processGroup = document.getDocumentElement();
final StringBuilder sb = new StringBuilder();
fingerprintFactory.addProcessGroupFingerprint(sb, processGroup, new FlowEncodingVersion(1, 0));
final String fingerprint = sb.toString();
final String[] criticalFingerprintValues = new String[] {
"1234",
"s1", "service1", "prop1", "value1", "org.apache.nifi.services.FingerprintControllerService",
"s2", "service2", "another property", "another value", "org.apache.nifi.services.AnotherService",
};
for (final String criticalValue : criticalFingerprintValues) {
assertTrue( fingerprint.contains(criticalValue),
"Fingerprint did not contain '" + criticalValue + "'");
}
// Ensure that 's1' comes before 's2' in the fingerprint
assertTrue(fingerprint.indexOf("FingerprintControllerService") < fingerprint.indexOf("AnotherService"));
}
private PropertyEncryptor createEncryptor() {
return new PropertyEncryptor() {
@Override
public String encrypt(String property) {
return Hex.encodeHexString(property.getBytes(StandardCharsets.UTF_8));
}
@Override
public String decrypt(String encryptedProperty) {
try {
return new String(Hex.decodeHex(encryptedProperty));
} catch (DecoderException e) {
throw new IllegalArgumentException(e);
}
}
};
}
private SensitiveValueEncoder createSensitiveValueEncoder() {
return new SensitiveValueEncoder() {
@Override
public String getEncoded(String sensitivePropertyValue) {
return String.format("[MASKED] %s", sensitivePropertyValue);
}
};
}
}

2
nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/persistence/TestFlowConfigurationArchiveManager.java
View File

@ -45,7 +45,7 @@ import static org.mockito.Mockito.when;
public class TestFlowConfigurationArchiveManager {
private final File flowXmlFile = new File("./target/flow-archive/flow.xml.gz");
private final File flowXmlFile = new File("./target/flow-archive/flow.json.gz");
private final File archiveDir = new File("./target/flow-archive");
@BeforeEach

0
nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/resources/conf/0bytes.xml → nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/resources/conf/0bytes.json
View File

420
nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/resources/conf/all-flow-corrupt.json Normal file
View File

@ -0,0 +1,420 @@
{
"encodingVersion": {
"majorVersion": 2,
"minorVersion": 0
},
"maxTimerDrivenThreadCount": 10,
"maxEventDrivenThreadCount": 5,
"registries": [],
"parameterContexts": [],
"parameterProviders": [],
"controllerServices": [],
"reportingTasks": [],
"templates": [],
"rootGroup": {
"identifier": "e4e64ef8-9ad3-39d2-bfef-fc985d5ad4ef",
"instanceIdentifier": "4190002f-8b17-4772-bd62-8efec12c0cd0",
"name": "new group 1",
"comments": "new comments",
"position": {
"x": 0,
"y": 0
},
"processGroups": [
{
"identifier": "a698ede2-d790-30dd-87ef-ea1115f3f4ba",
"instanceIdentifier": "994be764-7e03-421c-ab10-f21956073c2a",
"name": "new gr",
"comments": " some comments ",
"position": {
"x": 0,
"y": 0
},
"processGroups": [],
"remoteProcessGroups": [],
"processors": [
{
"identifier": "ef027853-683e-3fe9-a625-be7e85c6ad5b",
"instanceIdentifier": "d489d825-67da-45eb-a714-98a811608daf",
"name": "new AttributeLoggerProcessor",
"comments": "",
"position": {
"x": 0,
"y": 0
},
"bundle": {
"group": "default",
"artifact": "unknown",
"version": "unversioned"
},
"properties": {
"log.level": "enc{null}"
},
"propertyDescriptors": {},
"style": {
"type": "org.apache.nifi.test.processors.StubAttributeLoggerProcessor",
"penaltyDuration": "30 sec",
"yieldDuration": "1 sec",
"bulletinLevel": "WARN",
"runDurationMillis": 0
},
"annotationData": "",
"schedulingStrategy": "TIMER_DRIVEN",
"executionNode": "ALL",
"penaltyDuration": "30 sec",
"yieldDuration": "1 sec",
"bulletinLevel": "WARN",
"runDurationMillis": 0,
"concurrentlySchedulableTaskCount": 1,
"autoTerminatedRelationships": [],
"scheduledState": "ENABLED",
"retryCount": 10,
"retriedRelationships": [],
"backoffMechanism": "PENALIZE_FLOWFILE",
"maxBackoffPeriod": "10 mins",
"componentType": "PROCESSOR",
"groupIdentifier": "a698ede2-d790-30dd-87ef-ea1115f3f4ba"
}
],
"inputPorts": [
{
"identifier": "12fbd345-8712-3dd1-b9a7-e702599e7025",
"instanceIdentifier": "167e3556-b6f2-4e9a-b143-e02c91bb7fb6",
"name": "Group In",
"position": {
"x": 0,
"y": 0
},
"type": "INPUT_PORT",
"concurrentlySchedulableTaskCount": 1,
"scheduledState": "ENABLED",
"allowRemoteAccess": false,
"componentType": "INPUT_PORT",
"groupIdentifier": "a698ede2-d790-30dd-87ef-ea1115f3f4ba"
}
],
"outputPorts": [
{
"identifier": "5a75b785-518a-34b8-bf4f-1df2dde51ca4",
"instanceIdentifier": "b14135ee-411d-46be-b377-5ed4c8d72cbe",
"name": "Group Out",
"position": {
"x": 0,
"y": 0
},
"type": "OUTPUT_PORT",
"concurrentlySchedulableTaskCount": 1,
"scheduledState": "ENABLED",
"allowRemoteAccess": false,
"componentType": "OUTPUT_PORT",
"groupIdentifier": "a698ede2-d790-30dd-87ef-ea1115f3f4ba"
}
],
"connections": [
{
"identifier": "6aa2eaf4-c256-3570-b90e-8162a061a942",
"instanceIdentifier": "b8886cad-373f-4ac4-8028-3968fdcd5252",
"name": "new name",
"source": {
"id": "12fbd345-8712-3dd1-b9a7-e702599e7025",
"type": "INPUT_PORT",
"groupId": "a698ede2-d790-30dd-87ef-ea1115f3f4ba",
"name": "Group In",
"instanceIdentifier": "167e3556-b6f2-4e9a-b143-e02c91bb7fb6"
},
"destination": {
"id": "5a75b785-518a-34b8-bf4f-1df2dde51ca4",
"type": "OUTPUT_PORT",
"groupId": "a698ede2-d790-30dd-87ef-ea1115f3f4ba",
"name": "Group Out",
"instanceIdentifier": "b14135ee-411d-46be-b377-5ed4c8d72cbe"
},
"labelIndex": 1,
"zIndex": 0,
"selectedRelationships": [
""
],
"backPressureObjectThreshold": 10000,
"backPressureDataSizeThreshold": "1 GB",
"flowFileExpiration": "0 sec",
"prioritizers": [],
"bends": [],
"loadBalanceStrategy": "DO_NOT_LOAD_BALANCE",
"loadBalanceCompression": "DO_NOT_COMPRESS",
"componentType": "CONNECTION",
"groupIdentifier": "a698ede2-d790-30dd-87ef-ea1115f3f4ba"
}
],
"labels": [],
"funnels": [],
"controllerServices": [],
"variables": {},
"defaultFlowFileExpiration": "0 sec",
"defaultBackPressureObjectThreshold": 10000,
"defaultBackPressureDataSizeThreshold": "1 GB",
"componentType": "PROCESS_GROUP",
"flowFileOutboundPolicy": "STREAM_WHEN_AVAILABLE",
"flowFileConcurrency": "UNBOUNDED",
"groupIdentifier": "e4e64ef8-9ad3-39d2-bfef-fc985d5ad4ef"
}
],
"remoteProcessGroups": [
{
"identifier": "230fc891-7a06-3db7-9a53-f133e9880e9a",
"instanceIdentifier": "52c47e32-91c5-40d4-bf0a-37d672ce8190",
"name": "new Unable to Connect",
"comments": "",
"position": {
"x": 0,
"y": 0
},
"targetUri": "http://localhost:8080",
"targetUris": "http://localhost:8080",
"communicationsTimeout": "30 sec",
"yieldDuration": "10 sec",
"transportProtocol": "RAW",
"inputPorts": [],
"outputPorts": [],
"componentType": "REMOTE_PROCESS_GROUP",
"groupIdentifier": "e4e64ef8-9ad3-39d2-bfef-fc985d5ad4ef"
}
],
"processors": [
{
"identifier": "eb5a37b5-2869-3d20-8920-a92e5fe9d425",
"instanceIdentifier": "1bc2d455-a45d-43fc-a96c-909b6a00311e",
"name": "AttributeLoggerProcessor",
"comments": "",
"position": {
"x": 0,
"y": 0
},
"type": "org.apache.nifi.test.processors.StubAttributeLoggerProcessor",
"bundle": {
"group": "default",
"artifact": "unknown",
"version": "unversioned"
},
"properties": {
"log.level": "enc{null}",
"log.payload": "enc{null}"
},
"propertyDescriptors": {},
"style": {},
"annotationData": "",
"schedulingStrategy": "TIMER_DRIVEN",
"executionNode": "ALL",
"penaltyDuration": "30 sec",
"yieldDuration": "1 sec",
"bulletinLevel": "WARN",
"runDurationMillis": 0,
"concurrentlySchedulableTaskCount": 1,
"autoTerminatedRelationships": [],
"scheduledState": "ENABLED",
"retryCount": 10,
"retriedRelationships": [],
"backoffMechanism": "PENALIZE_FLOWFILE",
"maxBackoffPeriod": "10 mins",
"componentType": "PROCESSOR",
"groupIdentifier": "e4e64ef8-9ad3-39d2-bfef-fc985d5ad4ef"
},
{
"identifier": "0e09a297-3fe6-30c1-b289-7e3652dd75a7",
"instanceIdentifier": "7fc60e56-c7cd-47bc-8290-bcd52b42b4ac",
"name": "new TerminationFileProcessor",
"position": {
"x": 0,
"y": 0
},
"type": "org.apache.nifi.test.processors.StubTerminationFileProcessor",
"bundle": {
"group": "default",
"artifact": "unknown",
"version": "unversioned"
},
"properties": {},
"propertyDescriptors": {},
"style": {},
"annotationData": "some annotation data",
"schedulingStrategy": "TIMER_DRIVEN",
"executionNode": "ALL",
"penaltyDuration": "30 sec",
"yieldDuration": "1 sec",
"bulletinLevel": "WARN",
"runDurationMillis": 0,
"concurrentlySchedulableTaskCount": 10,
"autoTerminatedRelationships": [],
"scheduledState": "ENABLED",
"retryCount": 10,
"retriedRelationships": [],
"backoffMechanism": "PENALIZE_FLOWFILE",
"maxBackoffPeriod": "10 mins",
"componentType": "PROCESSOR",
"groupIdentifier": "e4e64ef8-9ad3-39d2-bfef-fc985d5ad4ef"
}
],
"inputPorts": [
{
"identifier": "46ed6e5d-bcc9-3ad2-a88e-c5e779e30b08",
"instanceIdentifier": "e8cf322a-7855-442b-8478-b76520ae62a2",
"name": "new Remote In",
"position": {
"x": 0,
"y": 0
},
"type": "INPUT_PORT",
"concurrentlySchedulableTaskCount": 1,
"scheduledState": "ENABLED",
"allowRemoteAccess": true,
"componentType": "INPUT_PORT",
"groupIdentifier": "e4e64ef8-9ad3-39d2-bfef-fc985d5ad4ef"
}
],
"outputPorts": [
{
"identifier": "915ae973-0cbf-3001-a587-9e0e7f55dc55",
"instanceIdentifier": "069757e4-a3c9-474f-9593-c55516c510f8",
"name": "new Remote Out",
"position": {
"x": 0,
"y": 0
},
"type": "OUTPUT_PORT",
"concurrentlySchedulableTaskCount": 1,
"scheduledState": "ENABLED",
"allowRemoteAccess": true,
"componentType": "OUTPUT_PORT",
"groupIdentifier": "e4e64ef8-9ad3-39d2-bfef-fc985d5ad4ef"
}
],
"connections": [
{
"identifier": "48cd96e2-eeaf-3ae6-8f20-9a072007ab35",
"instanceIdentifier": "17a9cdd4-de3d-4d14-a57b-3dee8d503d3a",
"name": "",
"source": {
"id": "5a75b785-518a-34b8-bf4f-1df2dde51ca4",
"type": "OUTPUT_PORT",
"groupId": "a698ede2-d790-30dd-87ef-ea1115f3f4ba",
"name": "Group Out",
"instanceIdentifier": "b14135ee-411d-46be-b377-5ed4c8d72cbe"
},
"destination": {
"id": "7fc60e56-c7cd-47bc-8290-bcd52b42b42c",
"type": "PROCESSOR",
"groupId": "e4e64ef8-9ad3-39d2-bfef-fc985d5ad4ef",
"name": "new TerminationFileProcessor",
"instanceIdentifier": "7fc60e56-c7cd-47bc-8290-bcd52b42b4ac"
},
"labelIndex": 1,
"zIndex": 0,
"selectedRelationships": [
""
],
"backPressureObjectThreshold": 10000,
"backPressureDataSizeThreshold": "1 GB",
"flowFileExpiration": "0 sec",
"prioritizers": [],
"bends": [],
"loadBalanceStrategy": "DO_NOT_LOAD_BALANCE",
"loadBalanceCompression": "DO_NOT_COMPRESS",
"componentType": "CONNECTION",
"groupIdentifier": "e4e64ef8-9ad3-39d2-bfef-fc985d5ad4ef"
},
{
"identifier": "8d8d8473-cc5a-3b78-be36-494f983cb067",
"instanceIdentifier": "edba74b5-8bd9-44a0-b0d7-1adfc946e948",
"name": "",
"source": {
"id": "eb5a37b5-2869-3d20-8920-a92e5fe9d425",
"type": "PROCESSOR",
"groupId": "e4e64ef8-9ad3-39d2-bfef-fc985d5ad4ef",
"name": "AttributeLoggerProcessor",
"comments": "",
"instanceIdentifier": "1bc2d455-a45d-43fc-a96c-909b6a00311e"
},
"destination": {
"id": "12fbd345-8712-3dd1-b9a7-e702599e7025",
"type": "INPUT_PORT",
"groupId": "a698ede2-d790-30dd-87ef-ea1115f3f4ba",
"name": "Group In",
"instanceIdentifier": "167e3556-b6f2-4e9a-b143-e02c91bb7fb6"
},
"labelIndex": 1,
"zIndex": 0,
"selectedRelationships": [
"success"
],
"backPressureObjectThreshold": 10000,
"backPressureDataSizeThreshold": "1 GB",
"flowFileExpiration": "0 sec",
"prioritizers": [],
"bends": [],
"loadBalanceStrategy": "DO_NOT_LOAD_BALANCE",
"loadBalanceCompression": "DO_NOT_COMPRESS",
"componentType": "CONNECTION",
"groupIdentifier": "e4e64ef8-9ad3-39d2-bfef-fc985d5ad4ef"
},
{
"identifier": "2cf91cf2-69f6-33aa-9a9a-423c7d1a9e7c",
"instanceIdentifier": "cede7d64-040e-4951-bea4-04c100bf1064",
"name": "",
"source": {
"id": "46ed6e5d-bcc9-3ad2-a88e-c5e779e30b08",
"type": "INPUT_PORT",
"groupId": "e4e64ef8-9ad3-39d2-bfef-fc985d5ad4ef",
"name": "new Remote In",
"instanceIdentifier": "e8cf322a-7855-442b-8478-b76520ae62a2"
},
"destination": {
"id": "eb5a37b5-2869-3d20-8920-a92e5fe9d425",
"type": "PROCESSOR",
"groupId": "e4e64ef8-9ad3-39d2-bfef-fc985d5ad4ef",
"name": "AttributeLoggerProcessor",
"comments": "",
"instanceIdentifier": "1bc2d455-a45d-43fc-a96c-909b6a00311e"
},
"labelIndex": 1,
"zIndex": 0,
"selectedRelationships": [
""
],
"backPressureObjectThreshold": 10000,
"backPressureDataSizeThreshold": "1 GB",
"flowFileExpiration": "0 sec",
"prioritizers": [],
"bends": [],
"loadBalanceStrategy": "DO_NOT_LOAD_BALANCE",
"loadBalanceCompression": "DO_NOT_COMPRESS",
"componentType": "CONNECTION",
"groupIdentifier": "e4e64ef8-9ad3-39d2-bfef-fc985d5ad4ef"
}
],
"labels": [
{
"identifier": "e64f82ca-1ee0-37aa-a3e2-26e35b247f30",
"instanceIdentifier": "59a5a3ff-1b87-45c6-b5b4-307735baf2b7",
"position": {
"x": 0,
"y": 0
},
"label": "",
"zIndex": 0,
"width": 150,
"height": 150,
"style": {},
"componentType": "LABEL",
"groupIdentifier": "e4e64ef8-9ad3-39d2-bfef-fc985d5ad4ef"
}
],
"funnels": [],
"controllerServices": [],
"variables": {},
"defaultFlowFileExpiration": "0 sec",
"defaultBackPressureObjectThreshold": 10000,
"defaultBackPressureDataSizeThreshold": "1 GB",
"componentType": "PROCESS_GROUP",
"flowFileOutboundPolicy": "STREAM_WHEN_AVAILABLE",
"flowFileConcurrency": "UNBOUNDED"
}
}

201
nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/resources/conf/all-flow-corrupt.xml
View File

@ -1,201 +0,0 @@
<?xml version="1.0" encoding="UTF-8" standalone="no"?>
<!--
Licensed to the Apache Software Foundation (ASF) under one or more
contributor license agreements. See the NOTICE file distributed with
this work for additional information regarding copyright ownership.
The ASF licenses this file to You under the Apache License, Version 2.0
(the "License"); you may not use this file except in compliance with
the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
-->
<flowController>
<maxThreadCount>15</maxThreadCount>
<rootGroup>
<id>4190002f-8b17-4772-bd62-8efec12c0cd0</id>
<name>new group 1</name>
<position x="0.0" y="0.0"/>
<style>new style</style>
<comment>new comments</comment>
<processor>
<id>7fc60e56-c7cd-47bc-8290-bcd52b42b4ac</id>
<name>new TerminationFileProcessor</name>
<position x="0.0" y="0.0"/>
<style>new processor
<class>org.apache.nifi.test.processors.StubTerminationFileProcessor</class>
<maxConcurrentTasks>10</maxConcurrentTasks>
<schedulingPeriodSeconds>10</schedulingPeriodSeconds>
<maxWorkQueueSize>10</maxWorkQueueSize>
<lossTolerant>false</lossTolerant>
<running>true</running>
<flowFileExpirationMinutes>10</flowFileExpirationMinutes>
<queuePrioritizerClass>org.apache.nifi.comparator.NewestFlowFileFirstComparator</queuePrioritizerClass>
<annotationData>some annotation data</annotationData>
</processor>
<processor>
<id>1bc2d455-a45d-43fc-a96c-909b6a00311e</id>
<name>AttributeLoggerProcessor</name>
<position x="0.0" y="0.0"/>
<style>processor</style>
<comment/>
<class>org.apache.nifi.test.processors.StubAttributeLoggerProcessor</class>
<maxConcurrentTasks>1</maxConcurrentTasks>
<schedulingPeriodSeconds>0</schedulingPeriodSeconds>
<maxWorkQueueSize>0</maxWorkQueueSize>
<lossTolerant>false</lossTolerant>
<running>false</running>
<flowFileExpirationMinutes>0</flowFileExpirationMinutes>
<property>
<name>log.level</name>
<value>trace</value>
</property>
<property>
<name>log.payload</name>
<value>true</value>
</property>
<annotationData/>
</processor>
<inputPort>
<id>e8cf322a-7855-442b-8478-b76520ae62a2</id>
<name>new Remote In</name>
<position x="0.0" y="0.0"/>
<style>new input-port</style>
</inputPort>
<outputPort>
<id>069757e4-a3c9-474f-9593-c55516c510f8</id>
<name>new Remote Out</name>
<position x="0.0" y="0.0"/>
<style>new output-port</style>
<destination>
<hostname>localhost</hostname>
<controllerPort>8080</controllerPort>
<protocol>http</protocol>
<remotePortId>e8cf322a-7855-442b-8478-b76520ae62a2</remotePortId>
</destination>
</outputPort>
<label>
<id>59a5a3ff-1b87-45c6-b5b4-307735baf2b7</id>
<position x="0.0" y="0.0"/>
<style>new label</style>
<value></value>
</label>
<processGroup>
<id>994be764-7e03-421c-ab10-f21956073c2a</id>
<name>new gr</name>
<position x="0.0" y="0.0"/>
<style>new process-group</style>
<comment> some comments </comment>
<processor>
<id>d489d825-67da-45eb-a714-98a811608daf</id>
<name>new AttributeLoggerProcessor</name>
<position x="0.0" y="0.0"/>
<style>new processor</style>
<comment/>
<class>org.apache.nifi.test.processors.StubAttributeLoggerProcessor</class>
<maxConcurrentTasks>1</maxConcurrentTasks>
<schedulingPeriodSeconds>0</schedulingPeriodSeconds>
<maxWorkQueueSize>0</maxWorkQueueSize>
<lossTolerant>false</lossTolerant>
<running>false</running>
<flowFileExpirationMinutes>0</flowFileExpirationMinutes>
<property>
<name>log.level</name>
<value>debug</value>
</property>
<annotationData/>
</processor>
<inputPort>
<id>167e3556-b6f2-4e9a-b143-e02c91bb7fb6</id>
<name>Group In</name>
<position x="0.0" y="0.0"/>
<style>input-port</style>
</inputPort>
<outputPort>
<id>b14135ee-411d-46be-b377-5ed4c8d72cbe</id>
<name>Group Out</name>
<position x="0.0" y="0.0"/>
<style>output-port</style>
</outputPort>
<connection>
<id>b8886cad-373f-4ac4-8028-3968fdcd5252</id>
<name>new name</name>
<bendPoints/>
<labelIndex>1</labelIndex>
<zIndex>0</zIndex>
<style>new style</style>
<sourceId>167e3556-b6f2-4e9a-b143-e02c91bb7fb6</sourceId>
<sourceGroupId>994be764-7e03-421c-ab10-f21956073c2a</sourceGroupId>
<sourceType>INPUT_PORT</sourceType>
<destinationId>b14135ee-411d-46be-b377-5ed4c8d72cbe</destinationId>
<destinationGroupId>994be764-7e03-421c-ab10-f21956073c2a</destinationGroupId>
<destinationType>OUTPUT_PORT</destinationType>
<countFlowFiles>false</countFlowFiles>
<countBytes>false</countBytes>
<relationship/>
</connection>
</processGroup>
<remoteProcessGroup>
<id>52c47e32-91c5-40d4-bf0a-37d672ce8190</id>
<name>new Unable to Connect</name>
<position x="0.0" y="0.0"/>
<style>new remote-process-group</style>
<comment/>
<url>http://localhost:8080</url>
</remoteProcessGroup>
<connection>
<id>cede7d64-040e-4951-bea4-04c100bf1064</id>
<name/>
<bendPoints/>
<labelIndex>1</labelIndex>
<zIndex>0</zIndex>
<style/>
<sourceId>e8cf322a-7855-442b-8478-b76520ae62a2</sourceId>
<sourceGroupId>4190002f-8b17-4772-bd62-8efec12c0cd0</sourceGroupId>
<sourceType>INPUT_PORT</sourceType>
<destinationId>1bc2d455-a45d-43fc-a96c-909b6a00311e</destinationId>
<destinationGroupId>4190002f-8b17-4772-bd62-8efec12c0cd0</destinationGroupId>
<destinationType>PROCESSOR</destinationType>
<countFlowFiles>false</countFlowFiles>
<countBytes>false</countBytes>
<relationship/>
</connection>
<connection>
<id>edba74b5-8bd9-44a0-b0d7-1adfc946e948</id>
<name/>
<bendPoints/>
<labelIndex>1</labelIndex>
<zIndex>0</zIndex>
<style/>
<sourceId>1bc2d455-a45d-43fc-a96c-909b6a00311e</sourceId>
<sourceGroupId>4190002f-8b17-4772-bd62-8efec12c0cd0</sourceGroupId>
<sourceType>PROCESSOR</sourceType>
<destinationId>167e3556-b6f2-4e9a-b143-e02c91bb7fb6</destinationId>
<destinationGroupId>994be764-7e03-421c-ab10-f21956073c2a</destinationGroupId>
<destinationType>INPUT_PORT</destinationType>
<countFlowFiles>false</countFlowFiles>
<countBytes>false</countBytes>
<relationship>success</relationship>
</connection>
<connection>
<id>17a9cdd4-de3d-4d14-a57b-3dee8d503d3a</id>
<name/>
<bendPoints/>
<labelIndex>1</labelIndex>
<zIndex>0</zIndex>
<style/>
<sourceId>b14135ee-411d-46be-b377-5ed4c8d72cbe</sourceId>
<sourceGroupId>994be764-7e03-421c-ab10-f21956073c2a</sourceGroupId>
<sourceType>OUTPUT_PORT</sourceType>
<destinationId>7fc60e56-c7cd-47bc-8290-bcd52b42b42c</destinationId>
<destinationGroupId>4190002f-8b17-4772-bd62-8efec12c0cd0</destinationGroupId>
<destinationType>PROCESSOR</destinationType>
<countFlowFiles>false</countFlowFiles>
<countBytes>false</countBytes>
<relationship/>
</connection>
</rootGroup>
</flowController>

417
nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/resources/conf/all-flow-inheritable.json Normal file
View File

@ -0,0 +1,417 @@
{
"encodingVersion": {
"majorVersion": 2,
"minorVersion": 0
},
"maxTimerDrivenThreadCount": 10,
"maxEventDrivenThreadCount": 5,
"registries": [],
"parameterContexts": [],
"parameterProviders": [],
"controllerServices": [],
"reportingTasks": [],
"templates": [],
"rootGroup": {
"identifier": "e4e64ef8-9ad3-39d2-bfef-fc985d5ad4ef",
"instanceIdentifier": "4190002f-8b17-4772-bd62-8efec12c0cd0",
"name": "new group 1",
"comments": "new comments",
"position": {
"x": 0,
"y": 0
},
"processGroups": [
{
"identifier": "a698ede2-d790-30dd-87ef-ea1115f3f4ba",
"instanceIdentifier": "994be764-7e03-421c-ab10-f21956073c2a",
"name": "new gr",
"comments": " some comments ",
"position": {
"x": 0,
"y": 0
},
"processGroups": [],
"remoteProcessGroups": [],
"processors": [
{
"identifier": "ef027853-683e-3fe9-a625-be7e85c6ad5b",
"instanceIdentifier": "d489d825-67da-45eb-a714-98a811608daf",
"name": "new AttributeLoggerProcessor",
"comments": "",
"position": {
"x": 0,
"y": 0
},
"type": "org.apache.nifi.test.processors.StubAttributeLoggerProcessor",
"bundle": {
"group": "default",
"artifact": "unknown",
"version": "unversioned"
},
"properties": {
"log.level": "enc{null}"
},
"propertyDescriptors": {},
"style": {},
"annotationData": "",
"schedulingStrategy": "TIMER_DRIVEN",
"executionNode": "ALL",
"penaltyDuration": "30 sec",
"yieldDuration": "1 sec",
"bulletinLevel": "WARN",
"runDurationMillis": 0,
"concurrentlySchedulableTaskCount": 1,
"autoTerminatedRelationships": [],
"scheduledState": "ENABLED",
"retryCount": 10,
"retriedRelationships": [],
"backoffMechanism": "PENALIZE_FLOWFILE",
"maxBackoffPeriod": "10 mins",
"componentType": "PROCESSOR",
"groupIdentifier": "a698ede2-d790-30dd-87ef-ea1115f3f4ba"
}
],
"inputPorts": [
{
"identifier": "12fbd345-8712-3dd1-b9a7-e702599e7025",
"instanceIdentifier": "167e3556-b6f2-4e9a-b143-e02c91bb7fb6",
"name": "Group In",
"position": {
"x": 0,
"y": 0
},
"type": "INPUT_PORT",
"concurrentlySchedulableTaskCount": 1,
"scheduledState": "ENABLED",
"allowRemoteAccess": false,
"componentType": "INPUT_PORT",
"groupIdentifier": "a698ede2-d790-30dd-87ef-ea1115f3f4ba"
}
],
"outputPorts": [
{
"identifier": "5a75b785-518a-34b8-bf4f-1df2dde51ca4",
"instanceIdentifier": "b14135ee-411d-46be-b377-5ed4c8d72cbe",
"name": "Group Out",
"position": {
"x": 0,
"y": 0
},
"type": "OUTPUT_PORT",
"concurrentlySchedulableTaskCount": 1,
"scheduledState": "ENABLED",
"allowRemoteAccess": false,
"componentType": "OUTPUT_PORT",
"groupIdentifier": "a698ede2-d790-30dd-87ef-ea1115f3f4ba"
}
],
"connections": [
{
"identifier": "6aa2eaf4-c256-3570-b90e-8162a061a942",
"instanceIdentifier": "b8886cad-373f-4ac4-8028-3968fdcd5252",
"name": "new name",
"source": {
"id": "12fbd345-8712-3dd1-b9a7-e702599e7025",
"type": "INPUT_PORT",
"groupId": "a698ede2-d790-30dd-87ef-ea1115f3f4ba",
"name": "Group In",
"instanceIdentifier": "167e3556-b6f2-4e9a-b143-e02c91bb7fb6"
},
"destination": {
"id": "5a75b785-518a-34b8-bf4f-1df2dde51ca4",
"type": "OUTPUT_PORT",
"groupId": "a698ede2-d790-30dd-87ef-ea1115f3f4ba",
"name": "Group Out",
"instanceIdentifier": "b14135ee-411d-46be-b377-5ed4c8d72cbe"
},
"labelIndex": 1,
"zIndex": 0,
"selectedRelationships": [
""
],
"backPressureObjectThreshold": 10000,
"backPressureDataSizeThreshold": "1 GB",
"flowFileExpiration": "0 sec",
"prioritizers": [],
"bends": [],
"loadBalanceStrategy": "DO_NOT_LOAD_BALANCE",
"loadBalanceCompression": "DO_NOT_COMPRESS",
"componentType": "CONNECTION",
"groupIdentifier": "a698ede2-d790-30dd-87ef-ea1115f3f4ba"
}
],
"labels": [],
"funnels": [],
"controllerServices": [],
"variables": {},
"defaultFlowFileExpiration": "0 sec",
"defaultBackPressureObjectThreshold": 10000,
"defaultBackPressureDataSizeThreshold": "1 GB",
"componentType": "PROCESS_GROUP",
"flowFileOutboundPolicy": "STREAM_WHEN_AVAILABLE",
"flowFileConcurrency": "UNBOUNDED",
"groupIdentifier": "e4e64ef8-9ad3-39d2-bfef-fc985d5ad4ef"
}
],
"remoteProcessGroups": [
{
"identifier": "230fc891-7a06-3db7-9a53-f133e9880e9a",
"instanceIdentifier": "52c47e32-91c5-40d4-bf0a-37d672ce8190",
"name": "new Unable to Connect",
"comments": "",
"position": {
"x": 0,
"y": 0
},
"targetUri": "http://localhost:8080",
"targetUris": "http://localhost:8080",
"communicationsTimeout": "30 sec",
"yieldDuration": "10 sec",
"transportProtocol": "RAW",
"inputPorts": [],
"outputPorts": [],
"componentType": "REMOTE_PROCESS_GROUP",
"groupIdentifier": "e4e64ef8-9ad3-39d2-bfef-fc985d5ad4ef"
}
],
"processors": [
{
"identifier": "eb5a37b5-2869-3d20-8920-a92e5fe9d425",
"instanceIdentifier": "1bc2d455-a45d-43fc-a96c-909b6a00311e",
"name": "AttributeLoggerProcessor",
"comments": "",
"position": {
"x": 0,
"y": 0
},
"type": "org.apache.nifi.test.processors.StubAttributeLoggerProcessor",
"bundle": {
"group": "default",
"artifact": "unknown",
"version": "unversioned"
},
"properties": {
"log.level": "enc{null}",
"log.payload": "enc{null}"
},
"propertyDescriptors": {},
"style": {},
"annotationData": "",
"schedulingStrategy": "TIMER_DRIVEN",
"executionNode": "ALL",
"penaltyDuration": "30 sec",
"yieldDuration": "1 sec",
"bulletinLevel": "WARN",
"runDurationMillis": 0,
"concurrentlySchedulableTaskCount": 1,
"autoTerminatedRelationships": [],
"scheduledState": "ENABLED",
"retryCount": 10,
"retriedRelationships": [],
"backoffMechanism": "PENALIZE_FLOWFILE",
"maxBackoffPeriod": "10 mins",
"componentType": "PROCESSOR",
"groupIdentifier": "e4e64ef8-9ad3-39d2-bfef-fc985d5ad4ef"
},
{
"identifier": "d1469df6-219a-3e33-ab40-e00f7a85344f",
"instanceIdentifier": "7fc60e56-c7cd-47bc-8290-bcd52b42b42c",
"name": "new TerminationFileProcessor",
"comments": "",
"position": {
"x": 0,
"y": 0
},
"type": "org.apache.nifi.test.processors.StubTerminationFileProcessor",
"bundle": {
"group": "default",
"artifact": "unknown",
"version": "unversioned"
},
"properties": {},
"propertyDescriptors": {},
"style": {},
"annotationData": "some annotation data",
"schedulingStrategy": "TIMER_DRIVEN",
"executionNode": "ALL",
"penaltyDuration": "30 sec",
"yieldDuration": "1 sec",
"bulletinLevel": "WARN",
"runDurationMillis": 0,
"concurrentlySchedulableTaskCount": 10,
"autoTerminatedRelationships": [],
"scheduledState": "ENABLED",
"retryCount": 10,
"retriedRelationships": [],
"backoffMechanism": "PENALIZE_FLOWFILE",
"maxBackoffPeriod": "10 mins",
"componentType": "PROCESSOR",
"groupIdentifier": "e4e64ef8-9ad3-39d2-bfef-fc985d5ad4ef"
}
],
"inputPorts": [
{
"identifier": "46ed6e5d-bcc9-3ad2-a88e-c5e779e30b08",
"instanceIdentifier": "e8cf322a-7855-442b-8478-b76520ae62a2",
"name": "new Remote In",
"position": {
"x": 0,
"y": 0
},
"type": "INPUT_PORT",
"concurrentlySchedulableTaskCount": 1,
"scheduledState": "ENABLED",
"allowRemoteAccess": true,
"componentType": "INPUT_PORT",
"groupIdentifier": "e4e64ef8-9ad3-39d2-bfef-fc985d5ad4ef"
}
],
"outputPorts": [
{
"identifier": "915ae973-0cbf-3001-a587-9e0e7f55dc55",
"instanceIdentifier": "069757e4-a3c9-474f-9593-c55516c510f8",
"name": "new Remote Out",
"position": {
"x": 0,
"y": 0
},
"type": "OUTPUT_PORT",
"concurrentlySchedulableTaskCount": 1,
"scheduledState": "ENABLED",
"allowRemoteAccess": true,
"componentType": "OUTPUT_PORT",
"groupIdentifier": "e4e64ef8-9ad3-39d2-bfef-fc985d5ad4ef"
}
],
"connections": [
{
"identifier": "48cd96e2-eeaf-3ae6-8f20-9a072007ab35",
"instanceIdentifier": "17a9cdd4-de3d-4d14-a57b-3dee8d503d3a",
"name": "",
"source": {
"id": "5a75b785-518a-34b8-bf4f-1df2dde51ca4",
"type": "OUTPUT_PORT",
"groupId": "a698ede2-d790-30dd-87ef-ea1115f3f4ba",
"name": "Group Out",
"instanceIdentifier": "b14135ee-411d-46be-b377-5ed4c8d72cbe"
},
"destination": {
"id": "d1469df6-219a-3e33-ab40-e00f7a85344f",
"type": "PROCESSOR",
"groupId": "e4e64ef8-9ad3-39d2-bfef-fc985d5ad4ef",
"name": "new TerminationFileProcessor",
"comments": "",
"instanceIdentifier": "7fc60e56-c7cd-47bc-8290-bcd52b42b42c"
},
"labelIndex": 1,
"zIndex": 0,
"selectedRelationships": [
""
],
"backPressureObjectThreshold": 10000,
"backPressureDataSizeThreshold": "1 GB",
"flowFileExpiration": "0 sec",
"prioritizers": [],
"bends": [],
"loadBalanceStrategy": "DO_NOT_LOAD_BALANCE",
"loadBalanceCompression": "DO_NOT_COMPRESS",
"componentType": "CONNECTION",
"groupIdentifier": "e4e64ef8-9ad3-39d2-bfef-fc985d5ad4ef"
},
{
"identifier": "8d8d8473-cc5a-3b78-be36-494f983cb067",
"instanceIdentifier": "edba74b5-8bd9-44a0-b0d7-1adfc946e948",
"name": "",
"source": {
"id": "eb5a37b5-2869-3d20-8920-a92e5fe9d425",
"type": "PROCESSOR",
"groupId": "e4e64ef8-9ad3-39d2-bfef-fc985d5ad4ef",
"name": "AttributeLoggerProcessor",
"comments": "",
"instanceIdentifier": "1bc2d455-a45d-43fc-a96c-909b6a00311e"
},
"destination": {
"id": "12fbd345-8712-3dd1-b9a7-e702599e7025",
"type": "INPUT_PORT",
"groupId": "a698ede2-d790-30dd-87ef-ea1115f3f4ba",
"name": "Group In",
"instanceIdentifier": "167e3556-b6f2-4e9a-b143-e02c91bb7fb6"
},
"labelIndex": 1,
"zIndex": 0,
"selectedRelationships": [
"success"
],
"backPressureObjectThreshold": 10000,
"backPressureDataSizeThreshold": "1 GB",
"flowFileExpiration": "0 sec",
"prioritizers": [],
"bends": [],
"loadBalanceStrategy": "DO_NOT_LOAD_BALANCE",
"loadBalanceCompression": "DO_NOT_COMPRESS",
"componentType": "CONNECTION",
"groupIdentifier": "e4e64ef8-9ad3-39d2-bfef-fc985d5ad4ef"
},
{
"identifier": "2cf91cf2-69f6-33aa-9a9a-423c7d1a9e7c",
"instanceIdentifier": "cede7d64-040e-4951-bea4-04c100bf1064",
"name": "",
"source": {
"id": "46ed6e5d-bcc9-3ad2-a88e-c5e779e30b08",
"type": "INPUT_PORT",
"groupId": "e4e64ef8-9ad3-39d2-bfef-fc985d5ad4ef",
"name": "new Remote In",
"instanceIdentifier": "e8cf322a-7855-442b-8478-b76520ae62a2"
},
"destination": {
"id": "eb5a37b5-2869-3d20-8920-a92e5fe9d425",
"type": "PROCESSOR",
"groupId": "e4e64ef8-9ad3-39d2-bfef-fc985d5ad4ef",
"name": "AttributeLoggerProcessor",
"comments": "",
"instanceIdentifier": "1bc2d455-a45d-43fc-a96c-909b6a00311e"
},
"labelIndex": 1,
"zIndex": 0,
"selectedRelationships": [
""
],
"backPressureObjectThreshold": 10000,
"backPressureDataSizeThreshold": "1 GB",
"flowFileExpiration": "0 sec",
"prioritizers": [],
"bends": [],
"loadBalanceStrategy": "DO_NOT_LOAD_BALANCE",
"loadBalanceCompression": "DO_NOT_COMPRESS",
"componentType": "CONNECTION",
"groupIdentifier": "e4e64ef8-9ad3-39d2-bfef-fc985d5ad4ef"
}
],
"labels": [
{
"identifier": "e64f82ca-1ee0-37aa-a3e2-26e35b247f30",
"instanceIdentifier": "59a5a3ff-1b87-45c6-b5b4-307735baf2b7",
"position": {
"x": 0,
"y": 0
},
"label": "",
"zIndex": 0,
"width": 150,
"height": 150,
"style": {},
"componentType": "LABEL",
"groupIdentifier": "e4e64ef8-9ad3-39d2-bfef-fc985d5ad4ef"
}
],
"funnels": [],
"controllerServices": [],
"variables": {},
"defaultFlowFileExpiration": "0 sec",
"defaultBackPressureObjectThreshold": 10000,
"defaultBackPressureDataSizeThreshold": "1 GB",
"componentType": "PROCESS_GROUP",
"flowFileOutboundPolicy": "STREAM_WHEN_AVAILABLE",
"flowFileConcurrency": "UNBOUNDED"
}
}

196
nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/resources/conf/all-flow-inheritable.xml
View File

@ -1,196 +0,0 @@
<?xml version="1.0" encoding="UTF-8" standalone="no"?>
<!--
Licensed to the Apache Software Foundation (ASF) under one or more
contributor license agreements. See the NOTICE file distributed with
this work for additional information regarding copyright ownership.
The ASF licenses this file to You under the Apache License, Version 2.0
(the "License"); you may not use this file except in compliance with
the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
-->
<flowController>
<maxThreadCount>15</maxThreadCount>
<rootGroup>
<id>4190002f-8b17-4772-bd62-8efec12c0cd0</id>
<name>new group 1</name>
<position x="0.0" y="0.0"/>
<style>new style</style>
<comment>new comments</comment>
<processor>
<id>7fc60e56-c7cd-47bc-8290-bcd52b42b42c</id>
<name>new TerminationFileProcessor</name>
<position x="0.0" y="0.0"/>
<style>new processor</style>
<comment/>
<class>org.apache.nifi.test.processors.StubTerminationFileProcessor</class>
<maxConcurrentTasks>10</maxConcurrentTasks>
<schedulingPeriodSeconds>10</schedulingPeriodSeconds>
<maxWorkQueueSize>10</maxWorkQueueSize>
<lossTolerant>false</lossTolerant>
<running>true</running>
<flowFileExpirationMinutes>10</flowFileExpirationMinutes>
<queuePrioritizerClass>org.apache.nifi.comparator.NewestFlowFileFirstComparator</queuePrioritizerClass>
<annotationData>some annotation data</annotationData>
</processor>
<processor>
<id>1bc2d455-a45d-43fc-a96c-909b6a00311e</id>
<name>AttributeLoggerProcessor</name>
<position x="0.0" y="0.0"/>
<style>processor</style>
<comment/>
<class>org.apache.nifi.test.processors.StubAttributeLoggerProcessor</class>
<maxConcurrentTasks>1</maxConcurrentTasks>
<schedulingPeriodSeconds>0</schedulingPeriodSeconds>
<maxWorkQueueSize>0</maxWorkQueueSize>
<lossTolerant>false</lossTolerant>
<running>false</running>
<flowFileExpirationMinutes>0</flowFileExpirationMinutes>
<property>
<name>log.level</name>
<value>trace</value>
</property>
<property>
<name>log.payload</name>
<value>true</value>
</property>
<annotationData/>
</processor>
<inputPort>
<id>e8cf322a-7855-442b-8478-b76520ae62a2</id>
<name>new Remote In</name>
<position x="0.0" y="0.0"/>
<style>new input-port</style>
</inputPort>
<outputPort>
<id>069757e4-a3c9-474f-9593-c55516c510f8</id>
<name>new Remote Out</name>
<position x="0.0" y="0.0"/>
<style>new output-port</style>
</outputPort>
<label>
<id>59a5a3ff-1b87-45c6-b5b4-307735baf2b7</id>
<position x="0.0" y="0.0"/>
<style>new label</style>
<value></value>
</label>
<processGroup>
<id>994be764-7e03-421c-ab10-f21956073c2a</id>
<name>new gr</name>
<position x="0.0" y="0.0"/>
<style>new process-group</style>
<comment> some comments </comment>
<processor>
<id>d489d825-67da-45eb-a714-98a811608daf</id>
<name>new AttributeLoggerProcessor</name>
<position x="0.0" y="0.0"/>
<style>new processor</style>
<comment/>
<class>org.apache.nifi.test.processors.StubAttributeLoggerProcessor</class>
<maxConcurrentTasks>1</maxConcurrentTasks>
<schedulingPeriodSeconds>0</schedulingPeriodSeconds>
<maxWorkQueueSize>0</maxWorkQueueSize>
<lossTolerant>false</lossTolerant>
<running>false</running>
<flowFileExpirationMinutes>0</flowFileExpirationMinutes>
<property>
<name>log.level</name>
<value>debug</value>
</property>
<annotationData/>
</processor>
<inputPort>
<id>167e3556-b6f2-4e9a-b143-e02c91bb7fb6</id>
<name>Group In</name>
<position x="0.0" y="0.0"/>
<style>input-port</style>
</inputPort>
<outputPort>
<id>b14135ee-411d-46be-b377-5ed4c8d72cbe</id>
<name>Group Out</name>
<position x="0.0" y="0.0"/>
<style>output-port</style>
</outputPort>
<connection>
<id>b8886cad-373f-4ac4-8028-3968fdcd5252</id>
<name>new name</name>
<bendPoints/>
<labelIndex>1</labelIndex>
<zIndex>0</zIndex>
<style>new style</style>
<sourceId>167e3556-b6f2-4e9a-b143-e02c91bb7fb6</sourceId>
<sourceGroupId>994be764-7e03-421c-ab10-f21956073c2a</sourceGroupId>
<sourceType>INPUT_PORT</sourceType>
<destinationId>b14135ee-411d-46be-b377-5ed4c8d72cbe</destinationId>
<destinationGroupId>994be764-7e03-421c-ab10-f21956073c2a</destinationGroupId>
<destinationType>OUTPUT_PORT</destinationType>
<countFlowFiles>false</countFlowFiles>
<countBytes>false</countBytes>
<relationship/>
</connection>
</processGroup>
<remoteProcessGroup>
<id>52c47e32-91c5-40d4-bf0a-37d672ce8190</id>
<name>new Unable to Connect</name>
<position x="0.0" y="0.0"/>
<style>new remote-process-group</style>
<comment/>
<url>http://localhost:8080</url>
</remoteProcessGroup>
<connection>
<id>cede7d64-040e-4951-bea4-04c100bf1064</id>
<name/>
<bendPoints/>
<labelIndex>1</labelIndex>
<zIndex>0</zIndex>
<style/>
<sourceId>e8cf322a-7855-442b-8478-b76520ae62a2</sourceId>
<sourceGroupId>4190002f-8b17-4772-bd62-8efec12c0cd0</sourceGroupId>
<sourceType>INPUT_PORT</sourceType>
<destinationId>1bc2d455-a45d-43fc-a96c-909b6a00311e</destinationId>
<destinationGroupId>4190002f-8b17-4772-bd62-8efec12c0cd0</destinationGroupId>
<destinationType>PROCESSOR</destinationType>
<countFlowFiles>false</countFlowFiles>
<countBytes>false</countBytes>
<relationship/>
</connection>
<connection>
<id>edba74b5-8bd9-44a0-b0d7-1adfc946e948</id>
<name/>
<bendPoints/>
<labelIndex>1</labelIndex>
<zIndex>0</zIndex>
<style/>
<sourceId>1bc2d455-a45d-43fc-a96c-909b6a00311e</sourceId>
<sourceGroupId>4190002f-8b17-4772-bd62-8efec12c0cd0</sourceGroupId>
<sourceType>PROCESSOR</sourceType>
<destinationId>167e3556-b6f2-4e9a-b143-e02c91bb7fb6</destinationId>
<destinationGroupId>994be764-7e03-421c-ab10-f21956073c2a</destinationGroupId>
<destinationType>INPUT_PORT</destinationType>
<countFlowFiles>false</countFlowFiles>
<countBytes>false</countBytes>
<relationship>success</relationship>
</connection>
<connection>
<id>17a9cdd4-de3d-4d14-a57b-3dee8d503d3a</id>
<name/>
<bendPoints/>
<labelIndex>1</labelIndex>
<zIndex>0</zIndex>
<style/>
<sourceId>b14135ee-411d-46be-b377-5ed4c8d72cbe</sourceId>
<sourceGroupId>994be764-7e03-421c-ab10-f21956073c2a</sourceGroupId>
<sourceType>OUTPUT_PORT</sourceType>
<destinationId>7fc60e56-c7cd-47bc-8290-bcd52b42b42c</destinationId>
<destinationGroupId>4190002f-8b17-4772-bd62-8efec12c0cd0</destinationGroupId>
<destinationType>PROCESSOR</destinationType>
<countFlowFiles>false</countFlowFiles>
<countBytes>false</countBytes>
<relationship/>
</connection>
</rootGroup>
</flowController>

417
nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/resources/conf/all-flow-uninheritable.json Normal file
View File

@ -0,0 +1,417 @@
{
"encodingVersion": {
"majorVersion": 2,
"minorVersion": 0
},
"maxTimerDrivenThreadCount": 10,
"maxEventDrivenThreadCount": 5,
"registries": [],
"parameterContexts": [],
"parameterProviders": [],
"controllerServices": [],
"reportingTasks": [],
"templates": [],
"rootGroup": {
"identifier": "e4e64ef8-9ad3-39d2-bfef-fc985d5ad4ef",
"instanceIdentifier": "4190002f-8b17-4772-bd62-8efec12c0cd0",
"name": "new group 1",
"comments": "new comments",
"position": {
"x": 0,
"y": 0
},
"processGroups": [
{
"identifier": "a698ede2-d790-30dd-87ef-ea1115f3f4ba",
"instanceIdentifier": "994be764-7e03-421c-ab10-f21956073c2a",
"name": "new gr",
"comments": " some comments ",
"position": {
"x": 0,
"y": 0
},
"processGroups": [],
"remoteProcessGroups": [],
"processors": [
{
"identifier": "ef027853-683e-3fe9-a625-be7e85c6ad5b",
"instanceIdentifier": "d489d825-67da-45eb-a714-98a811608daf",
"name": "new AttributeLoggerProcessor",
"comments": "",
"position": {
"x": 0,
"y": 0
},
"type": "org.apache.nifi.test.processors.StubAttributeLoggerProcessor",
"bundle": {
"group": "default",
"artifact": "unknown",
"version": "unversioned"
},
"properties": {
"log.level": "enc{null}"
},
"propertyDescriptors": {},
"style": {},
"annotationData": "",
"schedulingStrategy": "TIMER_DRIVEN",
"executionNode": "ALL",
"penaltyDuration": "30 sec",
"yieldDuration": "1 sec",
"bulletinLevel": "WARN",
"runDurationMillis": 0,
"concurrentlySchedulableTaskCount": 1,
"autoTerminatedRelationships": [],
"scheduledState": "ENABLED",
"retryCount": 10,
"retriedRelationships": [],
"backoffMechanism": "PENALIZE_FLOWFILE",
"maxBackoffPeriod": "10 mins",
"componentType": "PROCESSOR",
"groupIdentifier": "a698ede2-d790-30dd-87ef-ea1115f3f4ba"
}
],
"inputPorts": [
{
"identifier": "12fbd345-8712-3dd1-b9a7-e702599e7025",
"instanceIdentifier": "167e3556-b6f2-4e9a-b143-e02c91bb7fb6",
"name": "Group In",
"position": {
"x": 0,
"y": 0
},
"type": "INPUT_PORT",
"concurrentlySchedulableTaskCount": 1,
"scheduledState": "ENABLED",
"allowRemoteAccess": false,
"componentType": "INPUT_PORT",
"groupIdentifier": "a698ede2-d790-30dd-87ef-ea1115f3f4ba"
}
],
"outputPorts": [
{
"identifier": "5a75b785-518a-34b8-bf4f-1df2dde51ca4",
"instanceIdentifier": "b14135ee-411d-46be-b377-5ed4c8d72cbe",
"name": "Group Out",
"position": {
"x": 0,
"y": 0
},
"type": "OUTPUT_PORT",
"concurrentlySchedulableTaskCount": 1,
"scheduledState": "ENABLED",
"allowRemoteAccess": false,
"componentType": "OUTPUT_PORT",
"groupIdentifier": "a698ede2-d790-30dd-87ef-ea1115f3f4ba"
}
],
"connections": [
{
"identifier": "6aa2eaf4-c256-3570-b90e-8162a061a942",
"instanceIdentifier": "b8886cad-373f-4ac4-8028-3968fdcd5252",
"name": "new name",
"source": {
"id": "12fbd345-8712-3dd1-b9a7-e702599e7025",
"type": "INPUT_PORT",
"groupId": "a698ede2-d790-30dd-87ef-ea1115f3f4ba",
"name": "Group In",
"instanceIdentifier": "167e3556-b6f2-4e9a-b143-e02c91bb7fb6"
},
"destination": {
"id": "5a75b785-518a-34b8-bf4f-1df2dde51ca4",
"type": "OUTPUT_PORT",
"groupId": "a698ede2-d790-30dd-87ef-ea1115f3f4ba",
"name": "Group Out",
"instanceIdentifier": "b14135ee-411d-46be-b377-5ed4c8d72cbe"
},
"labelIndex": 1,
"zIndex": 0,
"selectedRelationships": [
""
],
"backPressureObjectThreshold": 10000,
"backPressureDataSizeThreshold": "1 GB",
"flowFileExpiration": "0 sec",
"prioritizers": [],
"bends": [],
"loadBalanceStrategy": "DO_NOT_LOAD_BALANCE",
"loadBalanceCompression": "DO_NOT_COMPRESS",
"componentType": "CONNECTION",
"groupIdentifier": "a698ede2-d790-30dd-87ef-ea1115f3f4ba"
}
],
"labels": [],
"funnels": [],
"controllerServices": [],
"variables": {},
"defaultFlowFileExpiration": "0 sec",
"defaultBackPressureObjectThreshold": 10000,
"defaultBackPressureDataSizeThreshold": "1 GB",
"componentType": "PROCESS_GROUP",
"flowFileConcurrency": "UNBOUNDED",
"flowFileOutboundPolicy": "STREAM_WHEN_AVAILABLE",
"groupIdentifier": "e4e64ef8-9ad3-39d2-bfef-fc985d5ad4ef"
}
],
"remoteProcessGroups": [
{
"identifier": "230fc891-7a06-3db7-9a53-f133e9880e9a",
"instanceIdentifier": "52c47e32-91c5-40d4-bf0a-37d672ce8190",
"name": "new Unable to Connect",
"comments": "",
"position": {
"x": 0,
"y": 0
},
"targetUri": "http://localhost:8080",
"targetUris": "http://localhost:8080",
"communicationsTimeout": "30 sec",
"yieldDuration": "10 sec",
"transportProtocol": "RAW",
"inputPorts": [],
"outputPorts": [],
"componentType": "REMOTE_PROCESS_GROUP",
"groupIdentifier": "e4e64ef8-9ad3-39d2-bfef-fc985d5ad4ef"
}
],
"processors": [
{
"identifier": "eb5a37b5-2869-3d20-8920-a92e5fe9d425",
"instanceIdentifier": "1bc2d455-a45d-43fc-a96c-909b6a00311e",
"name": "AttributeLoggerProcessor",
"comments": "",
"position": {
"x": 0,
"y": 0
},
"type": "org.apache.nifi.test.processors.StubAttributeLoggerProcessor",
"bundle": {
"group": "default",
"artifact": "unknown",
"version": "unversioned"
},
"properties": {
"log.level": "enc{null}",
"log.payload": "enc{null}"
},
"propertyDescriptors": {},
"style": {},
"annotationData": "",
"schedulingStrategy": "TIMER_DRIVEN",
"executionNode": "ALL",
"penaltyDuration": "30 sec",
"yieldDuration": "1 sec",
"bulletinLevel": "WARN",
"runDurationMillis": 0,
"concurrentlySchedulableTaskCount": 1,
"autoTerminatedRelationships": [],
"scheduledState": "ENABLED",
"retryCount": 10,
"retriedRelationships": [],
"backoffMechanism": "PENALIZE_FLOWFILE",
"maxBackoffPeriod": "10 mins",
"componentType": "PROCESSOR",
"groupIdentifier": "e4e64ef8-9ad3-39d2-bfef-fc985d5ad4ef"
},
{
"identifier": "0e09a297-3fe6-30c1-b289-7e3652dd75a7",
"instanceIdentifier": "7fc60e56-c7cd-47bc-8290-bcd52b42b4ac",
"name": "new TerminationFileProcessor",
"comments": "",
"position": {
"x": 0,
"y": 0
},
"type": "org.apache.nifi.test.processors.StubTerminationFileProcessor",
"bundle": {
"group": "default",
"artifact": "unknown",
"version": "unversioned"
},
"properties": {},
"propertyDescriptors": {},
"style": {},
"annotationData": "some annotation data",
"schedulingStrategy": "TIMER_DRIVEN",
"executionNode": "ALL",
"penaltyDuration": "30 sec",
"yieldDuration": "1 sec",
"bulletinLevel": "WARN",
"runDurationMillis": 0,
"concurrentlySchedulableTaskCount": 10,
"autoTerminatedRelationships": [],
"scheduledState": "ENABLED",
"retryCount": 10,
"retriedRelationships": [],
"backoffMechanism": "PENALIZE_FLOWFILE",
"maxBackoffPeriod": "10 mins",
"componentType": "PROCESSOR",
"groupIdentifier": "e4e64ef8-9ad3-39d2-bfef-fc985d5ad4ef"
}
],
"inputPorts": [
{
"identifier": "46ed6e5d-bcc9-3ad2-a88e-c5e779e30b08",
"instanceIdentifier": "e8cf322a-7855-442b-8478-b76520ae62a2",
"name": "new Remote In",
"position": {
"x": 0,
"y": 0
},
"type": "INPUT_PORT",
"concurrentlySchedulableTaskCount": 1,
"scheduledState": "ENABLED",
"allowRemoteAccess": true,
"componentType": "INPUT_PORT",
"groupIdentifier": "e4e64ef8-9ad3-39d2-bfef-fc985d5ad4ef"
}
],
"outputPorts": [
{
"identifier": "915ae973-0cbf-3001-a587-9e0e7f55dc55",
"instanceIdentifier": "069757e4-a3c9-474f-9593-c55516c510f8",
"name": "new Remote Out",
"position": {
"x": 0,
"y": 0
},
"type": "OUTPUT_PORT",
"concurrentlySchedulableTaskCount": 1,
"scheduledState": "ENABLED",
"allowRemoteAccess": true,
"componentType": "OUTPUT_PORT",
"groupIdentifier": "e4e64ef8-9ad3-39d2-bfef-fc985d5ad4ef"
}
],
"connections": [
{
"identifier": "48cd96e2-eeaf-3ae6-8f20-9a072007ab35",
"instanceIdentifier": "17a9cdd4-de3d-4d14-a57b-3dee8d503d3a",
"name": "",
"source": {
"id": "5a75b785-518a-34b8-bf4f-1df2dde51ca4",
"type": "OUTPUT_PORT",
"groupId": "a698ede2-d790-30dd-87ef-ea1115f3f4ba",
"name": "Group Out",
"instanceIdentifier": "b14135ee-411d-46be-b377-5ed4c8d72cbe"
},
"destination": {
"id": "0e09a297-3fe6-30c1-b289-7e3652dd75a7",
"type": "PROCESSOR",
"groupId": "e4e64ef8-9ad3-39d2-bfef-fc985d5ad4ef",
"name": "new TerminationFileProcessor",
"comments": "",
"instanceIdentifier": "7fc60e56-c7cd-47bc-8290-bcd52b42b42c"
},
"labelIndex": 1,
"zIndex": 0,
"selectedRelationships": [
""
],
"backPressureObjectThreshold": 10000,
"backPressureDataSizeThreshold": "1 GB",
"flowFileExpiration": "0 sec",
"prioritizers": [],
"bends": [],
"loadBalanceStrategy": "DO_NOT_LOAD_BALANCE",
"loadBalanceCompression": "DO_NOT_COMPRESS",
"componentType": "CONNECTION",
"groupIdentifier": "e4e64ef8-9ad3-39d2-bfef-fc985d5ad4ef"
},
{
"identifier": "8d8d8473-cc5a-3b78-be36-494f983cb067",
"instanceIdentifier": "edba74b5-8bd9-44a0-b0d7-1adfc946e948",
"name": "",
"source": {
"id": "eb5a37b5-2869-3d20-8920-a92e5fe9d425",
"type": "PROCESSOR",
"groupId": "e4e64ef8-9ad3-39d2-bfef-fc985d5ad4ef",
"name": "AttributeLoggerProcessor",
"comments": "",
"instanceIdentifier": "1bc2d455-a45d-43fc-a96c-909b6a00311e"
},
"destination": {
"id": "12fbd345-8712-3dd1-b9a7-e702599e7025",
"type": "INPUT_PORT",
"groupId": "a698ede2-d790-30dd-87ef-ea1115f3f4ba",
"name": "Group In",
"instanceIdentifier": "167e3556-b6f2-4e9a-b143-e02c91bb7fb6"
},
"labelIndex": 1,
"zIndex": 0,
"selectedRelationships": [
"success"
],
"backPressureObjectThreshold": 10000,
"backPressureDataSizeThreshold": "1 GB",
"flowFileExpiration": "0 sec",
"prioritizers": [],
"bends": [],
"loadBalanceStrategy": "DO_NOT_LOAD_BALANCE",
"loadBalanceCompression": "DO_NOT_COMPRESS",
"componentType": "CONNECTION",
"groupIdentifier": "e4e64ef8-9ad3-39d2-bfef-fc985d5ad4ef"
},
{
"identifier": "2cf91cf2-69f6-33aa-9a9a-423c7d1a9e7c",
"instanceIdentifier": "cede7d64-040e-4951-bea4-04c100bf1064",
"name": "",
"source": {
"id": "46ed6e5d-bcc9-3ad2-a88e-c5e779e30b08",
"type": "INPUT_PORT",
"groupId": "e4e64ef8-9ad3-39d2-bfef-fc985d5ad4ef",
"name": "new Remote In",
"instanceIdentifier": "e8cf322a-7855-442b-8478-b76520ae62a2"
},
"destination": {
"id": "eb5a37b5-2869-3d20-8920-a92e5fe9d425",
"type": "PROCESSOR",
"groupId": "e4e64ef8-9ad3-39d2-bfef-fc985d5ad4ef",
"name": "AttributeLoggerProcessor",
"comments": "",
"instanceIdentifier": "1bc2d455-a45d-43fc-a96c-909b6a00311e"
},
"labelIndex": 1,
"zIndex": 0,
"selectedRelationships": [
""
],
"backPressureObjectThreshold": 10000,
"backPressureDataSizeThreshold": "1 GB",
"flowFileExpiration": "0 sec",
"prioritizers": [],
"bends": [],
"loadBalanceStrategy": "DO_NOT_LOAD_BALANCE",
"loadBalanceCompression": "DO_NOT_COMPRESS",
"componentType": "CONNECTION",
"groupIdentifier": "e4e64ef8-9ad3-39d2-bfef-fc985d5ad4ef"
}
],
"labels": [
{
"identifier": "e64f82ca-1ee0-37aa-a3e2-26e35b247f30",
"instanceIdentifier": "59a5a3ff-1b87-45c6-b5b4-307735baf2b7",
"position": {
"x": 0,
"y": 0
},
"label": "",
"zIndex": 0,
"width": 150,
"height": 150,
"style": {},
"componentType": "LABEL",
"groupIdentifier": "e4e64ef8-9ad3-39d2-bfef-fc985d5ad4ef"
}
],
"funnels": [],
"controllerServices": [],
"variables": {},
"defaultFlowFileExpiration": "0 sec",
"defaultBackPressureObjectThreshold": 10000,
"defaultBackPressureDataSizeThreshold": "1 GB",
"componentType": "PROCESS_GROUP",
"flowFileConcurrency": "UNBOUNDED",
"flowFileOutboundPolicy": "STREAM_WHEN_AVAILABLE"
}
}

202
nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/resources/conf/all-flow-uninheritable.xml
View File

@ -1,202 +0,0 @@
<?xml version="1.0" encoding="UTF-8" standalone="no"?>
<!--
Licensed to the Apache Software Foundation (ASF) under one or more
contributor license agreements. See the NOTICE file distributed with
this work for additional information regarding copyright ownership.
The ASF licenses this file to You under the Apache License, Version 2.0
(the "License"); you may not use this file except in compliance with
the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
-->
<flowController>
<maxThreadCount>15</maxThreadCount>
<rootGroup>
<id>4190002f-8b17-4772-bd62-8efec12c0cd0</id>
<name>new group 1</name>
<position x="0.0" y="0.0"/>
<style>new style</style>
<comment>new comments</comment>
<processor>
<id>7fc60e56-c7cd-47bc-8290-bcd52b42b4ac</id>
<name>new TerminationFileProcessor</name>
<position x="0.0" y="0.0"/>
<style>new processor</style>
<comment/>
<class>org.apache.nifi.test.processors.StubTerminationFileProcessor</class>
<maxConcurrentTasks>10</maxConcurrentTasks>
<schedulingPeriodSeconds>10</schedulingPeriodSeconds>
<maxWorkQueueSize>10</maxWorkQueueSize>
<lossTolerant>false</lossTolerant>
<running>true</running>
<flowFileExpirationMinutes>10</flowFileExpirationMinutes>
<queuePrioritizerClass>org.apache.nifi.comparator.NewestFlowFileFirstComparator</queuePrioritizerClass>
<annotationData>some annotation data</annotationData>
</processor>
<processor>
<id>1bc2d455-a45d-43fc-a96c-909b6a00311e</id>
<name>AttributeLoggerProcessor</name>
<position x="0.0" y="0.0"/>
<style>processor</style>
<comment/>
<class>org.apache.nifi.test.processors.StubAttributeLoggerProcessor</class>
<maxConcurrentTasks>1</maxConcurrentTasks>
<schedulingPeriodSeconds>0</schedulingPeriodSeconds>
<maxWorkQueueSize>0</maxWorkQueueSize>
<lossTolerant>false</lossTolerant>
<running>false</running>
<flowFileExpirationMinutes>0</flowFileExpirationMinutes>
<property>
<name>log.level</name>
<value>trace</value>
</property>
<property>
<name>log.payload</name>
<value>true</value>
</property>
<annotationData/>
</processor>
<inputPort>
<id>e8cf322a-7855-442b-8478-b76520ae62a2</id>
<name>new Remote In</name>
<position x="0.0" y="0.0"/>
<style>new input-port</style>
</inputPort>
<outputPort>
<id>069757e4-a3c9-474f-9593-c55516c510f8</id>
<name>new Remote Out</name>
<position x="0.0" y="0.0"/>
<style>new output-port</style>
<destination>
<hostname>localhost</hostname>
<controllerPort>8080</controllerPort>
<protocol>http</protocol>
<remotePortId>e8cf322a-7855-442b-8478-b76520ae62a2</remotePortId>
</destination>
</outputPort>
<label>
<id>59a5a3ff-1b87-45c6-b5b4-307735baf2b7</id>
<position x="0.0" y="0.0"/>
<style>new label</style>
<value></value>
</label>
<processGroup>
<id>994be764-7e03-421c-ab10-f21956073c2a</id>
<name>new gr</name>
<position x="0.0" y="0.0"/>
<style>new process-group</style>
<comment> some comments </comment>
<processor>
<id>d489d825-67da-45eb-a714-98a811608daf</id>
<name>new AttributeLoggerProcessor</name>
<position x="0.0" y="0.0"/>
<style>new processor</style>
<comment/>
<class>org.apache.nifi.test.processors.StubAttributeLoggerProcessor</class>
<maxConcurrentTasks>1</maxConcurrentTasks>
<schedulingPeriodSeconds>0</schedulingPeriodSeconds>
<maxWorkQueueSize>0</maxWorkQueueSize>
<lossTolerant>false</lossTolerant>
<running>false</running>
<flowFileExpirationMinutes>0</flowFileExpirationMinutes>
<property>
<name>log.level</name>
<value>debug</value>
</property>
<annotationData/>
</processor>
<inputPort>
<id>167e3556-b6f2-4e9a-b143-e02c91bb7fb6</id>
<name>Group In</name>
<position x="0.0" y="0.0"/>
<style>input-port</style>
</inputPort>
<outputPort>
<id>b14135ee-411d-46be-b377-5ed4c8d72cbe</id>
<name>Group Out</name>
<position x="0.0" y="0.0"/>
<style>output-port</style>
</outputPort>
<connection>
<id>b8886cad-373f-4ac4-8028-3968fdcd5252</id>
<name>new name</name>
<bendPoints/>
<labelIndex>1</labelIndex>
<zIndex>0</zIndex>
<style>new style</style>
<sourceId>167e3556-b6f2-4e9a-b143-e02c91bb7fb6</sourceId>
<sourceGroupId>994be764-7e03-421c-ab10-f21956073c2a</sourceGroupId>
<sourceType>INPUT_PORT</sourceType>
<destinationId>b14135ee-411d-46be-b377-5ed4c8d72cbe</destinationId>
<destinationGroupId>994be764-7e03-421c-ab10-f21956073c2a</destinationGroupId>
<destinationType>OUTPUT_PORT</destinationType>
<countFlowFiles>false</countFlowFiles>
<countBytes>false</countBytes>
<relationship/>
</connection>
</processGroup>
<remoteProcessGroup>
<id>52c47e32-91c5-40d4-bf0a-37d672ce8190</id>
<name>new Unable to Connect</name>
<position x="0.0" y="0.0"/>
<style>new remote-process-group</style>
<comment/>
<url>http://localhost:8080</url>
</remoteProcessGroup>
<connection>
<id>cede7d64-040e-4951-bea4-04c100bf1064</id>
<name/>
<bendPoints/>
<labelIndex>1</labelIndex>
<zIndex>0</zIndex>
<style/>
<sourceId>e8cf322a-7855-442b-8478-b76520ae62a2</sourceId>
<sourceGroupId>4190002f-8b17-4772-bd62-8efec12c0cd0</sourceGroupId>
<sourceType>INPUT_PORT</sourceType>
<destinationId>1bc2d455-a45d-43fc-a96c-909b6a00311e</destinationId>
<destinationGroupId>4190002f-8b17-4772-bd62-8efec12c0cd0</destinationGroupId>
<destinationType>PROCESSOR</destinationType>
<countFlowFiles>false</countFlowFiles>
<countBytes>false</countBytes>
<relationship/>
</connection>
<connection>
<id>edba74b5-8bd9-44a0-b0d7-1adfc946e948</id>
<name/>
<bendPoints/>
<labelIndex>1</labelIndex>
<zIndex>0</zIndex>
<style/>
<sourceId>1bc2d455-a45d-43fc-a96c-909b6a00311e</sourceId>
<sourceGroupId>4190002f-8b17-4772-bd62-8efec12c0cd0</sourceGroupId>
<sourceType>PROCESSOR</sourceType>
<destinationId>167e3556-b6f2-4e9a-b143-e02c91bb7fb6</destinationId>
<destinationGroupId>994be764-7e03-421c-ab10-f21956073c2a</destinationGroupId>
<destinationType>INPUT_PORT</destinationType>
<countFlowFiles>false</countFlowFiles>
<countBytes>false</countBytes>
<relationship>success</relationship>
</connection>
<connection>
<id>17a9cdd4-de3d-4d14-a57b-3dee8d503d3a</id>
<name/>
<bendPoints/>
<labelIndex>1</labelIndex>
<zIndex>0</zIndex>
<style/>
<sourceId>b14135ee-411d-46be-b377-5ed4c8d72cbe</sourceId>
<sourceGroupId>994be764-7e03-421c-ab10-f21956073c2a</sourceGroupId>
<sourceType>OUTPUT_PORT</sourceType>
<destinationId>7fc60e56-c7cd-47bc-8290-bcd52b42b42c</destinationId>
<destinationGroupId>4190002f-8b17-4772-bd62-8efec12c0cd0</destinationGroupId>
<destinationType>PROCESSOR</destinationType>
<countFlowFiles>false</countFlowFiles>
<countBytes>false</countBytes>
<relationship/>
</connection>
</rootGroup>
</flowController>

Some files were not shown because too many files have changed in this diff Show More