diff --git a/minifi/minifi-bootstrap/src/main/java/org/apache/nifi/minifi/bootstrap/service/MiNiFiConfigurationChangeListener.java b/minifi/minifi-bootstrap/src/main/java/org/apache/nifi/minifi/bootstrap/service/MiNiFiConfigurationChangeListener.java index f043f6d5a1..b5d30ffe09 100644 --- a/minifi/minifi-bootstrap/src/main/java/org/apache/nifi/minifi/bootstrap/service/MiNiFiConfigurationChangeListener.java +++ b/minifi/minifi-bootstrap/src/main/java/org/apache/nifi/minifi/bootstrap/service/MiNiFiConfigurationChangeListener.java @@ -58,11 +58,6 @@ public class MiNiFiConfigurationChangeListener implements ConfigurationChangeLis this.flowEnrichService = flowEnrichService; } - @Override - public String getDescriptor() { - return "MiNiFiConfigurationChangeListener"; - } - @Override public void handleChange(InputStream flowConfigInputStream) throws ConfigurationChangeException { logger.info("Received notification of a change"); @@ -107,6 +102,11 @@ public class MiNiFiConfigurationChangeListener implements ConfigurationChangeLis } } + @Override + public String getDescriptor() { + return "MiNiFiConfigurationChangeListener"; + } + private void setActiveFlowReference(ByteBuffer flowConfig) { logger.debug("Setting active flow reference {} with content:\n{}", flowConfig, new String(flowConfig.array(), UTF_8)); runner.getConfigFileReference().set(flowConfig); diff --git a/minifi/minifi-bootstrap/src/main/java/org/apache/nifi/minifi/bootstrap/service/MiNiFiPropertiesGenerator.java b/minifi/minifi-bootstrap/src/main/java/org/apache/nifi/minifi/bootstrap/service/MiNiFiPropertiesGenerator.java index 53e39e1839..16a393d0c9 100644 --- a/minifi/minifi-bootstrap/src/main/java/org/apache/nifi/minifi/bootstrap/service/MiNiFiPropertiesGenerator.java +++ b/minifi/minifi-bootstrap/src/main/java/org/apache/nifi/minifi/bootstrap/service/MiNiFiPropertiesGenerator.java @@ -137,11 +137,11 @@ public class MiNiFiPropertiesGenerator { Triple.of(NiFiProperties.SECURITY_OCSP_RESPONDER_URL, EMPTY, EMPTY), Triple.of(NiFiProperties.SECURITY_OCSP_RESPONDER_CERTIFICATE, EMPTY, EMPTY), Triple.of(NiFiProperties.CLUSTER_IS_NODE, "false", EMPTY), - Triple.of(NiFiProperties.FLOW_CONFIGURATION_FILE, "./conf/flow.xml.gz", EMPTY) + Triple.of(NiFiProperties.FLOW_CONFIGURATION_FILE, "./conf/flow.json.gz", EMPTY) ); static final Map MINIFI_TO_NIFI_PROPERTY_MAPPING = Map.of( - MiNiFiProperties.NIFI_MINIFI_FLOW_CONFIG.getKey(), NiFiProperties.FLOW_CONFIGURATION_JSON_FILE, + MiNiFiProperties.NIFI_MINIFI_FLOW_CONFIG.getKey(), NiFiProperties.FLOW_CONFIGURATION_FILE, MiNiFiProperties.NIFI_MINIFI_SECURITY_KEYSTORE.getKey(), NiFiProperties.SECURITY_KEYSTORE, MiNiFiProperties.NIFI_MINIFI_SECURITY_KEYSTORE_TYPE.getKey(), NiFiProperties.SECURITY_KEYSTORE_TYPE, MiNiFiProperties.NIFI_MINIFI_SECURITY_KEYSTORE_PASSWD.getKey(), NiFiProperties.SECURITY_KEYSTORE_PASSWD, diff --git a/minifi/minifi-bootstrap/src/test/resources/MINIFI-216/nifi.properties.before b/minifi/minifi-bootstrap/src/test/resources/MINIFI-216/nifi.properties.before index 4f0e0b25e0..8df7e966ca 100644 --- a/minifi/minifi-bootstrap/src/test/resources/MINIFI-216/nifi.properties.before +++ b/minifi/minifi-bootstrap/src/test/resources/MINIFI-216/nifi.properties.before @@ -15,7 +15,7 @@ # Core Properties # -nifi.flow.configuration.file=./conf/flow.xml.gz +nifi.flow.configuration.file=./conf/flow.json.gz nifi.flow.configuration.archive.enabled=false nifi.flow.configuration.archive.dir=./conf/archive/ nifi.flowcontroller.autoResumeState=true diff --git a/minifi/minifi-bootstrap/src/test/resources/MINIFI-245/nifi.properties.before b/minifi/minifi-bootstrap/src/test/resources/MINIFI-245/nifi.properties.before index 4f0e0b25e0..8df7e966ca 100644 --- a/minifi/minifi-bootstrap/src/test/resources/MINIFI-245/nifi.properties.before +++ b/minifi/minifi-bootstrap/src/test/resources/MINIFI-245/nifi.properties.before @@ -15,7 +15,7 @@ # Core Properties # -nifi.flow.configuration.file=./conf/flow.xml.gz +nifi.flow.configuration.file=./conf/flow.json.gz nifi.flow.configuration.archive.enabled=false nifi.flow.configuration.archive.dir=./conf/archive/ nifi.flowcontroller.autoResumeState=true diff --git a/minifi/minifi-bootstrap/src/test/resources/MINIFI-277/nifi.properties b/minifi/minifi-bootstrap/src/test/resources/MINIFI-277/nifi.properties index 4f0e0b25e0..8df7e966ca 100644 --- a/minifi/minifi-bootstrap/src/test/resources/MINIFI-277/nifi.properties +++ b/minifi/minifi-bootstrap/src/test/resources/MINIFI-277/nifi.properties @@ -15,7 +15,7 @@ # Core Properties # -nifi.flow.configuration.file=./conf/flow.xml.gz +nifi.flow.configuration.file=./conf/flow.json.gz nifi.flow.configuration.archive.enabled=false nifi.flow.configuration.archive.dir=./conf/archive/ nifi.flowcontroller.autoResumeState=true diff --git a/minifi/minifi-bootstrap/src/test/resources/NIFI-8753/nifi.properties.before b/minifi/minifi-bootstrap/src/test/resources/NIFI-8753/nifi.properties.before index 4f0e0b25e0..8df7e966ca 100644 --- a/minifi/minifi-bootstrap/src/test/resources/NIFI-8753/nifi.properties.before +++ b/minifi/minifi-bootstrap/src/test/resources/NIFI-8753/nifi.properties.before @@ -15,7 +15,7 @@ # Core Properties # -nifi.flow.configuration.file=./conf/flow.xml.gz +nifi.flow.configuration.file=./conf/flow.json.gz nifi.flow.configuration.archive.enabled=false nifi.flow.configuration.archive.dir=./conf/archive/ nifi.flowcontroller.autoResumeState=true diff --git a/minifi/minifi-docs/src/main/markdown/System_Admin_Guide.md b/minifi/minifi-docs/src/main/markdown/System_Admin_Guide.md index 69d5a77500..f227fec440 100644 --- a/minifi/minifi-docs/src/main/markdown/System_Admin_Guide.md +++ b/minifi/minifi-docs/src/main/markdown/System_Admin_Guide.md @@ -434,7 +434,7 @@ The "Core Properties" section applies to the core framework as a whole. *Property* | *Description* ------------------------------------------ | ----------- `flow controller graceful shutdown period` | Indicates the shutdown period. The default value is `10 sec`. -`flow service write delay interval` | When many changes are made to the *flow.xml*, this property specifies how long to wait before writing out the changes, so as to batch the changes into a single write. The default value is `500 ms`. +`flow service write delay interval` | When many changes are made to the *flow.json*, this property specifies how long to wait before writing out the changes, so as to batch the changes into a single write. The default value is `500 ms`. `administrative yield duration` | If a component allows an unexpected exception to escape, it is considered a bug. As a result, the framework will pause (or administratively yield) the component for this amount of time. This is done so that the component does not use up massive amounts of system resources, since it is known to have problems in the existing state. The default value is `30 sec`. `bored yield duration` | When a component has no work to do (i.e., is "bored"), this is the amount of time it will wait before checking to see if it has new data to work on. This way, it does not use up CPU resources by checking for new work too often. When setting this property, be aware that it could add extra latency for components that do not constantly have work to do, as once they go into this "bored" state, they will wait this amount of time before checking for more work. The default value is `10 millis`. `max concurrent threads` | The maximum number of threads any processor can have running at one time. diff --git a/minifi/minifi-integration-tests/src/test/resources/conf/nifi.properties b/minifi/minifi-integration-tests/src/test/resources/conf/nifi.properties index b2530ed55e..85f8471bd0 100644 --- a/minifi/minifi-integration-tests/src/test/resources/conf/nifi.properties +++ b/minifi/minifi-integration-tests/src/test/resources/conf/nifi.properties @@ -14,7 +14,7 @@ # limitations under the License. # Core Properties # -nifi.flow.configuration.file=./target/flow.xml.gz +nifi.flow.configuration.file=./target/flow.json.gz nifi.flow.configuration.archive.dir=./target/archive/ nifi.flowcontroller.autoResumeState=true nifi.flowcontroller.graceful.shutdown.period=10 sec diff --git a/minifi/minifi-nar-bundles/minifi-framework-bundle/minifi-framework/minifi-framework-core/src/main/java/org/apache/nifi/minifi/c2/C2NifiClientService.java b/minifi/minifi-nar-bundles/minifi-framework-bundle/minifi-framework/minifi-framework-core/src/main/java/org/apache/nifi/minifi/c2/C2NifiClientService.java index 74109a6887..0ef45a92c2 100644 --- a/minifi/minifi-nar-bundles/minifi-framework-bundle/minifi-framework/minifi-framework-core/src/main/java/org/apache/nifi/minifi/c2/C2NifiClientService.java +++ b/minifi/minifi-nar-bundles/minifi-framework-bundle/minifi-framework/minifi-framework-core/src/main/java/org/apache/nifi/minifi/c2/C2NifiClientService.java @@ -45,7 +45,7 @@ import static org.apache.nifi.minifi.commons.api.MiNiFiProperties.C2_SECURITY_KE import static org.apache.nifi.minifi.commons.api.MiNiFiProperties.C2_SECURITY_TRUSTSTORE_LOCATION; import static org.apache.nifi.minifi.commons.api.MiNiFiProperties.C2_SECURITY_TRUSTSTORE_PASSWORD; import static org.apache.nifi.minifi.commons.api.MiNiFiProperties.C2_SECURITY_TRUSTSTORE_TYPE; -import static org.apache.nifi.util.NiFiProperties.FLOW_CONFIGURATION_JSON_FILE; +import static org.apache.nifi.util.NiFiProperties.FLOW_CONFIGURATION_FILE; import com.fasterxml.jackson.databind.ObjectMapper; import java.io.IOException; @@ -216,7 +216,7 @@ public class C2NifiClientService { FlowEnrichService flowEnrichService = new FlowEnrichService(niFiProperties); UpdateConfigurationStrategy updateConfigurationStrategy = - new DefaultUpdateConfigurationStrategy(flowController, flowService, flowEnrichService, niFiProperties.getProperty(FLOW_CONFIGURATION_JSON_FILE)); + new DefaultUpdateConfigurationStrategy(flowController, flowService, flowEnrichService, niFiProperties.getProperty(FLOW_CONFIGURATION_FILE)); return new C2OperationHandlerProvider(List.of( new UpdateConfigurationOperationHandler(client, flowIdHolder, updateConfigurationStrategy, emptyOperandPropertiesProvider), diff --git a/minifi/minifi-nar-bundles/minifi-framework-bundle/minifi-framework/minifi-server/src/main/java/org/apache/nifi/minifi/StandardMiNiFiServer.java b/minifi/minifi-nar-bundles/minifi-framework-bundle/minifi-framework/minifi-server/src/main/java/org/apache/nifi/minifi/StandardMiNiFiServer.java index 4263ac809f..1b648a5a0b 100644 --- a/minifi/minifi-nar-bundles/minifi-framework-bundle/minifi-framework/minifi-server/src/main/java/org/apache/nifi/minifi/StandardMiNiFiServer.java +++ b/minifi/minifi-nar-bundles/minifi-framework-bundle/minifi-framework/minifi-server/src/main/java/org/apache/nifi/minifi/StandardMiNiFiServer.java @@ -24,7 +24,6 @@ import java.io.IOException; import java.io.UncheckedIOException; import java.util.Optional; import org.apache.commons.lang3.StringUtils; -import org.apache.nifi.controller.FlowSerializationStrategy; import org.apache.nifi.headless.HeadlessNiFiServer; import org.apache.nifi.minifi.bootstrap.BootstrapListener; import org.apache.nifi.minifi.c2.C2NifiClientService; @@ -87,11 +86,6 @@ public class StandardMiNiFiServer extends HeadlessNiFiServer implements MiNiFiSe } } - @Override - protected FlowSerializationStrategy getFlowSerializationStrategy() { - return FlowSerializationStrategy.WRITE_JSON_ONLY; - } - private void initC2() { if (Boolean.parseBoolean(getNiFiProperties().getProperty(MiNiFiProperties.C2_ENABLE.getKey(), MiNiFiProperties.C2_ENABLE.getDefaultValue()))) { NiFiProperties niFiProperties = getNiFiProperties(); diff --git a/nifi-bootstrap/src/test/resources/nifi.properties.dns-sans b/nifi-bootstrap/src/test/resources/nifi.properties.dns-sans index 1528c1b871..74d25bbc7c 100644 --- a/nifi-bootstrap/src/test/resources/nifi.properties.dns-sans +++ b/nifi-bootstrap/src/test/resources/nifi.properties.dns-sans @@ -14,7 +14,7 @@ # limitations under the License. # Core Properties # -nifi.flow.configuration.file=./target/flow.xml.gz +nifi.flow.configuration.file=./target/flow.json.gz # Removing most properties for testing... diff --git a/nifi-bootstrap/src/test/resources/nifi.properties.ip-sans b/nifi-bootstrap/src/test/resources/nifi.properties.ip-sans index 35df24cfcb..1ff3f757a4 100644 --- a/nifi-bootstrap/src/test/resources/nifi.properties.ip-sans +++ b/nifi-bootstrap/src/test/resources/nifi.properties.ip-sans @@ -14,7 +14,7 @@ # limitations under the License. # Core Properties # -nifi.flow.configuration.file=./target/flow.xml.gz +nifi.flow.configuration.file=./target/flow.json.gz # Removing most properties for testing... diff --git a/nifi-bootstrap/src/test/resources/nifi.properties.keystore-password b/nifi-bootstrap/src/test/resources/nifi.properties.keystore-password index cd99e6de5f..7ade1760b5 100644 --- a/nifi-bootstrap/src/test/resources/nifi.properties.keystore-password +++ b/nifi-bootstrap/src/test/resources/nifi.properties.keystore-password @@ -14,7 +14,7 @@ # limitations under the License. # Core Properties # -nifi.flow.configuration.file=./target/flow.xml.gz +nifi.flow.configuration.file=./target/flow.json.gz # Removing most properties for testing... diff --git a/nifi-bootstrap/src/test/resources/nifi.properties.no-https b/nifi-bootstrap/src/test/resources/nifi.properties.no-https index ddc7080fa7..84408708b9 100644 --- a/nifi-bootstrap/src/test/resources/nifi.properties.no-https +++ b/nifi-bootstrap/src/test/resources/nifi.properties.no-https @@ -14,7 +14,7 @@ # limitations under the License. # Core Properties # -nifi.flow.configuration.file=./target/flow.xml.gz +nifi.flow.configuration.file=./target/flow.json.gz # Removing most properties for testing... diff --git a/nifi-bootstrap/src/test/resources/nifi.properties.no-keystore-types b/nifi-bootstrap/src/test/resources/nifi.properties.no-keystore-types index 37b9915f4f..e1c1c0b1e2 100644 --- a/nifi-bootstrap/src/test/resources/nifi.properties.no-keystore-types +++ b/nifi-bootstrap/src/test/resources/nifi.properties.no-keystore-types @@ -14,7 +14,7 @@ # limitations under the License. # Core Properties # -nifi.flow.configuration.file=./target/flow.xml.gz +nifi.flow.configuration.file=./target/flow.json.gz # Removing most properties for testing... diff --git a/nifi-bootstrap/src/test/resources/nifi.properties.no-keystores b/nifi-bootstrap/src/test/resources/nifi.properties.no-keystores index a4325f4a75..3eef8bc3bd 100644 --- a/nifi-bootstrap/src/test/resources/nifi.properties.no-keystores +++ b/nifi-bootstrap/src/test/resources/nifi.properties.no-keystores @@ -14,7 +14,7 @@ # limitations under the License. # Core Properties # -nifi.flow.configuration.file=./target/flow.xml.gz +nifi.flow.configuration.file=./target/flow.json.gz # Removing most properties for testing... diff --git a/nifi-bootstrap/src/test/resources/nifi.properties.only-keystore b/nifi-bootstrap/src/test/resources/nifi.properties.only-keystore index de83da7720..37d7ed2c61 100644 --- a/nifi-bootstrap/src/test/resources/nifi.properties.only-keystore +++ b/nifi-bootstrap/src/test/resources/nifi.properties.only-keystore @@ -14,7 +14,7 @@ # limitations under the License. # Core Properties # -nifi.flow.configuration.file=./target/flow.xml.gz +nifi.flow.configuration.file=./target/flow.json.gz # Removing most properties for testing... diff --git a/nifi-bootstrap/src/test/resources/nifi.properties.only-truststore b/nifi-bootstrap/src/test/resources/nifi.properties.only-truststore index 4b58c0cc93..3f5d57a04f 100644 --- a/nifi-bootstrap/src/test/resources/nifi.properties.only-truststore +++ b/nifi-bootstrap/src/test/resources/nifi.properties.only-truststore @@ -14,7 +14,7 @@ # limitations under the License. # Core Properties # -nifi.flow.configuration.file=./target/flow.xml.gz +nifi.flow.configuration.file=./target/flow.json.gz # Removing most properties for testing... diff --git a/nifi-bootstrap/src/test/resources/nifi.properties.stores-exist b/nifi-bootstrap/src/test/resources/nifi.properties.stores-exist index 6cc723eacc..6d974df1d0 100644 --- a/nifi-bootstrap/src/test/resources/nifi.properties.stores-exist +++ b/nifi-bootstrap/src/test/resources/nifi.properties.stores-exist @@ -14,7 +14,7 @@ # limitations under the License. # Core Properties # -nifi.flow.configuration.file=./target/flow.xml.gz +nifi.flow.configuration.file=./target/flow.json.gz # Removing most properties for testing... diff --git a/nifi-bootstrap/src/test/resources/nifi.properties.success b/nifi-bootstrap/src/test/resources/nifi.properties.success index 1272b12a75..67a2cb69c9 100644 --- a/nifi-bootstrap/src/test/resources/nifi.properties.success +++ b/nifi-bootstrap/src/test/resources/nifi.properties.success @@ -14,7 +14,7 @@ # limitations under the License. # Core Properties # -nifi.flow.configuration.file=./target/flow.xml.gz +nifi.flow.configuration.file=./target/flow.json.gz # Removing most properties for testing... diff --git a/nifi-bootstrap/src/test/resources/nifi.properties.truststore-password b/nifi-bootstrap/src/test/resources/nifi.properties.truststore-password index 5ac0d2072a..52c6eb73a7 100644 --- a/nifi-bootstrap/src/test/resources/nifi.properties.truststore-password +++ b/nifi-bootstrap/src/test/resources/nifi.properties.truststore-password @@ -14,7 +14,7 @@ # limitations under the License. # Core Properties # -nifi.flow.configuration.file=./target/flow.xml.gz +nifi.flow.configuration.file=./target/flow.json.gz # Removing most properties for testing... diff --git a/nifi-commons/nifi-flow-encryptor/src/main/java/org/apache/nifi/flow/encryptor/StandardFlowEncryptor.java b/nifi-commons/nifi-flow-encryptor/src/main/java/org/apache/nifi/flow/encryptor/StandardFlowEncryptor.java deleted file mode 100644 index 2cbbfd2f09..0000000000 --- a/nifi-commons/nifi-flow-encryptor/src/main/java/org/apache/nifi/flow/encryptor/StandardFlowEncryptor.java +++ /dev/null @@ -1,55 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.nifi.flow.encryptor; - -import org.apache.nifi.encrypt.PropertyEncryptor; - -import java.io.BufferedInputStream; -import java.io.IOException; -import java.io.InputStream; -import java.io.OutputStream; -import java.io.UncheckedIOException; - -/** - * Standard Flow Encryptor handles reading Input Steam and writing Output Stream - */ -public class StandardFlowEncryptor implements FlowEncryptor { - private static final int XML_DECLARATION = '<'; - - /** - * Process Flow Configuration Stream replacing existing encrypted properties with new encrypted properties - * - * @param inputStream Flow Configuration Input Stream - * @param outputStream Flow Configuration Output Stream encrypted using new password - * @param inputEncryptor Property Encryptor for Input Configuration - * @param outputEncryptor Property Encryptor for Output Configuration - */ - @Override - public void processFlow(final InputStream inputStream, final OutputStream outputStream, - final PropertyEncryptor inputEncryptor, final PropertyEncryptor outputEncryptor) { - final BufferedInputStream bufferedInputStream = new BufferedInputStream(inputStream); - bufferedInputStream.mark(1); - try { - final int firstByte = bufferedInputStream.read(); - bufferedInputStream.reset(); - final FlowEncryptor flowEncryptor = (firstByte == XML_DECLARATION) ? new XmlFlowEncryptor() : new JsonFlowEncryptor(); - flowEncryptor.processFlow(bufferedInputStream, outputStream, inputEncryptor, outputEncryptor); - } catch (final IOException e) { - throw new UncheckedIOException(e); - } - } -} diff --git a/nifi-commons/nifi-flow-encryptor/src/main/java/org/apache/nifi/flow/encryptor/XmlFlowEncryptor.java b/nifi-commons/nifi-flow-encryptor/src/main/java/org/apache/nifi/flow/encryptor/XmlFlowEncryptor.java deleted file mode 100644 index db5b22bf24..0000000000 --- a/nifi-commons/nifi-flow-encryptor/src/main/java/org/apache/nifi/flow/encryptor/XmlFlowEncryptor.java +++ /dev/null @@ -1,80 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.nifi.flow.encryptor; - -import org.apache.nifi.encrypt.PropertyEncryptor; -import org.apache.nifi.xml.processing.stream.StandardXMLEventReaderProvider; -import org.apache.nifi.xml.processing.stream.XMLEventReaderProvider; - -import javax.xml.stream.XMLEventFactory; -import javax.xml.stream.XMLEventReader; -import javax.xml.stream.XMLEventWriter; -import javax.xml.stream.XMLOutputFactory; -import javax.xml.stream.XMLStreamException; -import javax.xml.stream.events.Characters; -import javax.xml.stream.events.XMLEvent; -import javax.xml.transform.stream.StreamSource; -import java.io.IOException; -import java.io.InputStream; -import java.io.OutputStream; -import java.io.UncheckedIOException; -import java.nio.charset.StandardCharsets; -import java.util.regex.Matcher; - -public class XmlFlowEncryptor extends AbstractFlowEncryptor { - private static final XMLEventReaderProvider eventReaderProvider = new StandardXMLEventReaderProvider(); - - @Override - public void processFlow(final InputStream inputStream, final OutputStream outputStream, - final PropertyEncryptor inputEncryptor, final PropertyEncryptor outputEncryptor) { - final XMLOutputFactory xmlOutputFactory = XMLOutputFactory.newInstance(); - final XMLEventFactory eventFactory = XMLEventFactory.newInstance(); - - try { - final XMLEventReader reader = eventReaderProvider.getEventReader(new StreamSource(inputStream)); - final XMLEventWriter writer = xmlOutputFactory.createXMLEventWriter(outputStream, StandardCharsets.UTF_8.name()); - while (reader.hasNext()) { - final XMLEvent event = reader.nextEvent(); - if (event.getEventType() == XMLEvent.CHARACTERS) { - final Characters characters = event.asCharacters(); - final String value = characters.getData(); - final Matcher matcher = ENCRYPTED_PATTERN.matcher(value); - if (matcher.matches()) { - final String processedValue = getOutputEncrypted(matcher.group(FIRST_GROUP), inputEncryptor, outputEncryptor); - writer.add(eventFactory.createCharacters(processedValue)); - } else { - writer.add(characters); - } - } else if (event.getEventType() == XMLEvent.START_DOCUMENT) { - writer.add(event); - writer.add(eventFactory.createSpace(System.lineSeparator())); - } else { - writer.add(event); - } - } - writer.flush(); - writer.close(); - reader.close(); - outputStream.close(); - inputStream.close(); - } catch (final XMLStreamException e) { - throw new RuntimeException("Flow XML Processing Failed", e); - } catch (final IOException e) { - throw new UncheckedIOException("Failed Processing Flow Configuration", e); - } - } -} diff --git a/nifi-commons/nifi-flow-encryptor/src/main/java/org/apache/nifi/flow/encryptor/command/FlowEncryptorCommand.java b/nifi-commons/nifi-flow-encryptor/src/main/java/org/apache/nifi/flow/encryptor/command/FlowEncryptorCommand.java index 353b065b5d..b5f8cd03cb 100644 --- a/nifi-commons/nifi-flow-encryptor/src/main/java/org/apache/nifi/flow/encryptor/command/FlowEncryptorCommand.java +++ b/nifi-commons/nifi-flow-encryptor/src/main/java/org/apache/nifi/flow/encryptor/command/FlowEncryptorCommand.java @@ -16,12 +16,6 @@ */ package org.apache.nifi.flow.encryptor.command; -import org.apache.nifi.encrypt.PropertyEncryptionMethod; -import org.apache.nifi.encrypt.PropertyEncryptor; -import org.apache.nifi.encrypt.PropertyEncryptorBuilder; -import org.apache.nifi.flow.encryptor.FlowEncryptor; -import org.apache.nifi.flow.encryptor.StandardFlowEncryptor; - import java.io.File; import java.io.FileInputStream; import java.io.FileOutputStream; @@ -40,6 +34,11 @@ import java.util.Properties; import java.util.stream.Collectors; import java.util.zip.GZIPInputStream; import java.util.zip.GZIPOutputStream; +import org.apache.nifi.encrypt.PropertyEncryptionMethod; +import org.apache.nifi.encrypt.PropertyEncryptor; +import org.apache.nifi.encrypt.PropertyEncryptorBuilder; +import org.apache.nifi.flow.encryptor.FlowEncryptor; +import org.apache.nifi.flow.encryptor.JsonFlowEncryptor; /** * Flow Encryptor Command capable of updating Sensitive Properties Key or Algorithm as well as Flow Configuration @@ -53,9 +52,7 @@ class FlowEncryptorCommand implements Runnable { protected static final String CONFIGURATION_FILE = "nifi.flow.configuration.file"; - protected static final String CONFIGURATION_JSON_FILE = "nifi.flow.configuration.json.file"; - - private static final List CONFIGURATION_FILES = Arrays.asList(CONFIGURATION_FILE, CONFIGURATION_JSON_FILE); + private static final List CONFIGURATION_FILES = Arrays.asList(CONFIGURATION_FILE); private static final String FLOW_PREFIX = "nifi.flow."; @@ -131,7 +128,7 @@ class FlowEncryptorCommand implements Runnable { final String inputPropertiesKey = getKey(properties); final PropertyEncryptor inputEncryptor = getPropertyEncryptor(inputPropertiesKey, inputAlgorithm); - final FlowEncryptor flowEncryptor = new StandardFlowEncryptor(); + final FlowEncryptor flowEncryptor = new JsonFlowEncryptor(); flowEncryptor.processFlow(flowInputStream, flowOutputStream, inputEncryptor, outputEncryptor); } diff --git a/nifi-commons/nifi-flow-encryptor/src/test/java/org/apache/nifi/flow/encryptor/StandardFlowEncryptorTest.java b/nifi-commons/nifi-flow-encryptor/src/test/java/org/apache/nifi/flow/encryptor/JsonFlowEncryptorTest.java similarity index 62% rename from nifi-commons/nifi-flow-encryptor/src/test/java/org/apache/nifi/flow/encryptor/StandardFlowEncryptorTest.java rename to nifi-commons/nifi-flow-encryptor/src/test/java/org/apache/nifi/flow/encryptor/JsonFlowEncryptorTest.java index ef9b61ffd4..37a444dd95 100644 --- a/nifi-commons/nifi-flow-encryptor/src/test/java/org/apache/nifi/flow/encryptor/StandardFlowEncryptorTest.java +++ b/nifi-commons/nifi-flow-encryptor/src/test/java/org/apache/nifi/flow/encryptor/JsonFlowEncryptorTest.java @@ -16,11 +16,8 @@ */ package org.apache.nifi.flow.encryptor; -import org.apache.nifi.encrypt.PropertyEncryptionMethod; -import org.apache.nifi.encrypt.PropertyEncryptor; -import org.apache.nifi.encrypt.PropertyEncryptorBuilder; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; @@ -31,11 +28,13 @@ import java.util.Objects; import java.util.UUID; import java.util.regex.Matcher; import java.util.regex.Pattern; +import org.apache.nifi.encrypt.PropertyEncryptionMethod; +import org.apache.nifi.encrypt.PropertyEncryptor; +import org.apache.nifi.encrypt.PropertyEncryptorBuilder; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertTrue; - -public class StandardFlowEncryptorTest { +public class JsonFlowEncryptorTest { private static final String INPUT_KEY = UUID.randomUUID().toString(); private static final String OUTPUT_KEY = UUID.randomUUID().toString(); @@ -50,22 +49,23 @@ public class StandardFlowEncryptorTest { private PropertyEncryptor outputEncryptor; - private StandardFlowEncryptor flowEncryptor; + private FlowEncryptor flowEncryptor; @BeforeEach public void setEncryptors() { inputEncryptor = getPropertyEncryptor(INPUT_KEY, PropertyEncryptionMethod.NIFI_PBKDF2_AES_GCM_256.name()); outputEncryptor = getPropertyEncryptor(OUTPUT_KEY, PropertyEncryptionMethod.NIFI_ARGON2_AES_GCM_256.name()); - flowEncryptor = new StandardFlowEncryptor(); + flowEncryptor = new JsonFlowEncryptor(); } @Test public void testProcessEncrypted() { - final String property = StandardFlowEncryptorTest.class.getSimpleName(); + final String property = JsonFlowEncryptorTest.class.getSimpleName(); final String encryptedProperty = String.format(ENCRYPTED_FORMAT, inputEncryptor.encrypt(property)); - final String encryptedRow = String.format("%n" + - "%s", encryptedProperty); + final String encryptedRow = String.format( + "{\"properties\":{\"username\":\"sample_username\",\"test\":\"%s\",\"position\":1.123456789123456789}}", + encryptedProperty); final InputStream inputStream = new ByteArrayInputStream(encryptedRow.getBytes(StandardCharsets.UTF_8)); final ByteArrayOutputStream outputStream = new ByteArrayOutputStream(); @@ -83,8 +83,7 @@ public class StandardFlowEncryptorTest { @Test public void testProcessNoEncrypted() { - final String property = String.format("%n" + - "%s", StandardFlowEncryptorTest.class.getSimpleName()); + final String property = String.format("{\"properties\":{\"username\":\"sample_username\",\"test\":\"%s\",\"position\":1.123456789123456789}}", JsonFlowEncryptorTest.class.getSimpleName()); final InputStream inputStream = new ByteArrayInputStream(property.getBytes(StandardCharsets.UTF_8)); final ByteArrayOutputStream outputStream = new ByteArrayOutputStream(); @@ -92,12 +91,12 @@ public class StandardFlowEncryptorTest { flowEncryptor.processFlow(inputStream, outputStream, inputEncryptor, outputEncryptor); final String outputProperty = outputStream.toString(); - assertEquals(removeXmlDeclaration(property).trim(), removeXmlDeclaration(outputProperty).trim()); + assertEquals(property, outputProperty); } @Test public void testProcessJson() throws IOException { - final String password = StandardFlowEncryptorTest.class.getSimpleName(); + final String password = JsonFlowEncryptorTest.class.getSimpleName(); final String encryptedPassword = String.format(ENCRYPTED_FORMAT, inputEncryptor.encrypt(password)); final String sampleFlowJson = getSampleFlowJson(encryptedPassword); @@ -113,21 +112,6 @@ public class StandardFlowEncryptorTest { } } - @Test - public void testProcessXml() throws IOException { - final String password = StandardFlowEncryptorTest.class.getSimpleName(); - final String encryptedPassword = String.format(ENCRYPTED_FORMAT, inputEncryptor.encrypt(password)); - final String sampleFlowXml = getSampleFlowXml(encryptedPassword); - try (final InputStream inputStream = new ByteArrayInputStream(sampleFlowXml.getBytes(StandardCharsets.UTF_8))) { - try (final ByteArrayOutputStream outputStream = new ByteArrayOutputStream()) { - flowEncryptor.processFlow(inputStream, outputStream, inputEncryptor, outputEncryptor); - final String outputXml = outputStream.toString(); - - compareFlow(removeXmlDeclaration(sampleFlowXml).trim(), removeXmlDeclaration(outputXml).trim()); - } - } - } - private PropertyEncryptor getPropertyEncryptor(final String propertiesKey, final String propertiesAlgorithm) { return new PropertyEncryptorBuilder(propertiesKey).setAlgorithm(propertiesAlgorithm).build(); } @@ -145,43 +129,4 @@ public class StandardFlowEncryptorTest { Objects.requireNonNull(password); return String.format("{\"properties\":{\"username\":\"sample_username\",\"password\":\"%s\",\"position\":1.123456789123456789}}", password); } - - private String getSampleFlowXml(final String password) { - Objects.requireNonNull(password); - final String flowXml = String.format("%n" + - "%n" + - "\t%n" + - "\t\tUsername%n" + - "\t\tSAMPLE_USERNAME%n" + - "\t%n" + - "\t%n" + - "\t\tPassword%n" + - "\t\t%s%n" + - "\t%n" + - "", password); - - return getProcessedFlowXml(flowXml); - } - - private String getProcessedFlowXml(final String flowXml) { - final PropertyEncryptor encryptor = new PropertyEncryptor() { - @Override - public String encrypt(String property) { - return property; - } - - @Override - public String decrypt(String encryptedProperty) { - return encryptedProperty; - } - }; - final InputStream inputStream = new ByteArrayInputStream(flowXml.getBytes(StandardCharsets.UTF_8)); - final ByteArrayOutputStream outputStream = new ByteArrayOutputStream(); - flowEncryptor.processFlow(inputStream, outputStream, encryptor, encryptor); - return outputStream.toString(); - } - - private String removeXmlDeclaration(final String xmlFlow) { - return xmlFlow.replaceAll("<\\?xml.+\\?>", ""); - } } diff --git a/nifi-commons/nifi-flow-encryptor/src/test/java/org/apache/nifi/flow/encryptor/command/FlowEncryptorCommandTest.java b/nifi-commons/nifi-flow-encryptor/src/test/java/org/apache/nifi/flow/encryptor/command/FlowEncryptorCommandTest.java index 50837bd05e..1f37379bdd 100644 --- a/nifi-commons/nifi-flow-encryptor/src/test/java/org/apache/nifi/flow/encryptor/command/FlowEncryptorCommandTest.java +++ b/nifi-commons/nifi-flow-encryptor/src/test/java/org/apache/nifi/flow/encryptor/command/FlowEncryptorCommandTest.java @@ -44,12 +44,8 @@ public class FlowEncryptorCommandTest { private static final String FLOW_CONTENTS_JSON = "{\"property\":\"value\"}"; - private static final String FLOW_CONTENTS_XML = "PROPERTY"; - private static final String JSON_GZ = ".json.gz"; - private static final String XML_GZ = ".xml.gz"; - private static final String PROPERTIES_EXTENSION = ".properties"; private static final String BLANK_PROPERTIES = "/blank.nifi.properties"; @@ -123,19 +119,16 @@ public class FlowEncryptorCommandTest { } protected static Path getBlankNiFiProperties() throws IOException, URISyntaxException { - final Path flowConfiguration = getFlowConfiguration(FLOW_CONTENTS_XML, XML_GZ); final Path flowConfigurationJson = getFlowConfiguration(FLOW_CONTENTS_JSON, JSON_GZ); - return getNiFiProperties(flowConfiguration, flowConfigurationJson, BLANK_PROPERTIES); + return getNiFiProperties(flowConfigurationJson, BLANK_PROPERTIES); } protected static Path getPopulatedNiFiProperties() throws IOException, URISyntaxException { - final Path flowConfiguration = getFlowConfiguration(FLOW_CONTENTS_XML, XML_GZ); final Path flowConfigurationJson = getFlowConfiguration(FLOW_CONTENTS_JSON, JSON_GZ); - return getNiFiProperties(flowConfiguration, flowConfigurationJson, POPULATED_PROPERTIES); + return getNiFiProperties(flowConfigurationJson, POPULATED_PROPERTIES); } private static Path getNiFiProperties( - final Path flowConfigurationPath, final Path flowConfigurationJsonPath, String propertiesResource ) throws IOException, URISyntaxException { @@ -143,8 +136,6 @@ public class FlowEncryptorCommandTest { final List sourceProperties = Files.readAllLines(sourcePropertiesPath); final List flowProperties = sourceProperties.stream().map(line -> { if (line.startsWith(FlowEncryptorCommand.CONFIGURATION_FILE)) { - return line + flowConfigurationPath; - } else if (line.startsWith(FlowEncryptorCommand.CONFIGURATION_JSON_FILE)) { return flowConfigurationJsonPath == null ? line : line + flowConfigurationJsonPath; } else { return line; diff --git a/nifi-commons/nifi-flow-encryptor/src/test/resources/blank.nifi.properties b/nifi-commons/nifi-flow-encryptor/src/test/resources/blank.nifi.properties index b6e24e7eac..8c16c51f95 100644 --- a/nifi-commons/nifi-flow-encryptor/src/test/resources/blank.nifi.properties +++ b/nifi-commons/nifi-flow-encryptor/src/test/resources/blank.nifi.properties @@ -15,4 +15,3 @@ nifi.sensitive.props.key= nifi.sensitive.props.algorithm= nifi.flow.configuration.file= -nifi.flow.configuration.json.file= diff --git a/nifi-commons/nifi-flow-encryptor/src/test/resources/populated.nifi.properties b/nifi-commons/nifi-flow-encryptor/src/test/resources/populated.nifi.properties index 520019e21a..36e2707c71 100644 --- a/nifi-commons/nifi-flow-encryptor/src/test/resources/populated.nifi.properties +++ b/nifi-commons/nifi-flow-encryptor/src/test/resources/populated.nifi.properties @@ -15,4 +15,3 @@ nifi.sensitive.props.key=D5E41AC1-EEF8-4A54-930D-593F749AE95C nifi.sensitive.props.algorithm=NIFI_ARGON2_AES_GCM_256 nifi.flow.configuration.file= -nifi.flow.configuration.json.file= diff --git a/nifi-commons/nifi-properties/src/main/java/org/apache/nifi/util/NiFiProperties.java b/nifi-commons/nifi-properties/src/main/java/org/apache/nifi/util/NiFiProperties.java index 643c050edf..78432ce444 100644 --- a/nifi-commons/nifi-properties/src/main/java/org/apache/nifi/util/NiFiProperties.java +++ b/nifi-commons/nifi-properties/src/main/java/org/apache/nifi/util/NiFiProperties.java @@ -54,7 +54,6 @@ public class NiFiProperties extends ApplicationProperties { // core properties public static final String PROPERTIES_FILE_PATH = "nifi.properties.file.path"; public static final String FLOW_CONFIGURATION_FILE = "nifi.flow.configuration.file"; - public static final String FLOW_CONFIGURATION_JSON_FILE = "nifi.flow.configuration.json.file"; public static final String FLOW_CONFIGURATION_ARCHIVE_ENABLED = "nifi.flow.configuration.archive.enabled"; public static final String FLOW_CONFIGURATION_ARCHIVE_DIR = "nifi.flow.configuration.archive.dir"; public static final String FLOW_CONFIGURATION_ARCHIVE_MAX_TIME = "nifi.flow.configuration.archive.max.time"; @@ -452,7 +451,6 @@ public class NiFiProperties extends ApplicationProperties { super(props); } - // getters for core properties // public File getFlowConfigurationFile() { try { return new File(getProperty(FLOW_CONFIGURATION_FILE)); @@ -461,21 +459,6 @@ public class NiFiProperties extends ApplicationProperties { } } - public File getFlowConfigurationJsonFile() { - final String jsonFilename = getProperty(FLOW_CONFIGURATION_JSON_FILE); - if (jsonFilename != null) { - return new File(jsonFilename); - } - - final File xmlFile = getFlowConfigurationFile(); - final String xmlFilename = xmlFile.getName(); - if (xmlFilename.contains(".xml")) { - return new File(xmlFile.getParentFile(), xmlFilename.replace(".xml", ".json")); - } - - return new File(xmlFile.getParentFile(), xmlFilename.replace(".gz", "") + ".json.gz"); - } - public File getFlowConfigurationFileDir() { try { return getFlowConfigurationFile().getParentFile(); diff --git a/nifi-commons/nifi-properties/src/test/resources/NiFiProperties/conf/nifi.blank.properties b/nifi-commons/nifi-properties/src/test/resources/NiFiProperties/conf/nifi.blank.properties index 99931d1ae7..64dd766aa8 100644 --- a/nifi-commons/nifi-properties/src/test/resources/NiFiProperties/conf/nifi.blank.properties +++ b/nifi-commons/nifi-properties/src/test/resources/NiFiProperties/conf/nifi.blank.properties @@ -14,7 +14,7 @@ # limitations under the License. # Core Properties # -nifi.flow.configuration.file=./target/flow.xml.gz +nifi.flow.configuration.file=./target/flow.json.gz nifi.flow.configuration.archive.dir=./target/archive/ nifi.flowcontroller.autoResumeState=true nifi.flowcontroller.graceful.shutdown.period=10 sec diff --git a/nifi-commons/nifi-properties/src/test/resources/NiFiProperties/conf/nifi.missing.properties b/nifi-commons/nifi-properties/src/test/resources/NiFiProperties/conf/nifi.missing.properties index f94fbdbf05..31f0ef53a3 100644 --- a/nifi-commons/nifi-properties/src/test/resources/NiFiProperties/conf/nifi.missing.properties +++ b/nifi-commons/nifi-properties/src/test/resources/NiFiProperties/conf/nifi.missing.properties @@ -14,7 +14,7 @@ # limitations under the License. # Core Properties # -nifi.flow.configuration.file=./target/flow.xml.gz +nifi.flow.configuration.file=./target/flow.json.gz nifi.flow.configuration.archive.dir=./target/archive/ nifi.flowcontroller.autoResumeState=true nifi.flowcontroller.graceful.shutdown.period=10 sec diff --git a/nifi-commons/nifi-properties/src/test/resources/NiFiProperties/conf/nifi.properties b/nifi-commons/nifi-properties/src/test/resources/NiFiProperties/conf/nifi.properties index ec3616b435..853395b79e 100644 --- a/nifi-commons/nifi-properties/src/test/resources/NiFiProperties/conf/nifi.properties +++ b/nifi-commons/nifi-properties/src/test/resources/NiFiProperties/conf/nifi.properties @@ -14,7 +14,7 @@ # limitations under the License. # Core Properties # -nifi.flow.configuration.file=./target/flow.xml.gz +nifi.flow.configuration.file=./target/flow.json.gz nifi.flow.configuration.archive.dir=./target/archive/ nifi.flowcontroller.autoResumeState=true nifi.flowcontroller.graceful.shutdown.period=10 sec diff --git a/nifi-docs/src/main/asciidoc/administration-guide.adoc b/nifi-docs/src/main/asciidoc/administration-guide.adoc index 8fc594e9f9..2cc8d07726 100644 --- a/nifi-docs/src/main/asciidoc/administration-guide.adoc +++ b/nifi-docs/src/main/asciidoc/administration-guide.adoc @@ -3316,11 +3316,7 @@ The first section of the _nifi.properties_ file is for the Core Properties. Thes |=== |*Property*|*Description* -|`nifi.flow.configuration.file`*|The location of the XML-based flow configuration file. The default value is `./conf/flow.xml.gz`. This is a legacy property. Older versions of NiFi used an -XML-formatted file to store the flow configuration. However, newer versions use a JSON representation. In order to maintain backward compatibility of flows and still load flows developed using -older versions of NiFi, upon startup, NiFi will use the `nifi.flow.configuration.json.file` first. If the file exists, it will be used. However, if it does not exist, NiFi will fall back to this -property to determine the XML version of the file and use it. -|`nifi.flow.configuration.json.file`*|The location of the flow configuration file (i.e., the file that contains what is currently displayed on the NiFi graph). The default value is `./conf/flow.json.gz`. +|`nifi.flow.configuration.file`*|The location of the JSON-based flow configuration file. The default value is `./conf/flow.json.gz`. |`nifi.flow.configuration.archive.enabled`*|Specifies whether NiFi creates a backup copy of the flow automatically when the flow is updated. The default value is `true`. |`nifi.flow.configuration.archive.dir`*|The location of the archive directory where backup copies of the _flow.json_ are saved. The default value is `./conf/archive`. NiFi removes old archive files to limit disk usage based on archived file lifespan, total size, and number of files, as specified with `nifi.flow.configuration.archive.max.time`, `max.storage` and `max.count` properties respectively. If none of these limitation for archiving is specified, NiFi uses default conditions, that is `30 days` for `max.time` and `500 MB` for `max.storage`. + This cleanup mechanism takes into account only automatically created archived _flow.json_ files. If there are other files or directories in this archive directory, NiFi will ignore them. Automatically created archives have filename with ISO 8601 format timestamp prefix followed by ``. That is `T+_`. For example, `20160706T160719+0900_flow.json.gz`. NiFi checks filenames when it cleans archive directory. If you would like to keep a particular archive in this directory without worrying about NiFi deleting it, you can do so by copying it with a different filename pattern. @@ -4398,7 +4394,7 @@ If you are encrypting sensitive component properties in your dataflow via the se If the below properties point to directories inside the NiFi base installation path, you must copy the target directories to the new NiFi. Stop your existing NiFi installation before you do this. -|`nifi.flow.configuration.json.file=` +|`nifi.flow.configuration.file=` If you have retained the default value (`./conf/flow.json.gz`), copy _flow.json.gz_ from the existing to the new NiFi base install conf directory. @@ -4466,7 +4462,7 @@ When a value is set for `nifi.sensitive.props.key` in _nifi.properties_, the spe 2. Encrypts all the sensitive values with a specified new key. 3. Updates the _nifi.properties_ and _flow.json.gz_ files or creates new versions of them. -As an example, assume version 1.9.2 is the existing NiFi instance and the sensitive properties key is set to `password`. The goal is to move the 1.9.2 _flow.xml.gz_ to a 1.10.0 instance with a new sensitive properties key: `new_password`. Running the following Encrypt-Config command would read in the _flow.xml.gz_ and _nifi.properties_ files from 1.9.2 using the original sensitive properties key and write out new versions in 1.10.0 with the sensitive properties encrypted with the new password: +As an example, assume version 1.9.2 is the existing NiFi instance and the sensitive properties key is set to `password`. The goal is to move the 1.9.2 _flow.json.gz_ to a 1.10.0 instance with a new sensitive properties key: `new_password`. Running the following Encrypt-Config command would read in the _flow.json.gz_ and _nifi.properties_ files from 1.9.2 using the original sensitive properties key and write out new versions in 1.10.0 with the sensitive properties encrypted with the new password: ``` $ ./nifi-toolkit-1.10.0/bin/encrypt-config.sh -f /path/to/nifi/nifi-1.9.2/conf/flow.json.gz -g /path/to/nifi/nifi-1.10.0/conf/flow.json.gz -s new_password -n /path/to/nifi/nifi-1.9.2/conf/nifi @@ -4495,7 +4491,6 @@ $ ./bin/nifi.sh set-sensitive-properties-algorithm The command reads the following flow configuration file properties from _nifi.properties_: - `nifi.flow.configuration.file` -- `nifi.flow.configuration.json.file` The command checks for the existence of each file and updates the sensitive property values found. @@ -4511,12 +4506,11 @@ The following command can be used to read an existing flow configuration and set $ ./bin/nifi.sh set-sensitive-properties-key ``` -The command reads the following flow configuration file properties from _nifi.properties_: +The command reads the following flow configuration file property from _nifi.properties_: - `nifi.flow.configuration.file` -- `nifi.flow.configuration.json.file` -The command checks for the existence of each file and updates the sensitive property values found. +The command checks for the existence of the file file and updates the sensitive property values found. The minimum required length for a new sensitive properties key is 12 characters. diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-client-dto/src/main/java/org/apache/nifi/web/api/dto/PortDTO.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-client-dto/src/main/java/org/apache/nifi/web/api/dto/PortDTO.java index e9313186e4..eedcec6190 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-client-dto/src/main/java/org/apache/nifi/web/api/dto/PortDTO.java +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-client-dto/src/main/java/org/apache/nifi/web/api/dto/PortDTO.java @@ -20,7 +20,6 @@ import io.swagger.annotations.ApiModelProperty; import javax.xml.bind.annotation.XmlType; import java.util.Collection; -import java.util.Set; /** * The details for a port within this NiFi flow. @@ -34,8 +33,6 @@ public class PortDTO extends ComponentDTO { private String type; private Boolean transmitting; private Integer concurrentlySchedulableTaskCount; - private Set userAccessControl; - private Set groupAccessControl; private Boolean allowRemoteAccess; private String portFunction; @@ -129,34 +126,6 @@ public class PortDTO extends ComponentDTO { this.transmitting = transmitting; } - /** - * @return groups that are allowed to access this port - */ - @ApiModelProperty( - value = "The user groups that are allowed to access the port." - ) - public Set getGroupAccessControl() { - return groupAccessControl; - } - - public void setGroupAccessControl(Set groupAccessControl) { - this.groupAccessControl = groupAccessControl; - } - - /** - * @return users that are allowed to access this port - */ - @ApiModelProperty( - value = "The users that are allowed to access the port." - ) - public Set getUserAccessControl() { - return userAccessControl; - } - - public void setUserAccessControl(Set userAccessControl) { - this.userAccessControl = userAccessControl; - } - /** * Gets the validation errors from this port. These validation errors represent the problems with the port that must be resolved before it can be started. * diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-file-authorizer/src/main/java/org/apache/nifi/authorization/FileAccessPolicyProvider.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-file-authorizer/src/main/java/org/apache/nifi/authorization/FileAccessPolicyProvider.java index 5be8335a79..0d2ada3beb 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-file-authorizer/src/main/java/org/apache/nifi/authorization/FileAccessPolicyProvider.java +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-file-authorizer/src/main/java/org/apache/nifi/authorization/FileAccessPolicyProvider.java @@ -25,12 +25,10 @@ import org.apache.nifi.authorization.exception.UninheritableAuthorizationsExcept import org.apache.nifi.authorization.file.generated.Authorizations; import org.apache.nifi.authorization.file.generated.Policies; import org.apache.nifi.authorization.file.generated.Policy; -import org.apache.nifi.authorization.resource.ResourceFactory; import org.apache.nifi.authorization.resource.ResourceType; import org.apache.nifi.authorization.util.IdentityMapping; import org.apache.nifi.authorization.util.IdentityMappingUtil; import org.apache.nifi.components.PropertyValue; -import org.apache.nifi.user.generated.Users; import org.apache.nifi.util.FlowInfo; import org.apache.nifi.util.FlowParser; import org.apache.nifi.util.NiFiProperties; @@ -131,7 +129,6 @@ public class FileAccessPolicyProvider implements ConfigurableAccessPolicyProvide private File restoreAuthorizationsFile; private String rootGroupId; private String initialAdminIdentity; - private String legacyAuthorizedUsersFile; private Set nodeIdentities; private String nodeGroupIdentifier; private List ports = new ArrayList<>(); @@ -213,10 +210,6 @@ public class FileAccessPolicyProvider implements ConfigurableAccessPolicyProvide final PropertyValue initialAdminIdentityProp = configurationContext.getProperty(PROP_INITIAL_ADMIN_IDENTITY); initialAdminIdentity = initialAdminIdentityProp.isSet() ? IdentityMappingUtil.mapIdentity(initialAdminIdentityProp.getValue(), identityMappings) : null; - // get the value of the legacy authorized users file - final PropertyValue legacyAuthorizedUsersProp = configurationContext.getProperty(FileAuthorizer.PROP_LEGACY_AUTHORIZED_USERS_FILE); - legacyAuthorizedUsersFile = legacyAuthorizedUsersProp.isSet() ? legacyAuthorizedUsersProp.getValue() : null; - // extract any node identities nodeIdentities = new HashSet<>(); for (Map.Entry entry : configurationContext.getProperties().entrySet()) { @@ -594,20 +587,14 @@ public class FileAccessPolicyProvider implements ConfigurableAccessPolicyProvide final AuthorizationsHolder authorizationsHolder = new AuthorizationsHolder(authorizations); final boolean emptyAuthorizations = authorizationsHolder.getAllPolicies().isEmpty(); final boolean hasInitialAdminIdentity = (initialAdminIdentity != null && !StringUtils.isBlank(initialAdminIdentity)); - final boolean hasLegacyAuthorizedUsers = (legacyAuthorizedUsersFile != null && !StringUtils.isBlank(legacyAuthorizedUsersFile)); // if we are starting fresh then we might need to populate an initial admin or convert legacy users if (emptyAuthorizations) { parseFlow(); - if (hasInitialAdminIdentity && hasLegacyAuthorizedUsers) { - throw new AuthorizerCreationException("Cannot provide an Initial Admin Identity and a Legacy Authorized Users File"); - } else if (hasInitialAdminIdentity) { + if (hasInitialAdminIdentity) { logger.info("Populating authorizations for Initial Admin: " + initialAdminIdentity); populateInitialAdmin(authorizations); - } else if (hasLegacyAuthorizedUsers) { - logger.info("Converting " + legacyAuthorizedUsersFile + " to new authorizations model"); - convertLegacyAuthorizedUsers(authorizations); } populateNodes(authorizations); @@ -652,15 +639,7 @@ public class FileAccessPolicyProvider implements ConfigurableAccessPolicyProvide */ private void parseFlow() throws SAXException { final FlowParser flowParser = new FlowParser(); - - final File flowConfigurationFile; - final File jsonFile = properties.getFlowConfigurationJsonFile(); - if (jsonFile.exists()) { - flowConfigurationFile = jsonFile; - } else { - flowConfigurationFile = properties.getFlowConfigurationFile(); - } - + final File flowConfigurationFile = properties.getFlowConfigurationFile(); final FlowInfo flowInfo = flowParser.parse(flowConfigurationFile); if (flowInfo != null) { @@ -741,151 +720,6 @@ public class FileAccessPolicyProvider implements ConfigurableAccessPolicyProvide } } - /** - * Unmarshalls an existing authorized-users.xml and converts the object model to the new model. - * - * @param authorizations the current Authorizations instance that policies will be added to - * @throws AuthorizerCreationException if the legacy authorized users file that was provided does not exist - * @throws JAXBException if the legacy authorized users file that was provided could not be unmarshalled - */ - private void convertLegacyAuthorizedUsers(final Authorizations authorizations) throws AuthorizerCreationException, JAXBException { - final File authorizedUsersFile = new File(legacyAuthorizedUsersFile); - if (!authorizedUsersFile.exists()) { - throw new AuthorizerCreationException("Legacy Authorized Users File '" + legacyAuthorizedUsersFile + "' does not exists"); - } - - final Unmarshaller unmarshaller = JAXB_USERS_CONTEXT.createUnmarshaller(); - unmarshaller.setSchema(usersSchema); - - final XMLStreamReader xsr; - try { - final XMLStreamReaderProvider provider = new StandardXMLStreamReaderProvider(); - xsr = provider.getStreamReader(new StreamSource(authorizedUsersFile)); - } catch (final ProcessingException e) { - logger.error("Encountered an error reading authorized users file: ", e); - throw new JAXBException("Error reading authorized users file", e); - } - final JAXBElement element = unmarshaller.unmarshal( - xsr, org.apache.nifi.user.generated.Users.class); - - final org.apache.nifi.user.generated.Users users = element.getValue(); - if (users.getUser().isEmpty()) { - logger.info("Legacy Authorized Users File contained no users, nothing to convert"); - return; - } - - // get all the user DNs into a list - List userIdentities = new ArrayList<>(); - for (org.apache.nifi.user.generated.User legacyUser : users.getUser()) { - userIdentities.add(IdentityMappingUtil.mapIdentity(legacyUser.getDn(), identityMappings)); - } - - // sort the list and pull out the first identity - Collections.sort(userIdentities); - final String seedIdentity = userIdentities.get(0); - - // create mapping from Role to access policies - final Map> roleAccessPolicies = RoleAccessPolicy.getMappings(rootGroupId); - - final List allPolicies = new ArrayList<>(); - - for (org.apache.nifi.user.generated.User legacyUser : users.getUser()) { - // create the identifier of the new user based on the DN - final String legacyUserDn = IdentityMappingUtil.mapIdentity(legacyUser.getDn(), identityMappings); - final User user = userGroupProvider.getUserByIdentity(legacyUserDn); - if (user == null) { - throw new AuthorizerCreationException("Unable to locate legacy user " + legacyUserDn + " to seed policies."); - } - - // create policies based on the given role - for (org.apache.nifi.user.generated.Role jaxbRole : legacyUser.getRole()) { - Role role = Role.valueOf(jaxbRole.getName()); - Set policies = roleAccessPolicies.get(role); - - for (RoleAccessPolicy roleAccessPolicy : policies) { - - // get the matching policy, or create a new one - Policy policy = getOrCreatePolicy( - allPolicies, - seedIdentity, - roleAccessPolicy.getResource(), - roleAccessPolicy.getAction()); - - // add the user to the policy if it doesn't exist - addUserToPolicy(user.getIdentifier(), policy); - } - } - - } - - // convert any access controls on ports to the appropriate policies - for (PortDTO portDTO : ports) { - final Resource resource; - if (portDTO.getType() != null && portDTO.getType().equals("inputPort")) { - resource = ResourceFactory.getDataTransferResource(ResourceFactory.getComponentResource(ResourceType.InputPort, portDTO.getId(), portDTO.getName())); - } else { - resource = ResourceFactory.getDataTransferResource(ResourceFactory.getComponentResource(ResourceType.OutputPort, portDTO.getId(), portDTO.getName())); - } - - if (portDTO.getUserAccessControl() != null) { - for (String userAccessControl : portDTO.getUserAccessControl()) { - // need to perform the identity mapping on the access control so it matches the identities in the User objects - final String mappedUserAccessControl = IdentityMappingUtil.mapIdentity(userAccessControl, identityMappings); - final User foundUser = userGroupProvider.getUserByIdentity(mappedUserAccessControl); - - // couldn't find the user matching the access control so log a warning and skip - if (foundUser == null) { - logger.warn("Found port with user access control for {} but no user exists with this identity, skipping...", - new Object[] {mappedUserAccessControl}); - continue; - } - - // we found the user so create the appropriate policy and add the user to it - Policy policy = getOrCreatePolicy( - allPolicies, - seedIdentity, - resource.getIdentifier(), - WRITE_CODE); - - addUserToPolicy(foundUser.getIdentifier(), policy); - } - } - - if (portDTO.getGroupAccessControl() != null) { - for (String groupAccessControl : portDTO.getGroupAccessControl()) { - final String legacyGroupName = IdentityMappingUtil.mapIdentity(groupAccessControl, groupMappings); - - // find a group where the name is the groupAccessControl - Group foundGroup = null; - for (Group group : userGroupProvider.getGroups()) { - if (group.getName().equals(legacyGroupName)) { - foundGroup = group; - break; - } - } - - // couldn't find the group matching the access control so log a warning and skip - if (foundGroup == null) { - logger.warn("Found port with group access control for {} but no group exists with this name, skipping...", - new Object[] {legacyGroupName}); - continue; - } - - // we found the group so create the appropriate policy and add all the users to it - Policy policy = getOrCreatePolicy( - allPolicies, - seedIdentity, - resource.getIdentifier(), - WRITE_CODE); - - addGroupToPolicy(IdentifierUtil.getIdentifier(legacyGroupName), policy); - } - } - } - - authorizations.getPolicies().getPolicy().addAll(allPolicies); - } - /** * Creates and adds an access policy for the given resource, identity, and actions to the specified authorizations. * diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-file-authorizer/src/main/java/org/apache/nifi/authorization/FileAuthorizer.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-file-authorizer/src/main/java/org/apache/nifi/authorization/FileAuthorizer.java index 3700cf0c23..052f7d5f22 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-file-authorizer/src/main/java/org/apache/nifi/authorization/FileAuthorizer.java +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-file-authorizer/src/main/java/org/apache/nifi/authorization/FileAuthorizer.java @@ -39,8 +39,6 @@ public class FileAuthorizer extends AbstractPolicyBasedAuthorizer { private static final String FILE_USER_GROUP_PROVIDER_ID = "file-user-group-provider"; private static final String FILE_ACCESS_POLICY_PROVIDER_ID = "file-access-policy-provider"; - static final String PROP_LEGACY_AUTHORIZED_USERS_FILE = "Legacy Authorized Users File"; - private FileUserGroupProvider userGroupProvider = new FileUserGroupProvider(); private FileAccessPolicyProvider accessPolicyProvider = new FileAccessPolicyProvider(); @@ -93,9 +91,6 @@ public class FileAuthorizer extends AbstractPolicyBasedAuthorizer { if (configurationProperties.containsKey(FileUserGroupProvider.PROP_TENANTS_FILE)) { userGroupProperties.put(FileUserGroupProvider.PROP_TENANTS_FILE, configurationProperties.get(FileUserGroupProvider.PROP_TENANTS_FILE)); } - if (configurationProperties.containsKey(FileAuthorizer.PROP_LEGACY_AUTHORIZED_USERS_FILE)) { - userGroupProperties.put(FileAuthorizer.PROP_LEGACY_AUTHORIZED_USERS_FILE, configurationProperties.get(FileAuthorizer.PROP_LEGACY_AUTHORIZED_USERS_FILE)); - } // relay the relevant config final Map accessPolicyProperties = new HashMap<>(); @@ -106,9 +101,6 @@ public class FileAuthorizer extends AbstractPolicyBasedAuthorizer { if (configurationProperties.containsKey(FileAccessPolicyProvider.PROP_INITIAL_ADMIN_IDENTITY)) { accessPolicyProperties.put(FileAccessPolicyProvider.PROP_INITIAL_ADMIN_IDENTITY, configurationProperties.get(FileAccessPolicyProvider.PROP_INITIAL_ADMIN_IDENTITY)); } - if (configurationProperties.containsKey(FileAuthorizer.PROP_LEGACY_AUTHORIZED_USERS_FILE)) { - accessPolicyProperties.put(FileAuthorizer.PROP_LEGACY_AUTHORIZED_USERS_FILE, configurationProperties.get(FileAuthorizer.PROP_LEGACY_AUTHORIZED_USERS_FILE)); - } // ensure all node identities are seeded into the user provider configurationProperties.forEach((property, value) -> { diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-file-authorizer/src/main/java/org/apache/nifi/authorization/FileUserGroupProvider.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-file-authorizer/src/main/java/org/apache/nifi/authorization/FileUserGroupProvider.java index 855e80a331..5e39ff4657 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-file-authorizer/src/main/java/org/apache/nifi/authorization/FileUserGroupProvider.java +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-file-authorizer/src/main/java/org/apache/nifi/authorization/FileUserGroupProvider.java @@ -116,7 +116,6 @@ public class FileUserGroupProvider implements ConfigurableUserGroupProvider { private NiFiProperties properties; private File tenantsFile; private File restoreTenantsFile; - private String legacyAuthorizedUsersFile; private Set initialUserIdentities; private List identityMappings; private List groupMappings; @@ -178,10 +177,6 @@ public class FileUserGroupProvider implements ConfigurableUserGroupProvider { identityMappings = Collections.unmodifiableList(IdentityMappingUtil.getIdentityMappings(properties)); groupMappings = Collections.unmodifiableList(IdentityMappingUtil.getGroupMappings(properties)); - // get the value of the legacy authorized users file - final PropertyValue legacyAuthorizedUsersProp = configurationContext.getProperty(FileAuthorizer.PROP_LEGACY_AUTHORIZED_USERS_FILE); - legacyAuthorizedUsersFile = legacyAuthorizedUsersProp.isSet() ? legacyAuthorizedUsersProp.getValue() : null; - // extract any node identities initialUserIdentities = new HashSet<>(); for (Map.Entry entry : configurationContext.getProperties().entrySet()) { @@ -692,14 +687,8 @@ public class FileUserGroupProvider implements ConfigurableUserGroupProvider { final UserGroupHolder userGroupHolder = new UserGroupHolder(tenants); final boolean emptyTenants = userGroupHolder.getAllUsers().isEmpty() && userGroupHolder.getAllGroups().isEmpty(); - final boolean hasLegacyAuthorizedUsers = (legacyAuthorizedUsersFile != null && !StringUtils.isBlank(legacyAuthorizedUsersFile)); if (emptyTenants) { - if (hasLegacyAuthorizedUsers) { - logger.info("Loading users from legacy model " + legacyAuthorizedUsersFile + " into new users file."); - convertLegacyAuthorizedUsers(tenants); - } - populateInitialUsers(tenants); // save any changes that were made and repopulate the holder @@ -740,55 +729,6 @@ public class FileUserGroupProvider implements ConfigurableUserGroupProvider { } } - /** - * Unmarshalls an existing authorized-users.xml and converts the object model to the new model. - * - * @param tenants the current Tenants instance users and groups will be added to - * @throws AuthorizerCreationException if the legacy authorized users file that was provided does not exist - * @throws JAXBException if the legacy authorized users file that was provided could not be unmarshalled - */ - private void convertLegacyAuthorizedUsers(final Tenants tenants) throws AuthorizerCreationException, JAXBException { - final File authorizedUsersFile = new File(legacyAuthorizedUsersFile); - if (!authorizedUsersFile.exists()) { - throw new AuthorizerCreationException("Legacy Authorized Users File '" + legacyAuthorizedUsersFile + "' does not exists"); - } - - final XMLStreamReaderProvider provider = new StandardXMLStreamReaderProvider(); - final XMLStreamReader xsr; - try { - xsr = provider.getStreamReader(new StreamSource(authorizedUsersFile)); - } catch (final ProcessingException e) { - throw new AuthorizerCreationException("Error converting the legacy authorizers file", e); - } - - final Unmarshaller unmarshaller = JAXB_USERS_CONTEXT.createUnmarshaller(); - unmarshaller.setSchema(usersSchema); - - final JAXBElement element = unmarshaller.unmarshal( - xsr, org.apache.nifi.user.generated.Users.class); - - final org.apache.nifi.user.generated.Users users = element.getValue(); - if (users.getUser().isEmpty()) { - logger.info("Legacy Authorized Users File contained no users, nothing to convert"); - return; - } - - for (org.apache.nifi.user.generated.User legacyUser : users.getUser()) { - // create the identifier of the new user based on the DN - final String legacyUserDn = IdentityMappingUtil.mapIdentity(legacyUser.getDn(), identityMappings); - org.apache.nifi.authorization.file.tenants.generated.User user = getOrCreateUser(tenants, legacyUserDn); - - // if there was a group name find or create the group and add the user to it - if (StringUtils.isNotBlank(legacyUser.getGroup())) { - final String legacyGroupName = IdentityMappingUtil.mapIdentity(legacyUser.getGroup(), groupMappings); - org.apache.nifi.authorization.file.tenants.generated.Group group = getOrCreateGroup(tenants, legacyGroupName); - org.apache.nifi.authorization.file.tenants.generated.Group.User groupUser = new org.apache.nifi.authorization.file.tenants.generated.Group.User(); - groupUser.setIdentifier(user.getIdentifier()); - group.getUser().add(groupUser); - } - } - } - /** * Finds the User with the given identity, or creates a new one and adds it to the Tenants. * diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-file-authorizer/src/test/java/org/apache/nifi/authorization/FileAccessPolicyProviderTest.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-file-authorizer/src/test/java/org/apache/nifi/authorization/FileAccessPolicyProviderTest.java index 2c60e59d27..c4f487a18d 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-file-authorizer/src/test/java/org/apache/nifi/authorization/FileAccessPolicyProviderTest.java +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-file-authorizer/src/test/java/org/apache/nifi/authorization/FileAccessPolicyProviderTest.java @@ -16,12 +16,11 @@ */ package org.apache.nifi.authorization; -import org.apache.nifi.parameter.ParameterLookup; import org.apache.nifi.attribute.expression.language.StandardPropertyValue; import org.apache.nifi.authorization.exception.AuthorizerCreationException; -import org.apache.nifi.authorization.resource.ResourceFactory; import org.apache.nifi.authorization.resource.ResourceType; import org.apache.nifi.components.PropertyValue; +import org.apache.nifi.parameter.ParameterLookup; import org.apache.nifi.util.NiFiProperties; import org.apache.nifi.util.file.FileUtils; import org.junit.jupiter.api.AfterEach; @@ -36,7 +35,6 @@ import java.io.FileOutputStream; import java.io.IOException; import java.nio.charset.StandardCharsets; import java.util.HashMap; -import java.util.HashSet; import java.util.Map; import java.util.Properties; import java.util.Set; @@ -146,7 +144,6 @@ public class FileAccessPolicyProviderTest { private File restoreAuthorizations; private File restoreTenants; private File flow; - private File flowJson; private File flowNoPorts; private File flowWithDns; @@ -170,21 +167,18 @@ public class FileAccessPolicyProviderTest { restoreTenants = new File("target/restore/users.xml"); FileUtils.ensureDirectoryExistAndCanAccess(restoreTenants.getParentFile()); - flow = new File("src/test/resources/flow.xml.gz"); + flow = new File("src/test/resources/flow.json.gz"); FileUtils.ensureDirectoryExistAndCanAccess(flow.getParentFile()); - flowJson = new File("src/test/resources/flow.json.gz"); - - flowNoPorts = new File("src/test/resources/flow-no-ports.xml.gz"); + flowNoPorts = new File("src/test/resources/flow-no-ports.json.gz"); FileUtils.ensureDirectoryExistAndCanAccess(flowNoPorts.getParentFile()); - flowWithDns = new File("src/test/resources/flow-with-dns.xml.gz"); + flowWithDns = new File("src/test/resources/flow-with-dns.json.gz"); FileUtils.ensureDirectoryExistAndCanAccess(flowWithDns.getParentFile()); properties = mock(NiFiProperties.class); when(properties.getRestoreDirectory()).thenReturn(restoreAuthorizations.getParentFile()); when(properties.getFlowConfigurationFile()).thenReturn(flow); - when(properties.getFlowConfigurationJsonFile()).thenReturn(flowJson); userGroupProvider = new FileUserGroupProvider(); userGroupProvider.setNiFiProperties(properties); @@ -196,7 +190,6 @@ public class FileAccessPolicyProviderTest { ParameterLookup.EMPTY)); when(configurationContext.getProperty(eq(FileUserGroupProvider.PROP_TENANTS_FILE))).thenReturn(new StandardPropertyValue(primaryTenants.getPath(), null, ParameterLookup.EMPTY)); when(configurationContext.getProperty(eq(FileAccessPolicyProvider.PROP_INITIAL_ADMIN_IDENTITY))).thenReturn(new StandardPropertyValue(null, null, ParameterLookup.EMPTY)); - when(configurationContext.getProperty(eq(FileAuthorizer.PROP_LEGACY_AUTHORIZED_USERS_FILE))).thenReturn(new StandardPropertyValue(null, null, ParameterLookup.EMPTY)); when(configurationContext.getProperty(eq(FileAccessPolicyProvider.PROP_USER_GROUP_PROVIDER))).thenReturn(new StandardPropertyValue("user-group-provider", null, ParameterLookup.EMPTY)); when(configurationContext.getProperties()).then((invocation) -> { @@ -212,11 +205,6 @@ public class FileAccessPolicyProviderTest { properties.put(FileUserGroupProvider.PROP_TENANTS_FILE, tenantFile.getValue()); } - final PropertyValue legacyAuthFile = configurationContext.getProperty(FileAuthorizer.PROP_LEGACY_AUTHORIZED_USERS_FILE); - if (legacyAuthFile != null) { - properties.put(FileAuthorizer.PROP_LEGACY_AUTHORIZED_USERS_FILE, legacyAuthFile.getValue()); - } - final PropertyValue initialAdmin = configurationContext.getProperty(FileAccessPolicyProvider.PROP_INITIAL_ADMIN_IDENTITY); if (initialAdmin != null) { properties.put(FileAccessPolicyProvider.PROP_INITIAL_ADMIN_IDENTITY, initialAdmin.getValue()); @@ -280,262 +268,6 @@ public class FileAccessPolicyProviderTest { deleteFile(restoreTenants); } - @Test - public void testOnConfiguredWhenLegacyUsersFileProvidedWithOverlappingRoles() throws Exception { - when(configurationContext.getProperty(eq(FileAuthorizer.PROP_LEGACY_AUTHORIZED_USERS_FILE))) - .thenReturn(new StandardPropertyValue("src/test/resources/authorized-users-multirole.xml", null, ParameterLookup.EMPTY)); - - writeFile(primaryAuthorizations, EMPTY_AUTHORIZATIONS_CONCISE); - writeFile(primaryTenants, EMPTY_TENANTS_CONCISE); - - userGroupProvider.onConfigured(configurationContext); - accessPolicyProvider.onConfigured(configurationContext); - - assertNotNull(accessPolicyProvider.getAccessPolicy(ResourceType.Flow.getValue(), RequestAction.READ)); - assertNotNull(accessPolicyProvider.getAccessPolicy(ResourceType.Controller.getValue(), RequestAction.READ)); - assertNotNull(accessPolicyProvider.getAccessPolicy(ResourceType.Controller.getValue(), RequestAction.WRITE)); - assertNotNull(accessPolicyProvider.getAccessPolicy(ResourceType.System.getValue(), RequestAction.READ)); - assertNotNull(accessPolicyProvider.getAccessPolicy(ResourceType.ProcessGroup.getValue() + "/" + ROOT_GROUP_ID, RequestAction.READ)); - assertNotNull(accessPolicyProvider.getAccessPolicy(ResourceType.ProcessGroup.getValue() + "/" + ROOT_GROUP_ID, RequestAction.WRITE)); - } - - @Test - public void testOnConfiguredWhenLegacyUsersFileProvidedAndFlowHasNoPorts() throws Exception { - properties = mock(NiFiProperties.class); - when(properties.getRestoreDirectory()).thenReturn(restoreAuthorizations.getParentFile()); - when(properties.getFlowConfigurationFile()).thenReturn(flowNoPorts); - - when(configurationContext.getProperty(eq(FileAuthorizer.PROP_LEGACY_AUTHORIZED_USERS_FILE))) - .thenReturn(new StandardPropertyValue("src/test/resources/authorized-users.xml", null, ParameterLookup.EMPTY)); - - writeFile(primaryAuthorizations, EMPTY_AUTHORIZATIONS_CONCISE); - writeFile(primaryTenants, EMPTY_TENANTS_CONCISE); - - userGroupProvider.onConfigured(configurationContext); - accessPolicyProvider.onConfigured(configurationContext); - - boolean foundDataTransferPolicy = false; - for (AccessPolicy policy : accessPolicyProvider.getAccessPolicies()) { - if (policy.getResource().contains(ResourceType.DataTransfer.name())) { - foundDataTransferPolicy = true; - break; - } - } - - assertFalse(foundDataTransferPolicy); - } - - @Test - public void testOnConfiguredWhenLegacyUsersFileProvided() throws Exception { - when(configurationContext.getProperty(eq(FileAuthorizer.PROP_LEGACY_AUTHORIZED_USERS_FILE))) - .thenReturn(new StandardPropertyValue("src/test/resources/authorized-users.xml", null, ParameterLookup.EMPTY)); - - writeFile(primaryAuthorizations, EMPTY_AUTHORIZATIONS_CONCISE); - writeFile(primaryTenants, EMPTY_TENANTS_CONCISE); - - userGroupProvider.onConfigured(configurationContext); - accessPolicyProvider.onConfigured(configurationContext); - - final User user1 = userGroupProvider.getUserByIdentity("user1"); - final User user2 = userGroupProvider.getUserByIdentity("user2"); - final User user3 = userGroupProvider.getUserByIdentity("user3"); - final User user4 = userGroupProvider.getUserByIdentity("user4"); - final User user5 = userGroupProvider.getUserByIdentity("user5"); - final User user6 = userGroupProvider.getUserByIdentity("user6"); - - // verify one group got created - final Set groups = userGroupProvider.getGroups(); - final Group group1 = groups.iterator().next(); - - // verify more than one policy got created - final Set policies = accessPolicyProvider.getAccessPolicies(); - assertTrue(policies.size() > 0); - - // verify user1's policies - final Map> user1Policies = getResourceActions(policies, user1); - assertEquals(4, user1Policies.size()); - - assertTrue(user1Policies.containsKey(ResourceType.Flow.getValue())); - assertEquals(1, user1Policies.get(ResourceType.Flow.getValue()).size()); - assertTrue(user1Policies.get(ResourceType.Flow.getValue()).contains(RequestAction.READ)); - - assertTrue(user1Policies.containsKey(ResourceType.ProcessGroup.getValue() + "/" + ROOT_GROUP_ID)); - assertEquals(1, user1Policies.get(ResourceType.ProcessGroup.getValue() + "/" + ROOT_GROUP_ID).size()); - assertTrue(user1Policies.get(ResourceType.ProcessGroup.getValue() + "/" + ROOT_GROUP_ID).contains(RequestAction.READ)); - - // verify user2's policies - final Map> user2Policies = getResourceActions(policies, user2); - assertEquals(3, user2Policies.size()); - - assertTrue(user2Policies.containsKey(ResourceType.Provenance.getValue())); - assertEquals(1, user2Policies.get(ResourceType.Provenance.getValue()).size()); - assertTrue(user2Policies.get(ResourceType.Provenance.getValue()).contains(RequestAction.READ)); - - assertTrue(user2Policies.containsKey(ResourceType.ProvenanceData.getValue() + "/process-groups/" + ROOT_GROUP_ID)); - assertEquals(1, user2Policies.get(ResourceType.ProvenanceData.getValue() + "/process-groups/" + ROOT_GROUP_ID).size()); - assertTrue(user2Policies.get(ResourceType.ProvenanceData.getValue() + "/process-groups/" + ROOT_GROUP_ID).contains(RequestAction.READ)); - - assertTrue(user2Policies.containsKey(ResourceType.Data.getValue() + "/process-groups/" + ROOT_GROUP_ID)); - assertEquals(1, user2Policies.get(ResourceType.Data.getValue() + "/process-groups/" + ROOT_GROUP_ID).size()); - assertTrue(user2Policies.get(ResourceType.Data.getValue() + "/process-groups/" + ROOT_GROUP_ID).contains(RequestAction.READ)); - - // verify user3's policies - final Map> user3Policies = getResourceActions(policies, user3); - assertEquals(6, user3Policies.size()); - - assertTrue(user3Policies.containsKey(ResourceType.Flow.getValue())); - assertEquals(1, user3Policies.get(ResourceType.Flow.getValue()).size()); - assertTrue(user3Policies.get(ResourceType.Flow.getValue()).contains(RequestAction.READ)); - - assertTrue(user3Policies.containsKey(ResourceType.ProcessGroup.getValue() + "/" + ROOT_GROUP_ID)); - assertEquals(2, user3Policies.get(ResourceType.ProcessGroup.getValue() + "/" + ROOT_GROUP_ID).size()); - assertTrue(user3Policies.get(ResourceType.ProcessGroup.getValue() + "/" + ROOT_GROUP_ID).contains(RequestAction.WRITE)); - - // verify user4's policies - final Map> user4Policies = getResourceActions(policies, user4); - assertEquals(6, user4Policies.size()); - - assertTrue(user4Policies.containsKey(ResourceType.Flow.getValue())); - assertEquals(1, user4Policies.get(ResourceType.Flow.getValue()).size()); - assertTrue(user4Policies.get(ResourceType.Flow.getValue()).contains(RequestAction.READ)); - - assertTrue(user4Policies.containsKey(ResourceType.ProcessGroup.getValue() + "/" + ROOT_GROUP_ID)); - assertEquals(1, user4Policies.get(ResourceType.ProcessGroup.getValue() + "/" + ROOT_GROUP_ID).size()); - assertTrue(user4Policies.get(ResourceType.ProcessGroup.getValue() + "/" + ROOT_GROUP_ID).contains(RequestAction.READ)); - - assertTrue(user4Policies.containsKey(ResourceType.Tenant.getValue())); - assertEquals(2, user4Policies.get(ResourceType.Tenant.getValue()).size()); - assertTrue(user4Policies.get(ResourceType.Tenant.getValue()).contains(RequestAction.WRITE)); - - assertTrue(user4Policies.containsKey(ResourceType.Policy.getValue())); - assertEquals(2, user4Policies.get(ResourceType.Policy.getValue()).size()); - assertTrue(user4Policies.get(ResourceType.Policy.getValue()).contains(RequestAction.WRITE)); - - // verify user5's policies - final Map> user5Policies = getResourceActions(policies, user5); - assertEquals(2, user5Policies.size()); - - assertTrue(user5Policies.containsKey(ResourceType.Proxy.getValue())); - assertEquals(1, user5Policies.get(ResourceType.Proxy.getValue()).size()); - assertTrue(user5Policies.get(ResourceType.Proxy.getValue()).contains(RequestAction.WRITE)); - - // verify user6's policies - final Map> user6Policies = getResourceActions(policies, user6); - assertEquals(3, user6Policies.size()); - - assertTrue(user6Policies.containsKey(ResourceType.SiteToSite.getValue())); - assertEquals(1, user6Policies.get(ResourceType.SiteToSite.getValue()).size()); - assertTrue(user6Policies.get(ResourceType.SiteToSite.getValue()).contains(RequestAction.READ)); - - final Resource inputPortResource = ResourceFactory.getDataTransferResource( - ResourceFactory.getComponentResource(ResourceType.InputPort, "2f7d1606-b090-4be7-a592-a5b70fb55531", "TCP Input")); - final AccessPolicy inputPortPolicy = accessPolicyProvider.getAccessPolicy(inputPortResource.getIdentifier(), RequestAction.WRITE); - assertNotNull(inputPortPolicy); - assertEquals(1, inputPortPolicy.getUsers().size()); - assertTrue(inputPortPolicy.getUsers().contains(user6.getIdentifier())); - assertEquals(1, inputPortPolicy.getGroups().size()); - assertTrue(inputPortPolicy.getGroups().contains(group1.getIdentifier())); - - final Resource outputPortResource = ResourceFactory.getDataTransferResource( - ResourceFactory.getComponentResource(ResourceType.OutputPort, "2f7d1606-b090-4be7-a592-a5b70fb55532", "TCP Output")); - final AccessPolicy outputPortPolicy = accessPolicyProvider.getAccessPolicy(outputPortResource.getIdentifier(), RequestAction.WRITE); - assertNotNull(outputPortPolicy); - assertEquals(1, outputPortPolicy.getUsers().size()); - assertTrue(outputPortPolicy.getUsers().contains(user4.getIdentifier())); - } - - private Map> getResourceActions(final Set policies, final User user) { - Map> resourceActionMap = new HashMap<>(); - - for (AccessPolicy accessPolicy : policies) { - if (accessPolicy.getUsers().contains(user.getIdentifier())) { - Set actions = resourceActionMap.get(accessPolicy.getResource()); - if (actions == null) { - actions = new HashSet<>(); - resourceActionMap.put(accessPolicy.getResource(), actions); - } - actions.add(accessPolicy.getAction()); - } - } - - return resourceActionMap; - } - - @Test - public void testOnConfiguredWhenLegacyUsersFileProvidedWithIdentityMappings() throws Exception { - final Properties props = new Properties(); - props.setProperty("nifi.security.identity.mapping.pattern.dn1", "^CN=(.*?), OU=(.*?), O=(.*?), L=(.*?), ST=(.*?), C=(.*?)$"); - props.setProperty("nifi.security.identity.mapping.value.dn1", "$1"); - - properties = getNiFiProperties(props); - when(properties.getRestoreDirectory()).thenReturn(restoreAuthorizations.getParentFile()); - when(properties.getFlowConfigurationFile()).thenReturn(flowWithDns); - when(properties.getFlowConfigurationJsonFile()).thenReturn(flowJson); - - userGroupProvider.setNiFiProperties(properties); - accessPolicyProvider.setNiFiProperties(properties); - - when(configurationContext.getProperty(eq(FileAuthorizer.PROP_LEGACY_AUTHORIZED_USERS_FILE))) - .thenReturn(new StandardPropertyValue("src/test/resources/authorized-users-with-dns.xml", null, ParameterLookup.EMPTY)); - - writeFile(primaryAuthorizations, EMPTY_AUTHORIZATIONS_CONCISE); - writeFile(primaryTenants, EMPTY_TENANTS_CONCISE); - - userGroupProvider.onConfigured(configurationContext); - accessPolicyProvider.onConfigured(configurationContext); - - final User user4 = userGroupProvider.getUserByIdentity("user4"); - final User user6 = userGroupProvider.getUserByIdentity("user6"); - - // verify one group got created - final Set groups = userGroupProvider.getGroups(); - final Group group1 = groups.iterator().next(); - - final Resource inputPortResource = ResourceFactory.getDataTransferResource( - ResourceFactory.getComponentResource(ResourceType.InputPort, "2f7d1606-b090-4be7-a592-a5b70fb55531", "TCP Input")); - final AccessPolicy inputPortPolicy = accessPolicyProvider.getAccessPolicy(inputPortResource.getIdentifier(), RequestAction.WRITE); - assertNotNull(inputPortPolicy); - assertEquals(1, inputPortPolicy.getUsers().size()); - assertTrue(inputPortPolicy.getUsers().contains(user6.getIdentifier())); - assertEquals(1, inputPortPolicy.getGroups().size()); - assertTrue(inputPortPolicy.getGroups().contains(group1.getIdentifier())); - - final Resource outputPortResource = ResourceFactory.getDataTransferResource( - ResourceFactory.getComponentResource(ResourceType.OutputPort, "2f7d1606-b090-4be7-a592-a5b70fb55532", "TCP Output")); - final AccessPolicy outputPortPolicy = accessPolicyProvider.getAccessPolicy(outputPortResource.getIdentifier(), RequestAction.WRITE); - assertNotNull(outputPortPolicy); - assertEquals(1, outputPortPolicy.getUsers().size()); - assertTrue(outputPortPolicy.getUsers().contains(user4.getIdentifier())); - } - - @Test - public void testOnConfiguredWhenBadLegacyUsersFileProvided() throws Exception { - when(configurationContext.getProperty(eq(FileAuthorizer.PROP_LEGACY_AUTHORIZED_USERS_FILE))) - .thenReturn(new StandardPropertyValue("src/test/resources/does-not-exist.xml", null, ParameterLookup.EMPTY)); - - writeFile(primaryAuthorizations, EMPTY_AUTHORIZATIONS_CONCISE); - writeFile(primaryTenants, EMPTY_TENANTS_CONCISE); - - assertThrows(AuthorizerCreationException.class, - () -> accessPolicyProvider.onConfigured(configurationContext)); - } - - @Test - public void testOnConfiguredWhenInitialAdminAndLegacyUsersProvided() throws Exception { - final String adminIdentity = "admin-user"; - when(configurationContext.getProperty(eq(FileAccessPolicyProvider.PROP_INITIAL_ADMIN_IDENTITY))) - .thenReturn(new StandardPropertyValue(adminIdentity, null, ParameterLookup.EMPTY)); - - when(configurationContext.getProperty(eq(FileAuthorizer.PROP_LEGACY_AUTHORIZED_USERS_FILE))) - .thenReturn(new StandardPropertyValue("src/test/resources/authorized-users.xml", null, ParameterLookup.EMPTY)); - - writeFile(primaryAuthorizations, EMPTY_AUTHORIZATIONS_CONCISE); - writeFile(primaryTenants, EMPTY_TENANTS_CONCISE); - - assertThrows(AuthorizerCreationException.class, - () -> accessPolicyProvider.onConfigured(configurationContext)); - } - @Test public void testOnConfiguredWhenInitialAdminNotProvided() throws Exception { writeFile(primaryAuthorizations, EMPTY_AUTHORIZATIONS_CONCISE); @@ -586,8 +318,7 @@ public class FileAccessPolicyProviderTest { // setup NiFi properties to return a file that does not exist properties = mock(NiFiProperties.class); when(properties.getRestoreDirectory()).thenReturn(restoreAuthorizations.getParentFile()); - when(properties.getFlowConfigurationFile()).thenReturn(new File("src/test/resources/does-not-exist.xml.gz")); - when(properties.getFlowConfigurationJsonFile()).thenReturn(flowJson); + when(properties.getFlowConfigurationFile()).thenReturn(new File("src/test/resources/does-not-exist.json.gz")); userGroupProvider.setNiFiProperties(properties); accessPolicyProvider.setNiFiProperties(properties); @@ -627,8 +358,7 @@ public class FileAccessPolicyProviderTest { // setup NiFi properties to return a file that does not exist properties = mock(NiFiProperties.class); when(properties.getRestoreDirectory()).thenReturn(restoreAuthorizations.getParentFile()); - when(properties.getFlowConfigurationFile()).thenReturn(null); - when(properties.getFlowConfigurationJsonFile()).thenReturn(flowJson); + when(properties.getFlowConfigurationFile()).thenReturn(new File("src/test/resources/does-not-exist.json.gz")); userGroupProvider.setNiFiProperties(properties); accessPolicyProvider.setNiFiProperties(properties); @@ -672,7 +402,6 @@ public class FileAccessPolicyProviderTest { properties = getNiFiProperties(props); when(properties.getRestoreDirectory()).thenReturn(restoreAuthorizations.getParentFile()); when(properties.getFlowConfigurationFile()).thenReturn(flow); - when(properties.getFlowConfigurationJsonFile()).thenReturn(flowJson); userGroupProvider.setNiFiProperties(properties); accessPolicyProvider.setNiFiProperties(properties); diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-file-authorizer/src/test/java/org/apache/nifi/authorization/FileAuthorizerTest.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-file-authorizer/src/test/java/org/apache/nifi/authorization/FileAuthorizerTest.java index 85aa1410d3..b567325554 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-file-authorizer/src/test/java/org/apache/nifi/authorization/FileAuthorizerTest.java +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-file-authorizer/src/test/java/org/apache/nifi/authorization/FileAuthorizerTest.java @@ -17,19 +17,19 @@ package org.apache.nifi.authorization; import org.apache.commons.lang3.SystemUtils; -import org.apache.nifi.parameter.ParameterLookup; import org.apache.nifi.attribute.expression.language.StandardPropertyValue; import org.apache.nifi.authorization.AuthorizationResult.Result; import org.apache.nifi.authorization.exception.AuthorizerCreationException; import org.apache.nifi.authorization.resource.ResourceFactory; import org.apache.nifi.authorization.resource.ResourceType; import org.apache.nifi.components.PropertyValue; +import org.apache.nifi.parameter.ParameterLookup; import org.apache.nifi.util.NiFiProperties; import org.apache.nifi.util.file.FileUtils; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.Assumptions; -import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.mockito.Mockito; import org.mockito.invocation.InvocationOnMock; @@ -40,7 +40,6 @@ import java.io.FileOutputStream; import java.io.IOException; import java.nio.charset.StandardCharsets; import java.util.HashMap; -import java.util.HashSet; import java.util.Map; import java.util.Properties; import java.util.Set; @@ -157,7 +156,6 @@ public class FileAuthorizerTest { private File restoreAuthorizations; private File restoreTenants; private File flow; - private File flowJson; private File flowNoPorts; private File flowWithDns; @@ -186,21 +184,18 @@ public class FileAuthorizerTest { restoreTenants = new File("target/restore/users.xml"); FileUtils.ensureDirectoryExistAndCanAccess(restoreTenants.getParentFile()); - flow = new File("src/test/resources/flow.xml.gz"); + flow = new File("src/test/resources/flow.json.gz"); FileUtils.ensureDirectoryExistAndCanAccess(flow.getParentFile()); - flowJson = new File("src/test/resources/flow.json.gz"); - - flowNoPorts = new File("src/test/resources/flow-no-ports.xml.gz"); + flowNoPorts = new File("src/test/resources/flow-no-ports.json.gz"); FileUtils.ensureDirectoryExistAndCanAccess(flowNoPorts.getParentFile()); - flowWithDns = new File("src/test/resources/flow-with-dns.xml.gz"); + flowWithDns = new File("src/test/resources/flow-with-dns.json.gz"); FileUtils.ensureDirectoryExistAndCanAccess(flowWithDns.getParentFile()); properties = mock(NiFiProperties.class); when(properties.getRestoreDirectory()).thenReturn(restoreAuthorizations.getParentFile()); when(properties.getFlowConfigurationFile()).thenReturn(flow); - when(properties.getFlowConfigurationJsonFile()).thenReturn(flowJson); configurationContext = mock(AuthorizerConfigurationContext.class); when(configurationContext.getProperty(Mockito.eq(FileAccessPolicyProvider.PROP_AUTHORIZATIONS_FILE))).thenReturn(new StandardPropertyValue(primaryAuthorizations.getPath(), null, @@ -219,11 +214,6 @@ public class FileAuthorizerTest { properties.put(FileUserGroupProvider.PROP_TENANTS_FILE, tenantFile.getValue()); } - final PropertyValue legacyAuthFile = configurationContext.getProperty(FileAuthorizer.PROP_LEGACY_AUTHORIZED_USERS_FILE); - if (legacyAuthFile != null) { - properties.put(FileAuthorizer.PROP_LEGACY_AUTHORIZED_USERS_FILE, legacyAuthFile.getValue()); - } - final PropertyValue initialAdmin = configurationContext.getProperty(FileAccessPolicyProvider.PROP_INITIAL_ADMIN_IDENTITY); if (initialAdmin != null) { properties.put(FileAccessPolicyProvider.PROP_INITIAL_ADMIN_IDENTITY, initialAdmin.getValue()); @@ -256,294 +246,6 @@ public class FileAuthorizerTest { deleteFile(restoreTenants); } - @Test - public void testOnConfiguredWhenLegacyUsersFileProvidedWithOverlappingRoles() throws Exception { - when(configurationContext.getProperty(Mockito.eq(FileAuthorizer.PROP_LEGACY_AUTHORIZED_USERS_FILE))) - .thenReturn(new StandardPropertyValue("src/test/resources/authorized-users-multirole.xml", null, ParameterLookup.EMPTY)); - - writeFile(primaryAuthorizations, EMPTY_AUTHORIZATIONS_CONCISE); - writeFile(primaryTenants, EMPTY_TENANTS_CONCISE); - authorizer.onConfigured(configurationContext); - - final Set users = authorizer.getUsers(); - assertEquals(1, users.size()); - - UsersAndAccessPolicies usersAndAccessPolicies = authorizer.getUsersAndAccessPolicies(); - assertNotNull(usersAndAccessPolicies.getAccessPolicy(ResourceType.Flow.getValue(), RequestAction.READ)); - assertNotNull(usersAndAccessPolicies.getAccessPolicy(ResourceType.Controller.getValue(), RequestAction.READ)); - assertNotNull(usersAndAccessPolicies.getAccessPolicy(ResourceType.Controller.getValue(), RequestAction.WRITE)); - assertNotNull(usersAndAccessPolicies.getAccessPolicy(ResourceType.System.getValue(), RequestAction.READ)); - assertNotNull(usersAndAccessPolicies.getAccessPolicy(ResourceType.ProcessGroup.getValue() + "/" + ROOT_GROUP_ID, RequestAction.READ)); - assertNotNull(usersAndAccessPolicies.getAccessPolicy(ResourceType.ProcessGroup.getValue() + "/" + ROOT_GROUP_ID, RequestAction.WRITE)); - } - - @Test - public void testOnConfiguredWhenLegacyUsersFileProvidedAndFlowHasNoPorts() throws Exception { - properties = mock(NiFiProperties.class); - when(properties.getRestoreDirectory()).thenReturn(restoreAuthorizations.getParentFile()); - when(properties.getFlowConfigurationFile()).thenReturn(flowNoPorts); - - when(configurationContext.getProperty(Mockito.eq(FileAuthorizer.PROP_LEGACY_AUTHORIZED_USERS_FILE))) - .thenReturn(new StandardPropertyValue("src/test/resources/authorized-users.xml", null, ParameterLookup.EMPTY)); - - writeFile(primaryAuthorizations, EMPTY_AUTHORIZATIONS_CONCISE); - writeFile(primaryTenants, EMPTY_TENANTS_CONCISE); - authorizer.onConfigured(configurationContext); - - boolean foundDataTransferPolicy = false; - for (AccessPolicy policy : authorizer.getAccessPolicies()) { - if (policy.getResource().contains(ResourceType.DataTransfer.name())) { - foundDataTransferPolicy = true; - break; - } - } - - assertFalse(foundDataTransferPolicy); - } - - @Test - public void testOnConfiguredWhenLegacyUsersFileProvided() throws Exception { - when(configurationContext.getProperty(Mockito.eq(FileAuthorizer.PROP_LEGACY_AUTHORIZED_USERS_FILE))) - .thenReturn(new StandardPropertyValue("src/test/resources/authorized-users.xml", null, ParameterLookup.EMPTY)); - - writeFile(primaryAuthorizations, EMPTY_AUTHORIZATIONS_CONCISE); - writeFile(primaryTenants, EMPTY_TENANTS_CONCISE); - authorizer.onConfigured(configurationContext); - - // verify all users got created correctly - final Set users = authorizer.getUsers(); - assertEquals(6, users.size()); - - final User user1 = authorizer.getUserByIdentity("user1"); - assertNotNull(user1); - - final User user2 = authorizer.getUserByIdentity("user2"); - assertNotNull(user2); - - final User user3 = authorizer.getUserByIdentity("user3"); - assertNotNull(user3); - - final User user4 = authorizer.getUserByIdentity("user4"); - assertNotNull(user4); - - final User user5 = authorizer.getUserByIdentity("user5"); - assertNotNull(user5); - - final User user6 = authorizer.getUserByIdentity("user6"); - assertNotNull(user6); - - // verify one group got created - final Set groups = authorizer.getGroups(); - assertEquals(1, groups.size()); - final Group group1 = groups.iterator().next(); - assertEquals("group1", group1.getName()); - - // verify more than one policy got created - final Set policies = authorizer.getAccessPolicies(); - assertTrue(policies.size() > 0); - - // verify user1's policies - final Map> user1Policies = getResourceActions(policies, user1); - assertEquals(4, user1Policies.size()); - - assertTrue(user1Policies.containsKey(ResourceType.Flow.getValue())); - assertEquals(1, user1Policies.get(ResourceType.Flow.getValue()).size()); - assertTrue(user1Policies.get(ResourceType.Flow.getValue()).contains(RequestAction.READ)); - - assertTrue(user1Policies.containsKey(ResourceType.ProcessGroup.getValue() + "/" + ROOT_GROUP_ID)); - assertEquals(1, user1Policies.get(ResourceType.ProcessGroup.getValue() + "/" + ROOT_GROUP_ID).size()); - assertTrue(user1Policies.get(ResourceType.ProcessGroup.getValue() + "/" + ROOT_GROUP_ID).contains(RequestAction.READ)); - - // verify user2's policies - final Map> user2Policies = getResourceActions(policies, user2); - assertEquals(3, user2Policies.size()); - - assertTrue(user2Policies.containsKey(ResourceType.Provenance.getValue())); - assertEquals(1, user2Policies.get(ResourceType.Provenance.getValue()).size()); - assertTrue(user2Policies.get(ResourceType.Provenance.getValue()).contains(RequestAction.READ)); - - assertTrue(user2Policies.containsKey(ResourceType.ProvenanceData.getValue() + "/process-groups/" + ROOT_GROUP_ID)); - assertEquals(1, user2Policies.get(ResourceType.ProvenanceData.getValue() + "/process-groups/" + ROOT_GROUP_ID).size()); - assertTrue(user2Policies.get(ResourceType.ProvenanceData.getValue() + "/process-groups/" + ROOT_GROUP_ID).contains(RequestAction.READ)); - - assertTrue(user2Policies.containsKey(ResourceType.Data.getValue() + "/process-groups/" + ROOT_GROUP_ID)); - assertEquals(1, user2Policies.get(ResourceType.Data.getValue() + "/process-groups/" + ROOT_GROUP_ID).size()); - assertTrue(user2Policies.get(ResourceType.Data.getValue() + "/process-groups/" + ROOT_GROUP_ID).contains(RequestAction.READ)); - - // verify user3's policies - final Map> user3Policies = getResourceActions(policies, user3); - assertEquals(6, user3Policies.size()); - - assertTrue(user3Policies.containsKey(ResourceType.Flow.getValue())); - assertEquals(1, user3Policies.get(ResourceType.Flow.getValue()).size()); - assertTrue(user3Policies.get(ResourceType.Flow.getValue()).contains(RequestAction.READ)); - - assertTrue(user3Policies.containsKey(ResourceType.ProcessGroup.getValue() + "/" + ROOT_GROUP_ID)); - assertEquals(2, user3Policies.get(ResourceType.ProcessGroup.getValue() + "/" + ROOT_GROUP_ID).size()); - assertTrue(user3Policies.get(ResourceType.ProcessGroup.getValue() + "/" + ROOT_GROUP_ID).contains(RequestAction.WRITE)); - - // verify user4's policies - final Map> user4Policies = getResourceActions(policies, user4); - assertEquals(6, user4Policies.size()); - - assertTrue(user4Policies.containsKey(ResourceType.Flow.getValue())); - assertEquals(1, user4Policies.get(ResourceType.Flow.getValue()).size()); - assertTrue(user4Policies.get(ResourceType.Flow.getValue()).contains(RequestAction.READ)); - - assertTrue(user4Policies.containsKey(ResourceType.ProcessGroup.getValue() + "/" + ROOT_GROUP_ID)); - assertEquals(1, user4Policies.get(ResourceType.ProcessGroup.getValue() + "/" + ROOT_GROUP_ID).size()); - assertTrue(user4Policies.get(ResourceType.ProcessGroup.getValue() + "/" + ROOT_GROUP_ID).contains(RequestAction.READ)); - - assertTrue(user4Policies.containsKey(ResourceType.Tenant.getValue())); - assertEquals(2, user4Policies.get(ResourceType.Tenant.getValue()).size()); - assertTrue(user4Policies.get(ResourceType.Tenant.getValue()).contains(RequestAction.WRITE)); - - assertTrue(user4Policies.containsKey(ResourceType.Policy.getValue())); - assertEquals(2, user4Policies.get(ResourceType.Policy.getValue()).size()); - assertTrue(user4Policies.get(ResourceType.Policy.getValue()).contains(RequestAction.WRITE)); - - // verify user5's policies - final Map> user5Policies = getResourceActions(policies, user5); - assertEquals(2, user5Policies.size()); - - assertTrue(user5Policies.containsKey(ResourceType.Proxy.getValue())); - assertEquals(1, user5Policies.get(ResourceType.Proxy.getValue()).size()); - assertTrue(user5Policies.get(ResourceType.Proxy.getValue()).contains(RequestAction.WRITE)); - - // verify user6's policies - final Map> user6Policies = getResourceActions(policies, user6); - assertEquals(3, user6Policies.size()); - - assertTrue(user6Policies.containsKey(ResourceType.SiteToSite.getValue())); - assertEquals(1, user6Policies.get(ResourceType.SiteToSite.getValue()).size()); - assertTrue(user6Policies.get(ResourceType.SiteToSite.getValue()).contains(RequestAction.READ)); - - final Resource inputPortResource = ResourceFactory.getDataTransferResource( - ResourceFactory.getComponentResource(ResourceType.InputPort, "2f7d1606-b090-4be7-a592-a5b70fb55531", "TCP Input")); - final AccessPolicy inputPortPolicy = authorizer.getUsersAndAccessPolicies().getAccessPolicy(inputPortResource.getIdentifier(), RequestAction.WRITE); - assertNotNull(inputPortPolicy); - assertEquals(1, inputPortPolicy.getUsers().size()); - assertTrue(inputPortPolicy.getUsers().contains(user6.getIdentifier())); - assertEquals(1, inputPortPolicy.getGroups().size()); - assertTrue(inputPortPolicy.getGroups().contains(group1.getIdentifier())); - - final Resource outputPortResource = ResourceFactory.getDataTransferResource( - ResourceFactory.getComponentResource(ResourceType.OutputPort, "2f7d1606-b090-4be7-a592-a5b70fb55532", "TCP Output")); - final AccessPolicy outputPortPolicy = authorizer.getUsersAndAccessPolicies().getAccessPolicy(outputPortResource.getIdentifier(), RequestAction.WRITE); - assertNotNull(outputPortPolicy); - assertEquals(1, outputPortPolicy.getUsers().size()); - assertTrue(outputPortPolicy.getUsers().contains(user4.getIdentifier())); - } - - private Map> getResourceActions(final Set policies, final User user) { - Map> resourceActionMap = new HashMap<>(); - - for (AccessPolicy accessPolicy : policies) { - if (accessPolicy.getUsers().contains(user.getIdentifier())) { - Set actions = resourceActionMap.get(accessPolicy.getResource()); - if (actions == null) { - actions = new HashSet<>(); - resourceActionMap.put(accessPolicy.getResource(), actions); - } - actions.add(accessPolicy.getAction()); - } - } - - return resourceActionMap; - } - - @Test - public void testOnConfiguredWhenLegacyUsersFileProvidedWithIdentityMappings() throws Exception { - final Properties props = new Properties(); - props.setProperty("nifi.security.identity.mapping.pattern.dn1", "^CN=(.*?), OU=(.*?), O=(.*?), L=(.*?), ST=(.*?), C=(.*?)$"); - props.setProperty("nifi.security.identity.mapping.value.dn1", "$1"); - - props.setProperty("nifi.security.group.mapping.pattern.anygroup", "^(.*)$"); - props.setProperty("nifi.security.group.mapping.value.anygroup", "$1"); - props.setProperty("nifi.security.group.mapping.transform.anygroup", "UPPER"); - - properties = getNiFiProperties(props); - when(properties.getRestoreDirectory()).thenReturn(restoreAuthorizations.getParentFile()); - when(properties.getFlowConfigurationFile()).thenReturn(flowWithDns); - when(properties.getFlowConfigurationJsonFile()).thenReturn(flowJson); - authorizer.setNiFiProperties(properties); - - when(configurationContext.getProperty(Mockito.eq(FileAuthorizer.PROP_LEGACY_AUTHORIZED_USERS_FILE))) - .thenReturn(new StandardPropertyValue("src/test/resources/authorized-users-with-dns.xml", null, ParameterLookup.EMPTY)); - - writeFile(primaryAuthorizations, EMPTY_AUTHORIZATIONS_CONCISE); - writeFile(primaryTenants, EMPTY_TENANTS_CONCISE); - authorizer.onConfigured(configurationContext); - - final User user1 = authorizer.getUserByIdentity("user1"); - assertNotNull(user1); - - final User user2 = authorizer.getUserByIdentity("user2"); - assertNotNull(user2); - - final User user3 = authorizer.getUserByIdentity("user3"); - assertNotNull(user3); - - final User user4 = authorizer.getUserByIdentity("user4"); - assertNotNull(user4); - - final User user5 = authorizer.getUserByIdentity("user5"); - assertNotNull(user5); - - final User user6 = authorizer.getUserByIdentity("user6"); - assertNotNull(user6); - - // verify one group got created - final Set groups = authorizer.getGroups(); - assertEquals(1, groups.size()); - final Group group1 = groups.iterator().next(); - assertEquals("GROUP1", group1.getName()); - - final Resource inputPortResource = ResourceFactory.getDataTransferResource( - ResourceFactory.getComponentResource(ResourceType.InputPort, "2f7d1606-b090-4be7-a592-a5b70fb55531", "TCP Input")); - final AccessPolicy inputPortPolicy = authorizer.getUsersAndAccessPolicies().getAccessPolicy(inputPortResource.getIdentifier(), RequestAction.WRITE); - assertNotNull(inputPortPolicy); - assertEquals(1, inputPortPolicy.getUsers().size()); - assertTrue(inputPortPolicy.getUsers().contains(user6.getIdentifier())); - assertEquals(1, inputPortPolicy.getGroups().size()); - assertTrue(inputPortPolicy.getGroups().contains(group1.getIdentifier())); - - final Resource outputPortResource = ResourceFactory.getDataTransferResource( - ResourceFactory.getComponentResource(ResourceType.OutputPort, "2f7d1606-b090-4be7-a592-a5b70fb55532", "TCP Output")); - final AccessPolicy outputPortPolicy = authorizer.getUsersAndAccessPolicies().getAccessPolicy(outputPortResource.getIdentifier(), RequestAction.WRITE); - assertNotNull(outputPortPolicy); - assertEquals(1, outputPortPolicy.getUsers().size()); - assertTrue(outputPortPolicy.getUsers().contains(user4.getIdentifier())); - } - - @Test - public void testOnConfiguredWhenBadLegacyUsersFileProvided() throws Exception { - when(configurationContext.getProperty(Mockito.eq(FileAuthorizer.PROP_LEGACY_AUTHORIZED_USERS_FILE))) - .thenReturn(new StandardPropertyValue("src/test/resources/does-not-exist.xml", null, ParameterLookup.EMPTY)); - - writeFile(primaryAuthorizations, EMPTY_AUTHORIZATIONS_CONCISE); - writeFile(primaryTenants, EMPTY_TENANTS_CONCISE); - - assertThrows(AuthorizerCreationException.class, - () -> authorizer.onConfigured(configurationContext)); - } - - @Test - public void testOnConfiguredWhenInitialAdminAndLegacyUsersProvided() throws Exception { - final String adminIdentity = "admin-user"; - when(configurationContext.getProperty(Mockito.eq(FileAccessPolicyProvider.PROP_INITIAL_ADMIN_IDENTITY))) - .thenReturn(new StandardPropertyValue(adminIdentity, null, ParameterLookup.EMPTY)); - - when(configurationContext.getProperty(Mockito.eq(FileAuthorizer.PROP_LEGACY_AUTHORIZED_USERS_FILE))) - .thenReturn(new StandardPropertyValue("src/test/resources/authorized-users.xml", null, ParameterLookup.EMPTY)); - - writeFile(primaryAuthorizations, EMPTY_AUTHORIZATIONS_CONCISE); - writeFile(primaryTenants, EMPTY_TENANTS_CONCISE); - - assertThrows(AuthorizerCreationException.class, - () -> authorizer.onConfigured(configurationContext)); - } - @Test public void testOnConfiguredWhenInitialAdminNotProvided() throws Exception { writeFile(primaryAuthorizations, EMPTY_AUTHORIZATIONS_CONCISE); @@ -595,8 +297,7 @@ public class FileAuthorizerTest { // setup NiFi properties to return a file that does not exist properties = mock(NiFiProperties.class); when(properties.getRestoreDirectory()).thenReturn(restoreAuthorizations.getParentFile()); - when(properties.getFlowConfigurationFile()).thenReturn(new File("src/test/resources/does-not-exist.xml.gz")); - when(properties.getFlowConfigurationJsonFile()).thenReturn(flowJson); + when(properties.getFlowConfigurationFile()).thenReturn(new File("src/test/resources/does-not-exist.json.gz")); authorizer.setNiFiProperties(properties); final String adminIdentity = "admin-user"; @@ -634,8 +335,7 @@ public class FileAuthorizerTest { // setup NiFi properties to return a file that does not exist properties = mock(NiFiProperties.class); when(properties.getRestoreDirectory()).thenReturn(restoreAuthorizations.getParentFile()); - when(properties.getFlowConfigurationFile()).thenReturn(null); - when(properties.getFlowConfigurationJsonFile()).thenReturn(flowJson); + when(properties.getFlowConfigurationFile()).thenReturn(new File("src/test/resources/does-not-exist.json.gz")); authorizer.setNiFiProperties(properties); final String adminIdentity = "admin-user"; @@ -677,7 +377,6 @@ public class FileAuthorizerTest { properties = getNiFiProperties(props); when(properties.getRestoreDirectory()).thenReturn(restoreAuthorizations.getParentFile()); when(properties.getFlowConfigurationFile()).thenReturn(flow); - when(properties.getFlowConfigurationJsonFile()).thenReturn(flowJson); authorizer.setNiFiProperties(properties); final String adminIdentity = "CN=localhost, OU=Apache NiFi, O=Apache, L=Santa Monica, ST=CA, C=US"; @@ -772,7 +471,6 @@ public class FileAuthorizerTest { properties = getNiFiProperties(props); when(properties.getRestoreDirectory()).thenReturn(restoreAuthorizations.getParentFile()); when(properties.getFlowConfigurationFile()).thenReturn(flow); - when(properties.getFlowConfigurationJsonFile()).thenReturn(flowJson); authorizer.setNiFiProperties(properties); final String adminIdentity = "CN=user1, OU=Apache NiFi, O=Apache, L=Santa Monica, ST=CA, C=US"; diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-file-authorizer/src/test/java/org/apache/nifi/authorization/FileUserGroupProviderTest.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-file-authorizer/src/test/java/org/apache/nifi/authorization/FileUserGroupProviderTest.java index 6bcae42c8e..3dac3ff76d 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-file-authorizer/src/test/java/org/apache/nifi/authorization/FileUserGroupProviderTest.java +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-file-authorizer/src/test/java/org/apache/nifi/authorization/FileUserGroupProviderTest.java @@ -111,7 +111,6 @@ public class FileUserGroupProviderTest { when(properties.getRestoreDirectory()).thenReturn(restoreTenants.getParentFile()); configurationContext = mock(AuthorizerConfigurationContext.class); - when(configurationContext.getProperty(eq(FileAuthorizer.PROP_LEGACY_AUTHORIZED_USERS_FILE))).thenReturn(new StandardPropertyValue(null, null, ParameterLookup.EMPTY)); when(configurationContext.getProperty(eq(FileUserGroupProvider.PROP_TENANTS_FILE))).thenReturn(new StandardPropertyValue(primaryTenants.getPath(), null, ParameterLookup.EMPTY)); when(configurationContext.getProperties()).then((invocation) -> { final Map properties = new HashMap<>(); @@ -121,11 +120,6 @@ public class FileUserGroupProviderTest { properties.put(FileUserGroupProvider.PROP_TENANTS_FILE, tenantFile.getValue()); } - final PropertyValue legacyAuthFile = configurationContext.getProperty(FileAuthorizer.PROP_LEGACY_AUTHORIZED_USERS_FILE); - if (legacyAuthFile != null) { - properties.put(FileAuthorizer.PROP_LEGACY_AUTHORIZED_USERS_FILE, legacyAuthFile.getValue()); - } - int i = 1; while (true) { final String key = FileUserGroupProvider.PROP_INITIAL_USER_IDENTITY_PREFIX + i++; @@ -151,141 +145,6 @@ public class FileUserGroupProviderTest { deleteFile(restoreTenants); } - @Test - public void testOnConfiguredWhenLegacyUsersFileProvided() throws Exception { - when(configurationContext.getProperty(eq(FileAuthorizer.PROP_LEGACY_AUTHORIZED_USERS_FILE))) - .thenReturn(new StandardPropertyValue("src/test/resources/authorized-users.xml", null, ParameterLookup.EMPTY)); - - writeFile(primaryTenants, EMPTY_TENANTS_CONCISE); - userGroupProvider.onConfigured(configurationContext); - - // verify all users got created correctly - final Set users = userGroupProvider.getUsers(); - assertEquals(6, users.size()); - - final User user1 = userGroupProvider.getUserByIdentity("user1"); - assertNotNull(user1); - - final User user2 = userGroupProvider.getUserByIdentity("user2"); - assertNotNull(user2); - - final User user3 = userGroupProvider.getUserByIdentity("user3"); - assertNotNull(user3); - - final User user4 = userGroupProvider.getUserByIdentity("user4"); - assertNotNull(user4); - - final User user5 = userGroupProvider.getUserByIdentity("user5"); - assertNotNull(user5); - - final User user6 = userGroupProvider.getUserByIdentity("user6"); - assertNotNull(user6); - - // verify one group got created - final Set groups = userGroupProvider.getGroups(); - assertEquals(1, groups.size()); - final Group group1 = groups.iterator().next(); - assertEquals("group1", group1.getName()); - } - - @Test - public void testOnConfiguredWhenLegacyUsersFileProvidedWithIdentityMappings() throws Exception { - final Properties props = new Properties(); - props.setProperty("nifi.security.identity.mapping.pattern.dn1", "^CN=(.*?), OU=(.*?), O=(.*?), L=(.*?), ST=(.*?), C=(.*?)$"); - props.setProperty("nifi.security.identity.mapping.value.dn1", "$1"); - - properties = getNiFiProperties(props); - when(properties.getRestoreDirectory()).thenReturn(restoreTenants.getParentFile()); - userGroupProvider.setNiFiProperties(properties); - - when(configurationContext.getProperty(eq(FileAuthorizer.PROP_LEGACY_AUTHORIZED_USERS_FILE))) - .thenReturn(new StandardPropertyValue("src/test/resources/authorized-users-with-dns.xml", null, ParameterLookup.EMPTY)); - - writeFile(primaryTenants, EMPTY_TENANTS_CONCISE); - userGroupProvider.onConfigured(configurationContext); - - final User user1 = userGroupProvider.getUserByIdentity("user1"); - assertNotNull(user1); - - final User user2 = userGroupProvider.getUserByIdentity("user2"); - assertNotNull(user2); - - final User user3 = userGroupProvider.getUserByIdentity("user3"); - assertNotNull(user3); - - final User user4 = userGroupProvider.getUserByIdentity("user4"); - assertNotNull(user4); - - final User user5 = userGroupProvider.getUserByIdentity("user5"); - assertNotNull(user5); - - final User user6 = userGroupProvider.getUserByIdentity("user6"); - assertNotNull(user6); - - // verify one group got created - final Set groups = userGroupProvider.getGroups(); - assertEquals(1, groups.size()); - final Group group1 = groups.iterator().next(); - assertEquals("group1", group1.getName()); - } - - @Test - public void testOnConfiguredWhenLegacyUsersFileProvidedWithIdentityMappingsAndTransforms() throws Exception { - final Properties props = new Properties(); - props.setProperty("nifi.security.identity.mapping.pattern.dn1", "^CN=(.*?), OU=(.*?), O=(.*?), L=(.*?), ST=(.*?), C=(.*?)$"); - props.setProperty("nifi.security.identity.mapping.value.dn1", "$1"); - props.setProperty("nifi.security.identity.mapping.transform.dn1", "UPPER"); - - props.setProperty("nifi.security.group.mapping.pattern.anygroup", "^(.*)$"); - props.setProperty("nifi.security.group.mapping.value.anygroup", "$1"); - props.setProperty("nifi.security.group.mapping.transform.anygroup", "UPPER"); - - properties = getNiFiProperties(props); - when(properties.getRestoreDirectory()).thenReturn(restoreTenants.getParentFile()); - userGroupProvider.setNiFiProperties(properties); - - when(configurationContext.getProperty(eq(FileAuthorizer.PROP_LEGACY_AUTHORIZED_USERS_FILE))) - .thenReturn(new StandardPropertyValue("src/test/resources/authorized-users-with-dns.xml", null, ParameterLookup.EMPTY)); - - writeFile(primaryTenants, EMPTY_TENANTS_CONCISE); - userGroupProvider.onConfigured(configurationContext); - - final User user1 = userGroupProvider.getUserByIdentity("USER1"); - assertNotNull(user1); - - final User user2 = userGroupProvider.getUserByIdentity("USER2"); - assertNotNull(user2); - - final User user3 = userGroupProvider.getUserByIdentity("USER3"); - assertNotNull(user3); - - final User user4 = userGroupProvider.getUserByIdentity("USER4"); - assertNotNull(user4); - - final User user5 = userGroupProvider.getUserByIdentity("USER5"); - assertNotNull(user5); - - final User user6 = userGroupProvider.getUserByIdentity("USER6"); - assertNotNull(user6); - - // verify one group got created - final Set groups = userGroupProvider.getGroups(); - assertEquals(1, groups.size()); - final Group group1 = groups.iterator().next(); - assertEquals("GROUP1", group1.getName()); - } - - @Test - public void testOnConfiguredWhenBadLegacyUsersFileProvided() throws Exception { - when(configurationContext.getProperty(eq(FileAuthorizer.PROP_LEGACY_AUTHORIZED_USERS_FILE))) - .thenReturn(new StandardPropertyValue("src/test/resources/does-not-exist.xml", null, ParameterLookup.EMPTY)); - - writeFile(primaryTenants, EMPTY_TENANTS_CONCISE); - - assertThrows(AuthorizerCreationException.class, - () -> userGroupProvider.onConfigured(configurationContext)); - } - @Test public void testOnConfiguredWhenInitialUsersNotProvided() throws Exception { writeFile(primaryTenants, EMPTY_TENANTS_CONCISE); diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-file-authorizer/src/test/resources/flow-no-ports.json.gz b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-file-authorizer/src/test/resources/flow-no-ports.json.gz new file mode 100644 index 0000000000..02f95c2f28 Binary files /dev/null and b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-file-authorizer/src/test/resources/flow-no-ports.json.gz differ diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-file-authorizer/src/test/resources/flow-no-ports.xml.gz b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-file-authorizer/src/test/resources/flow-no-ports.xml.gz deleted file mode 100644 index 95cca27554..0000000000 Binary files a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-file-authorizer/src/test/resources/flow-no-ports.xml.gz and /dev/null differ diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-file-authorizer/src/test/resources/flow-with-dns.json.gz b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-file-authorizer/src/test/resources/flow-with-dns.json.gz new file mode 100644 index 0000000000..02f95c2f28 Binary files /dev/null and b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-file-authorizer/src/test/resources/flow-with-dns.json.gz differ diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-file-authorizer/src/test/resources/flow-with-dns.xml.gz b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-file-authorizer/src/test/resources/flow-with-dns.xml.gz deleted file mode 100644 index e4b4ed6440..0000000000 Binary files a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-file-authorizer/src/test/resources/flow-with-dns.xml.gz and /dev/null differ diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-file-authorizer/src/test/resources/flow.json.gz b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-file-authorizer/src/test/resources/flow.json.gz new file mode 100644 index 0000000000..02f95c2f28 Binary files /dev/null and b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-file-authorizer/src/test/resources/flow.json.gz differ diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-file-authorizer/src/test/resources/flow.xml.gz b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-file-authorizer/src/test/resources/flow.xml.gz deleted file mode 100644 index 95cca27554..0000000000 Binary files a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-file-authorizer/src/test/resources/flow.xml.gz and /dev/null differ diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-cluster-protocol/src/main/java/org/apache/nifi/cluster/protocol/StandardDataFlow.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-cluster-protocol/src/main/java/org/apache/nifi/cluster/protocol/StandardDataFlow.java index 30cce26b5c..fecaf1857c 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-cluster-protocol/src/main/java/org/apache/nifi/cluster/protocol/StandardDataFlow.java +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-cluster-protocol/src/main/java/org/apache/nifi/cluster/protocol/StandardDataFlow.java @@ -60,7 +60,7 @@ public class StandardDataFlow implements Serializable, DataFlow { * Constructs an instance. * * @param flow a valid flow as bytes, which cannot be null - * @param snippetBytes an XML representation of snippets. May be null. + * @param snippetBytes a JSON representation of snippets. May be null. * @param authorizerFingerprint the bytes of the Authorizer's fingerprint. May be null when using an external Authorizer. * @param missingComponentIds the ids of components that were created as missing ghost components * @@ -161,12 +161,4 @@ public class StandardDataFlow implements Serializable, DataFlow { throw new FlowSerializationException("Could not parse flow as a VersionedDataflow", e); } } - - public boolean isXml() { - if (flow == null || flow.length == 0) { - return true; - } - - return flow[0] == '<'; - } } diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-cluster/src/main/java/org/apache/nifi/cluster/coordination/flow/PopularVoteFlowElection.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-cluster/src/main/java/org/apache/nifi/cluster/coordination/flow/PopularVoteFlowElection.java index 5e88021d09..201b4c367d 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-cluster/src/main/java/org/apache/nifi/cluster/coordination/flow/PopularVoteFlowElection.java +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-cluster/src/main/java/org/apache/nifi/cluster/coordination/flow/PopularVoteFlowElection.java @@ -20,7 +20,7 @@ package org.apache.nifi.cluster.coordination.flow; import org.apache.commons.codec.digest.DigestUtils; import org.apache.nifi.cluster.protocol.DataFlow; import org.apache.nifi.cluster.protocol.NodeIdentifier; -import org.apache.nifi.controller.serialization.StandardFlowSynchronizer; +import org.apache.nifi.controller.serialization.VersionedFlowSynchronizer; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -239,7 +239,7 @@ public class PopularVoteFlowElection implements FlowElection { } public boolean isFlowEmpty() { - return StandardFlowSynchronizer.isFlowEmpty(dataFlow); + return VersionedFlowSynchronizer.isFlowEmpty(dataFlow); } public Set getNodes() { diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-cluster/src/test/java/org/apache/nifi/cluster/coordination/flow/TestPopularVoteFlowElection.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-cluster/src/test/java/org/apache/nifi/cluster/coordination/flow/TestPopularVoteFlowElection.java index 6270a14211..4c4b612587 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-cluster/src/test/java/org/apache/nifi/cluster/coordination/flow/TestPopularVoteFlowElection.java +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-cluster/src/test/java/org/apache/nifi/cluster/coordination/flow/TestPopularVoteFlowElection.java @@ -40,7 +40,7 @@ public class TestPopularVoteFlowElection { @Test public void testOnlyEmptyFlows() throws IOException { final PopularVoteFlowElection election = new PopularVoteFlowElection(1, TimeUnit.MINUTES, 3); - final byte[] flow = Files.readAllBytes(Paths.get("src/test/resources/conf/empty-flow.xml")); + final byte[] flow = Files.readAllBytes(Paths.get("src/test/resources/conf/empty-flow.json")); assertFalse(election.isElectionComplete()); assertNull(election.getElectedDataFlow()); @@ -62,8 +62,8 @@ public class TestPopularVoteFlowElection { @Test public void testDifferentEmptyFlows() throws IOException { final PopularVoteFlowElection election = new PopularVoteFlowElection(1, TimeUnit.MINUTES, 3); - final byte[] flow1 = Files.readAllBytes(Paths.get("src/test/resources/conf/empty-flow.xml")); - final byte[] flow2 = Files.readAllBytes(Paths.get("src/test/resources/conf/different-empty-flow.xml")); + final byte[] flow1 = Files.readAllBytes(Paths.get("src/test/resources/conf/empty-flow.json")); + final byte[] flow2 = Files.readAllBytes(Paths.get("src/test/resources/conf/different-empty-flow.json")); assertFalse(election.isElectionComplete()); assertNull(election.getElectedDataFlow()); @@ -87,8 +87,8 @@ public class TestPopularVoteFlowElection { @Test public void testEmptyFlowIgnoredIfNonEmptyFlowExists() throws IOException { final PopularVoteFlowElection election = new PopularVoteFlowElection(1, TimeUnit.MINUTES, 8); - final byte[] emptyFlow = Files.readAllBytes(Paths.get("src/test/resources/conf/empty-flow.xml")); - final byte[] nonEmptyFlow = Files.readAllBytes(Paths.get("src/test/resources/conf/non-empty-flow.xml")); + final byte[] emptyFlow = Files.readAllBytes(Paths.get("src/test/resources/conf/empty-flow.json")); + final byte[] nonEmptyFlow = Files.readAllBytes(Paths.get("src/test/resources/conf/non-empty-flow.json")); for (int i = 0; i < 8; i++) { assertFalse(election.isElectionComplete()); @@ -114,8 +114,8 @@ public class TestPopularVoteFlowElection { @Test public void testAutoGeneratedVsPopulatedFlowElection() throws IOException { final PopularVoteFlowElection election = new PopularVoteFlowElection(1, TimeUnit.MINUTES, 4); - final byte[] emptyFlow = Files.readAllBytes(Paths.get("src/test/resources/conf/auto-generated-empty-flow.xml")); - final byte[] nonEmptyFlow = Files.readAllBytes(Paths.get("src/test/resources/conf/reporting-task-flow.xml")); + final byte[] emptyFlow = Files.readAllBytes(Paths.get("src/test/resources/conf/auto-generated-empty-flow.json")); + final byte[] nonEmptyFlow = Files.readAllBytes(Paths.get("src/test/resources/conf/reporting-task-flow.json")); for (int i = 0; i < 4; i++) { assertFalse(election.isElectionComplete()); diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-cluster/src/test/resources/conf/auto-generated-empty-flow.json b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-cluster/src/test/resources/conf/auto-generated-empty-flow.json new file mode 100644 index 0000000000..cd3d6db07a --- /dev/null +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-cluster/src/test/resources/conf/auto-generated-empty-flow.json @@ -0,0 +1 @@ +{"encodingVersion":{"majorVersion":2,"minorVersion":0},"maxTimerDrivenThreadCount":10,"registries":[],"parameterContexts":[],"parameterProviders":[],"controllerServices":[],"reportingTasks":[],"templates":[],"rootGroup":{"identifier":"0bc6db14-095b-392d-a063-0df1646ddf72","instanceIdentifier":"ee207cce-015d-1000-30bf-36cd2fd1ea5c","name":"NiFi Flow","comments":"","position":{"x":0.0,"y":0.0},"processGroups":[],"remoteProcessGroups":[],"processors":[],"inputPorts":[],"outputPorts":[],"connections":[],"labels":[],"funnels":[],"controllerServices":[],"variables":{},"defaultFlowFileExpiration":"0 sec","defaultBackPressureObjectThreshold":10000,"defaultBackPressureDataSizeThreshold":"1 GB","scheduledState":"ENABLED","executionEngine":"INHERITED","maxConcurrentTasks":1,"statelessFlowTimeout":"1 min","componentType":"PROCESS_GROUP","flowFileConcurrency":"UNBOUNDED","flowFileOutboundPolicy":"STREAM_WHEN_AVAILABLE"}} \ No newline at end of file diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-cluster/src/test/resources/conf/auto-generated-empty-flow.xml b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-cluster/src/test/resources/conf/auto-generated-empty-flow.xml deleted file mode 100644 index 720fc0b4d9..0000000000 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-cluster/src/test/resources/conf/auto-generated-empty-flow.xml +++ /dev/null @@ -1,27 +0,0 @@ - - - - 10 - 5 - - ee207cce-015d-1000-30bf-36cd2fd1ea5c - NiFi Flow - - - - - - \ No newline at end of file diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-cluster/src/test/resources/conf/controller-service-flow.json b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-cluster/src/test/resources/conf/controller-service-flow.json new file mode 100644 index 0000000000..b6fcf18abc --- /dev/null +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-cluster/src/test/resources/conf/controller-service-flow.json @@ -0,0 +1,64 @@ +{ + "encodingVersion": { + "majorVersion": 2, + "minorVersion": 0 + }, + "maxTimerDrivenThreadCount": 10, + "registries": [], + "parameterContexts": [], + "parameterProviders": [], + "controllerServices": [], + "reportingTasks": [], + "templates": [], + "rootGroup": { + "identifier": "3ddb274d-5d53-3a68-8db3-1e95b702852d", + "instanceIdentifier": "778f676e-6542-4c18-9d06-24b6fd3a1b29", + "name": "NiFi Flow", + "position": { + "x": 0, + "y": 0 + }, + "processGroups": [], + "remoteProcessGroups": [], + "processors": [], + "inputPorts": [], + "outputPorts": [], + "connections": [], + "labels": [], + "funnels": [], + "controllerServices": [ + { + "identifier": "e5c9d095-39ab-311a-80a7-260efcdd0498", + "instanceIdentifier": "edf22ee5-376a-46dc-a38a-919351124457", + "name": "ControllerService", + "comments": "", + "type": "org.apache.nifi.controller.service.mock.ServiceD", + "bundle": { + "group": "default", + "artifact": "unknown", + "version": "unversioned" + }, + "properties": { + "Foo1": "enc{265708cd5e497f7581a37f48445f634879020a73382e2e761cc55587b8a2e55170774348}" + }, + "propertyDescriptors": {}, + "controllerServiceApis": [], + "scheduledState": "DISABLED", + "bulletinLevel": "WARN", + "componentType": "CONTROLLER_SERVICE", + "groupIdentifier": "3ddb274d-5d53-3a68-8db3-1e95b702852d" + } + ], + "variables": {}, + "defaultFlowFileExpiration": "0 sec", + "defaultBackPressureObjectThreshold": 10000, + "defaultBackPressureDataSizeThreshold": "1 GB", + "scheduledState": "ENABLED", + "executionEngine": "INHERITED", + "maxConcurrentTasks": 1, + "statelessFlowTimeout": "1 min", + "componentType": "PROCESS_GROUP", + "flowFileConcurrency": "UNBOUNDED", + "flowFileOutboundPolicy": "STREAM_WHEN_AVAILABLE" + } +} \ No newline at end of file diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-cluster/src/test/resources/conf/controller-service-flow.xml b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-cluster/src/test/resources/conf/controller-service-flow.xml deleted file mode 100644 index 4278311558..0000000000 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-cluster/src/test/resources/conf/controller-service-flow.xml +++ /dev/null @@ -1,37 +0,0 @@ - - - - 10 - 5 - - 778f676e-6542-4c18-9d06-24b6fd3a1b29 - NiFi Flow - - - - - edf22ee5-376a-46dc-a38a-919351124457 - ControllerService - - org.apache.nifi.controller.service.mock.ServiceD - false - - Foo1 - Bar1 - - - - diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-cluster/src/test/resources/conf/different-empty-flow.json b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-cluster/src/test/resources/conf/different-empty-flow.json new file mode 100644 index 0000000000..32c5a55521 --- /dev/null +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-cluster/src/test/resources/conf/different-empty-flow.json @@ -0,0 +1 @@ +{"encodingVersion":{"majorVersion":2,"minorVersion":0},"maxTimerDrivenThreadCount":10,"registries":[],"parameterContexts":[],"parameterProviders":[],"controllerServices":[],"reportingTasks":[],"templates":[],"rootGroup":{"identifier":"38c6cbd2-8bf1-3507-8d07-0980dd1fb595","instanceIdentifier":"11111111-1111-1111-1111-111111111111","name":"Empty NiFi Flow","comments":"","position":{"x":0.0,"y":0.0},"processGroups":[],"remoteProcessGroups":[],"processors":[],"inputPorts":[],"outputPorts":[],"connections":[],"labels":[],"funnels":[],"controllerServices":[],"variables":{},"defaultFlowFileExpiration":"0 sec","defaultBackPressureObjectThreshold":10000,"defaultBackPressureDataSizeThreshold":"1 GB","scheduledState":"ENABLED","executionEngine":"INHERITED","maxConcurrentTasks":1,"statelessFlowTimeout":"1 min","componentType":"PROCESS_GROUP","flowFileConcurrency":"UNBOUNDED","flowFileOutboundPolicy":"STREAM_WHEN_AVAILABLE"}} \ No newline at end of file diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-cluster/src/test/resources/conf/different-empty-flow.xml b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-cluster/src/test/resources/conf/different-empty-flow.xml deleted file mode 100644 index 8c9641ae25..0000000000 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-cluster/src/test/resources/conf/different-empty-flow.xml +++ /dev/null @@ -1,27 +0,0 @@ - - - - 10 - 5 - - 11111111-1111-1111-1111-111111111111 - Empty NiFi Flow - - - - - - \ No newline at end of file diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-cluster/src/test/resources/conf/empty-flow.json b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-cluster/src/test/resources/conf/empty-flow.json new file mode 100644 index 0000000000..76c2f69b01 --- /dev/null +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-cluster/src/test/resources/conf/empty-flow.json @@ -0,0 +1 @@ +{"encodingVersion":{"majorVersion":2,"minorVersion":0},"maxTimerDrivenThreadCount":10,"registries":[],"parameterContexts":[],"parameterProviders":[],"controllerServices":[],"reportingTasks":[],"templates":[],"rootGroup":{"identifier":"9f89c84a-559f-3736-b6a4-7ff8daed0d33","instanceIdentifier":"00000000-0000-0000-0000-000000000000","name":"Empty NiFi Flow","comments":"","position":{"x":0.0,"y":0.0},"processGroups":[],"remoteProcessGroups":[],"processors":[],"inputPorts":[],"outputPorts":[],"connections":[],"labels":[],"funnels":[],"controllerServices":[],"variables":{},"defaultFlowFileExpiration":"0 sec","defaultBackPressureObjectThreshold":10000,"defaultBackPressureDataSizeThreshold":"1 GB","scheduledState":"ENABLED","executionEngine":"INHERITED","maxConcurrentTasks":1,"statelessFlowTimeout":"1 min","componentType":"PROCESS_GROUP","flowFileConcurrency":"UNBOUNDED","flowFileOutboundPolicy":"STREAM_WHEN_AVAILABLE"}} \ No newline at end of file diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-cluster/src/test/resources/conf/empty-flow.xml b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-cluster/src/test/resources/conf/empty-flow.xml deleted file mode 100644 index c0cb6de05d..0000000000 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-cluster/src/test/resources/conf/empty-flow.xml +++ /dev/null @@ -1,27 +0,0 @@ - - - - 10 - 5 - - 00000000-0000-0000-0000-000000000000 - Empty NiFi Flow - - - - - - \ No newline at end of file diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-cluster/src/test/resources/conf/nifi.properties b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-cluster/src/test/resources/conf/nifi.properties index d3170ef316..dc5764fee0 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-cluster/src/test/resources/conf/nifi.properties +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-cluster/src/test/resources/conf/nifi.properties @@ -14,7 +14,7 @@ # limitations under the License. # Core Properties # -nifi.flow.configuration.file=./target/flow.xml.gz +nifi.flow.configuration.file=./target/flow.json.gz nifi.flow.configuration.archive.dir=./target/archive/ nifi.flowcontroller.autoResumeState=true nifi.flowcontroller.graceful.shutdown.period=10 sec diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-cluster/src/test/resources/conf/non-empty-flow.json b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-cluster/src/test/resources/conf/non-empty-flow.json new file mode 100644 index 0000000000..ac9f0e063f --- /dev/null +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-cluster/src/test/resources/conf/non-empty-flow.json @@ -0,0 +1 @@ +{"encodingVersion":{"majorVersion":2,"minorVersion":0},"maxTimerDrivenThreadCount":10,"registries":[],"parameterContexts":[],"parameterProviders":[],"controllerServices":[],"reportingTasks":[],"templates":[],"rootGroup":{"identifier":"9f89c84a-559f-3736-b6a4-7ff8daed0d33","instanceIdentifier":"00000000-0000-0000-0000-000000000000","name":"Integration Test Flow","comments":"","position":{"x":0.0,"y":0.0},"processGroups":[],"remoteProcessGroups":[],"processors":[],"inputPorts":[],"outputPorts":[],"connections":[],"labels":[],"funnels":[],"controllerServices":[],"variables":{},"defaultFlowFileExpiration":"0 sec","defaultBackPressureObjectThreshold":10000,"defaultBackPressureDataSizeThreshold":"1 GB","scheduledState":"ENABLED","executionEngine":"INHERITED","maxConcurrentTasks":1,"statelessFlowTimeout":"1 min","componentType":"PROCESS_GROUP","flowFileConcurrency":"UNBOUNDED","flowFileOutboundPolicy":"STREAM_WHEN_AVAILABLE"}} \ No newline at end of file diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-cluster/src/test/resources/conf/non-empty-flow.xml b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-cluster/src/test/resources/conf/non-empty-flow.xml deleted file mode 100644 index ed5451d625..0000000000 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-cluster/src/test/resources/conf/non-empty-flow.xml +++ /dev/null @@ -1,26 +0,0 @@ - - - - 10 - 5 - - 00000000-0000-0000-0000-000000000000 - Integration Test Flow - - - 1234 - - \ No newline at end of file diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-cluster/src/test/resources/conf/reporting-task-flow.json b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-cluster/src/test/resources/conf/reporting-task-flow.json new file mode 100644 index 0000000000..e25ce454f6 --- /dev/null +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-cluster/src/test/resources/conf/reporting-task-flow.json @@ -0,0 +1 @@ +{"encodingVersion":{"majorVersion":2,"minorVersion":0},"maxTimerDrivenThreadCount":10,"registries":[],"parameterContexts":[],"parameterProviders":[],"controllerServices":[],"reportingTasks":[{"identifier":"3b80ba0f-a6c0-48db-b721-4dbc04cef28e","instanceIdentifier":"3b80ba0f-a6c0-48db-b721-4dbc04cef28e","name":"AmbariReportingTask","comments":"","type":"org.apache.nifi.reporting.ambari.AmbariReportingTask","bundle":{"group":"org.apache.nifi","artifact":"nifi-standard-nar","version":"1.1.0"},"properties":{"Application ID":"enc{398dad79bd3a82ae9c8c3dfb8627b023d970ee4740813fc452b5a930f2475eeb7b82c1c69755b7ad8d67f63a02149cce318a5edb95ae4203}","Hostname":"enc{e9364c72126486f3d1aa1e454a8382ab3fd561456003e614f054fb8d143193428e9e4cc1ddf1b347c8a20ee884c953f110}","Metrics Collector URL":"enc{1ee753dc18ce639e890c3a8649817929b275f3b803ab751776a499fdc770e65521ec14785f818fa5118ec8d4bee1c572f8af1771803e246b17012dc8a44b99}"},"propertyDescriptors":{},"scheduledState":"RUNNING","schedulingPeriod":"{{nifi_ambari_reporting_frequency}}","schedulingStrategy":"TIMER_DRIVEN","componentType":"REPORTING_TASK"}],"templates":[],"rootGroup":{"identifier":"46f95fe5-3024-31d8-aee8-2537d68316f4","instanceIdentifier":"7c84501d-d10c-407c-b9f3-1d80e38fe36a","name":"NiFi Flow","comments":"","position":{"x":0.0,"y":0.0},"processGroups":[],"remoteProcessGroups":[],"processors":[],"inputPorts":[],"outputPorts":[],"connections":[],"labels":[],"funnels":[],"controllerServices":[],"variables":{},"defaultFlowFileExpiration":"0 sec","defaultBackPressureObjectThreshold":10000,"defaultBackPressureDataSizeThreshold":"1 GB","scheduledState":"ENABLED","executionEngine":"INHERITED","maxConcurrentTasks":1,"statelessFlowTimeout":"1 min","componentType":"PROCESS_GROUP","flowFileConcurrency":"UNBOUNDED","flowFileOutboundPolicy":"STREAM_WHEN_AVAILABLE"}} \ No newline at end of file diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-cluster/src/test/resources/conf/reporting-task-flow.xml b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-cluster/src/test/resources/conf/reporting-task-flow.xml deleted file mode 100644 index 751517c33d..0000000000 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-cluster/src/test/resources/conf/reporting-task-flow.xml +++ /dev/null @@ -1,54 +0,0 @@ - - - - 10 - 5 - - 7c84501d-d10c-407c-b9f3-1d80e38fe36a - NiFi Flow - - - - - - - 3b80ba0f-a6c0-48db-b721-4dbc04cef28e - AmbariReportingTask - - org.apache.nifi.reporting.ambari.AmbariReportingTask - - org.apache.nifi - nifi-standard-nar - 1.1.0 - - {{nifi_ambari_reporting_frequency}} - RUNNING - TIMER_DRIVEN - - Metrics Collector URL - ${ambari.metrics.collector.url} - - - Application ID - ${ambari.application.id} - - - Hostname - ${hostname(true)} - - - - \ No newline at end of file diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/main/java/org/apache/nifi/flow/synchronization/StandardVersionedComponentSynchronizer.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/main/java/org/apache/nifi/flow/synchronization/StandardVersionedComponentSynchronizer.java index 63385c879f..eb6f97037e 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/main/java/org/apache/nifi/flow/synchronization/StandardVersionedComponentSynchronizer.java +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/main/java/org/apache/nifi/flow/synchronization/StandardVersionedComponentSynchronizer.java @@ -1472,7 +1472,7 @@ public class StandardVersionedComponentSynchronizer implements VersionedComponen return decryptor.decrypt(value.substring(ENC_PREFIX.length(), value.length() - ENC_SUFFIX.length())); } catch (EncryptionException e) { final String moreDescriptiveMessage = "There was a problem decrypting a sensitive flow configuration value. " + - "Check that the nifi.sensitive.props.key value in nifi.properties matches the value used to encrypt the flow.xml.gz file"; + "Check that the nifi.sensitive.props.key value in nifi.properties matches the value used to encrypt the flow.json.gz file"; throw new EncryptionException(moreDescriptiveMessage, e); } } else { diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/test/resources/conf/nifi.properties b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/test/resources/conf/nifi.properties index c2bf7dfcad..c3971d803b 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/test/resources/conf/nifi.properties +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/test/resources/conf/nifi.properties @@ -14,7 +14,7 @@ # limitations under the License. # Core Properties # -nifi.flow.configuration.file=./target/flow.xml.gz +nifi.flow.configuration.file=./target/flow.json.gz nifi.flow.configuration.archive.dir=./target/archive/ nifi.flowcontroller.autoResumeState=true nifi.flowcontroller.graceful.shutdown.period=10 sec diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core-api/src/main/java/org/apache/nifi/cluster/protocol/DataFlow.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core-api/src/main/java/org/apache/nifi/cluster/protocol/DataFlow.java index 854cc7999d..238d48cba9 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core-api/src/main/java/org/apache/nifi/cluster/protocol/DataFlow.java +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core-api/src/main/java/org/apache/nifi/cluster/protocol/DataFlow.java @@ -50,9 +50,4 @@ public interface DataFlow { * @return the component ids of components that were created as a missing ghost component */ Set getMissingComponents(); - - /** - * @return true if the contents are empty or are made up of XML, false if the contents are JSON - */ - boolean isXml(); } diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core-api/src/main/java/org/apache/nifi/remote/PublicPort.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core-api/src/main/java/org/apache/nifi/remote/PublicPort.java index fdd5653313..83012a05b0 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core-api/src/main/java/org/apache/nifi/remote/PublicPort.java +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core-api/src/main/java/org/apache/nifi/remote/PublicPort.java @@ -23,8 +23,6 @@ import org.apache.nifi.remote.exception.NotAuthorizedException; import org.apache.nifi.remote.exception.RequestExpiredException; import org.apache.nifi.remote.protocol.ServerProtocol; -import java.util.Set; - /** * Represents an input or output port that can receive or transfer data via Site-to-Site protocol. */ @@ -32,14 +30,6 @@ public interface PublicPort extends Port { boolean isTransmitting(); - void setGroupAccessControl(Set groups); - - Set getGroupAccessControl(); - - void setUserAccessControl(Set users); - - Set getUserAccessControl(); - /** * Verifies that the specified user is authorized to interact with this port * and returns a {@link PortAuthorizationResult} indicating why the user is diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core-api/src/main/java/org/apache/nifi/services/FlowService.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core-api/src/main/java/org/apache/nifi/services/FlowService.java index 8d69550d0f..02cd20e86e 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core-api/src/main/java/org/apache/nifi/services/FlowService.java +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core-api/src/main/java/org/apache/nifi/services/FlowService.java @@ -86,7 +86,7 @@ public interface FlowService extends LifeCycle { void copyCurrentFlow(OutputStream os) throws IOException; /** - * Copies the contents of the current flow.xml.gz to the given file, overwriting the file if it exists + * Copies the contents of the current flow.json.gz to the given file, overwriting the file if it exists * @param file the file to write the current flow to * @throws IOException if unable to read the current flow or unable to write to the given file */ diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/FlowSerializationStrategy.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/FlowSerializationStrategy.java deleted file mode 100644 index d5e0f162e0..0000000000 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/FlowSerializationStrategy.java +++ /dev/null @@ -1,31 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.nifi.controller; - -public enum FlowSerializationStrategy { - WRITE_XML_ONLY, - WRITE_JSON_ONLY, - WRITE_XML_AND_JSON; - - public boolean writesJson() { - return this == WRITE_JSON_ONLY || this == WRITE_XML_AND_JSON; - } - - public boolean writesXml() { - return this == WRITE_XML_ONLY || this == WRITE_XML_AND_JSON; - } -} diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/StandardFlowService.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/StandardFlowService.java index 26f835a08c..b1c9f06035 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/StandardFlowService.java +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/StandardFlowService.java @@ -52,7 +52,7 @@ import org.apache.nifi.connectable.Connection; import org.apache.nifi.controller.flow.FlowManager; import org.apache.nifi.controller.serialization.FlowSerializationException; import org.apache.nifi.controller.serialization.FlowSynchronizationException; -import org.apache.nifi.controller.serialization.StandardFlowSynchronizer; +import org.apache.nifi.controller.serialization.VersionedFlowSynchronizer; import org.apache.nifi.controller.status.ProcessGroupStatus; import org.apache.nifi.engine.FlowEngine; import org.apache.nifi.events.BulletinFactory; @@ -111,7 +111,7 @@ public class StandardFlowService implements FlowService, ProtocolHandler { private final boolean autoResumeState; private final Authorizer authorizer; - // Lock is used to protect the flow.xml file. + // Lock is used to protect the flow.json.gz file. private final ReentrantReadWriteLock rwLock = new ReentrantReadWriteLock(); private final Lock readLock = rwLock.readLock(); private final Lock writeLock = rwLock.writeLock(); @@ -150,11 +150,9 @@ public class StandardFlowService implements FlowService, ProtocolHandler { final FlowController controller, final NiFiProperties nifiProperties, final RevisionManager revisionManager, - final Authorizer authorizer, - final FlowSerializationStrategy serializationStrategy) throws IOException { + final Authorizer authorizer) throws IOException { - return new StandardFlowService(controller, nifiProperties, null, false, null, revisionManager, authorizer, - serializationStrategy); + return new StandardFlowService(controller, nifiProperties, null, false, null, revisionManager, authorizer); } public static StandardFlowService createClusteredInstance( @@ -165,8 +163,7 @@ public class StandardFlowService implements FlowService, ProtocolHandler { final RevisionManager revisionManager, final Authorizer authorizer) throws IOException { - return new StandardFlowService(controller, nifiProperties, senderListener, true, coordinator, revisionManager, authorizer, - FlowSerializationStrategy.WRITE_XML_AND_JSON); + return new StandardFlowService(controller, nifiProperties, senderListener, true, coordinator, revisionManager, authorizer); } private StandardFlowService( @@ -176,17 +173,15 @@ public class StandardFlowService implements FlowService, ProtocolHandler { final boolean configuredForClustering, final ClusterCoordinator clusterCoordinator, final RevisionManager revisionManager, - final Authorizer authorizer, - final FlowSerializationStrategy serializationStrategy) throws IOException { + final Authorizer authorizer) throws IOException { this.nifiProperties = nifiProperties; this.controller = controller; - gracefulShutdownSeconds = (int) FormatUtils.getTimeDuration(nifiProperties.getProperty(NiFiProperties.FLOW_CONTROLLER_GRACEFUL_SHUTDOWN_PERIOD), TimeUnit.SECONDS); autoResumeState = nifiProperties.getAutoResumeState(); - dao = new StandardFlowConfigurationDAO(nifiProperties, controller.getExtensionManager(), serializationStrategy); + dao = new StandardFlowConfigurationDAO(nifiProperties, controller.getExtensionManager()); this.clusterCoordinator = clusterCoordinator; if (clusterCoordinator != null) { clusterCoordinator.setFlowService(this); @@ -465,7 +460,7 @@ public class StandardFlowService implements FlowService, ProtocolHandler { * the response will be null and we should load the local dataflow * and heartbeat until a manager is located. */ - final boolean localFlowEmpty = StandardFlowSynchronizer.isFlowEmpty(proposedFlow); + final boolean localFlowEmpty = VersionedFlowSynchronizer.isFlowEmpty(proposedFlow); final ConnectionResponse response = connect(true, localFlowEmpty, proposedFlow); // obtain write lock while we are updating the controller. We need to ensure that we don't diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/StandardFlowSnippet.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/StandardFlowSnippet.java index a1b6b5c74c..244a1e1ba5 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/StandardFlowSnippet.java +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/StandardFlowSnippet.java @@ -46,7 +46,6 @@ import org.apache.nifi.processor.Relationship; import org.apache.nifi.processor.StandardProcessContext; import org.apache.nifi.registry.flow.StandardVersionControlInformation; import org.apache.nifi.registry.flow.VersionControlInformation; -import org.apache.nifi.remote.PublicPort; import org.apache.nifi.remote.StandardRemoteProcessGroupPortDescriptor; import org.apache.nifi.remote.protocol.SiteToSiteTransportProtocol; import org.apache.nifi.scheduling.ExecutionNode; @@ -309,12 +308,6 @@ public class StandardFlowSnippet implements FlowSnippet { if (group.isRootGroup() || Boolean.TRUE.equals(portDTO.getAllowRemoteAccess())) { final String portName = generatePublicInputPortName(flowManager, portDTO.getName()); inputPort = flowManager.createPublicInputPort(portDTO.getId(), portName); - if (portDTO.getGroupAccessControl() != null) { - ((PublicPort) inputPort).setGroupAccessControl(portDTO.getGroupAccessControl()); - } - if (portDTO.getUserAccessControl() != null) { - ((PublicPort) inputPort).setUserAccessControl(portDTO.getUserAccessControl()); - } } else { inputPort = flowManager.createLocalInputPort(portDTO.getId(), portDTO.getName()); } @@ -337,12 +330,6 @@ public class StandardFlowSnippet implements FlowSnippet { if (group.isRootGroup() || Boolean.TRUE.equals(portDTO.getAllowRemoteAccess())) { final String portName = generatePublicOutputPortName(flowManager, portDTO.getName()); outputPort = flowManager.createPublicOutputPort(portDTO.getId(), portName); - if (portDTO.getGroupAccessControl() != null) { - ((PublicPort) outputPort).setGroupAccessControl(portDTO.getGroupAccessControl()); - } - if (portDTO.getUserAccessControl() != null) { - ((PublicPort) outputPort).setUserAccessControl(portDTO.getUserAccessControl()); - } } else { outputPort = flowManager.createLocalOutputPort(portDTO.getId(), portDTO.getName()); } diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/XmlFlowSynchronizer.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/XmlFlowSynchronizer.java deleted file mode 100644 index c29b9b5695..0000000000 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/XmlFlowSynchronizer.java +++ /dev/null @@ -1,1981 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.nifi.controller; - -import org.apache.commons.collections4.CollectionUtils; -import org.apache.commons.lang3.StringUtils; -import org.apache.nifi.authorization.Authorizer; -import org.apache.nifi.authorization.AuthorizerCapabilityDetection; -import org.apache.nifi.authorization.ManagedAuthorizer; -import org.apache.nifi.bundle.BundleCoordinate; -import org.apache.nifi.cluster.protocol.DataFlow; -import org.apache.nifi.cluster.protocol.StandardDataFlow; -import org.apache.nifi.components.PropertyDescriptor; -import org.apache.nifi.connectable.Connectable; -import org.apache.nifi.connectable.ConnectableType; -import org.apache.nifi.connectable.Connection; -import org.apache.nifi.connectable.Funnel; -import org.apache.nifi.connectable.Port; -import org.apache.nifi.connectable.Position; -import org.apache.nifi.connectable.Size; -import org.apache.nifi.controller.flow.FlowManager; -import org.apache.nifi.controller.flowanalysis.FlowAnalysisRuleInstantiationException; -import org.apache.nifi.controller.inheritance.AuthorizerCheck; -import org.apache.nifi.controller.inheritance.BundleCompatibilityCheck; -import org.apache.nifi.controller.inheritance.ConnectionMissingCheck; -import org.apache.nifi.controller.inheritance.FlowFingerprintCheck; -import org.apache.nifi.controller.inheritance.FlowInheritability; -import org.apache.nifi.controller.inheritance.FlowInheritabilityCheck; -import org.apache.nifi.controller.inheritance.MissingComponentsCheck; -import org.apache.nifi.controller.label.Label; -import org.apache.nifi.controller.queue.LoadBalanceCompression; -import org.apache.nifi.controller.queue.LoadBalanceStrategy; -import org.apache.nifi.controller.reporting.ReportingTaskInstantiationException; -import org.apache.nifi.controller.serialization.FlowEncodingVersion; -import org.apache.nifi.controller.serialization.FlowFromDOMFactory; -import org.apache.nifi.controller.serialization.FlowSerializationException; -import org.apache.nifi.controller.serialization.FlowSynchronizationException; -import org.apache.nifi.controller.serialization.FlowSynchronizer; -import org.apache.nifi.controller.serialization.StandardFlowSerializer; -import org.apache.nifi.controller.service.ControllerServiceLoader; -import org.apache.nifi.controller.service.ControllerServiceNode; -import org.apache.nifi.controller.service.ControllerServiceProvider; -import org.apache.nifi.controller.service.ControllerServiceState; -import org.apache.nifi.encrypt.PropertyEncryptor; -import org.apache.nifi.events.BulletinFactory; -import org.apache.nifi.flowanalysis.FlowAnalysisRuleState; -import org.apache.nifi.flowanalysis.EnforcementPolicy; -import org.apache.nifi.flowfile.FlowFilePrioritizer; -import org.apache.nifi.groups.BundleUpdateStrategy; -import org.apache.nifi.groups.FlowFileConcurrency; -import org.apache.nifi.groups.FlowFileOutboundPolicy; -import org.apache.nifi.groups.ProcessGroup; -import org.apache.nifi.groups.RemoteProcessGroup; -import org.apache.nifi.groups.RemoteProcessGroupPortDescriptor; -import org.apache.nifi.logging.LogLevel; -import org.apache.nifi.nar.ExtensionManager; -import org.apache.nifi.parameter.Parameter; -import org.apache.nifi.parameter.ParameterContext; -import org.apache.nifi.parameter.ParameterContextManager; -import org.apache.nifi.parameter.ParameterDescriptor; -import org.apache.nifi.parameter.ParameterProviderConfiguration; -import org.apache.nifi.parameter.StandardParameterProviderConfiguration; -import org.apache.nifi.processor.Relationship; -import org.apache.nifi.registry.flow.FlowRegistryClientNode; -import org.apache.nifi.registry.flow.StandardVersionControlInformation; -import org.apache.nifi.registry.flow.VersionedFlowState; -import org.apache.nifi.remote.PublicPort; -import org.apache.nifi.remote.RemoteGroupPort; -import org.apache.nifi.remote.protocol.SiteToSiteTransportProtocol; -import org.apache.nifi.reporting.Severity; -import org.apache.nifi.scheduling.ExecutionNode; -import org.apache.nifi.scheduling.SchedulingStrategy; -import org.apache.nifi.services.FlowService; -import org.apache.nifi.util.BundleUtils; -import org.apache.nifi.util.DomUtils; -import org.apache.nifi.util.NiFiProperties; -import org.apache.nifi.util.file.FileUtils; -import org.apache.nifi.web.api.dto.BundleDTO; -import org.apache.nifi.web.api.dto.ComponentReferenceDTO; -import org.apache.nifi.web.api.dto.ConnectableDTO; -import org.apache.nifi.web.api.dto.ConnectionDTO; -import org.apache.nifi.web.api.dto.ControllerServiceDTO; -import org.apache.nifi.web.api.dto.FlowAnalysisRuleDTO; -import org.apache.nifi.web.api.dto.FlowRegistryClientDTO; -import org.apache.nifi.web.api.dto.FlowSnippetDTO; -import org.apache.nifi.web.api.dto.FunnelDTO; -import org.apache.nifi.web.api.dto.LabelDTO; -import org.apache.nifi.web.api.dto.ParameterContextDTO; -import org.apache.nifi.web.api.dto.ParameterDTO; -import org.apache.nifi.web.api.dto.ParameterProviderConfigurationDTO; -import org.apache.nifi.web.api.dto.ParameterProviderDTO; -import org.apache.nifi.web.api.dto.PortDTO; -import org.apache.nifi.web.api.dto.PositionDTO; -import org.apache.nifi.web.api.dto.ProcessGroupDTO; -import org.apache.nifi.web.api.dto.ProcessorConfigDTO; -import org.apache.nifi.web.api.dto.ProcessorDTO; -import org.apache.nifi.web.api.dto.RemoteProcessGroupDTO; -import org.apache.nifi.web.api.dto.ReportingTaskDTO; -import org.apache.nifi.web.api.dto.VersionControlInformationDTO; -import org.apache.nifi.web.api.entity.ComponentReferenceEntity; -import org.apache.nifi.web.api.entity.ParameterContextReferenceEntity; -import org.apache.nifi.web.api.entity.ParameterEntity; -import org.apache.nifi.web.api.entity.ParameterProviderConfigurationEntity; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.w3c.dom.Document; -import org.w3c.dom.Element; -import org.w3c.dom.Node; -import org.w3c.dom.NodeList; - -import java.io.ByteArrayOutputStream; -import java.io.File; -import java.io.IOException; -import java.io.InputStream; -import java.nio.charset.StandardCharsets; -import java.nio.file.Files; -import java.nio.file.Path; -import java.nio.file.StandardOpenOption; -import java.text.DateFormat; -import java.text.SimpleDateFormat; -import java.util.ArrayList; -import java.util.Collections; -import java.util.Date; -import java.util.HashMap; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.Objects; -import java.util.Set; -import java.util.concurrent.TimeUnit; -import java.util.function.Function; -import java.util.stream.Collectors; -import java.util.stream.Stream; -import java.util.zip.GZIPInputStream; - -/** - * XML implementation of Flow Synchronizer for reading configuration using XML Document Object Model - */ -public class XmlFlowSynchronizer implements FlowSynchronizer { - - private static final Logger logger = LoggerFactory.getLogger(XmlFlowSynchronizer.class); - private final boolean autoResumeState; - private final NiFiProperties nifiProperties; - private final ExtensionManager extensionManager; - - public XmlFlowSynchronizer(final NiFiProperties nifiProperties, final ExtensionManager extensionManager) { - this.autoResumeState = nifiProperties.getAutoResumeState(); - this.nifiProperties = nifiProperties; - this.extensionManager = extensionManager; - } - - @Override - public void sync(final FlowController controller, final DataFlow proposedFlow, final FlowService flowService, final BundleUpdateStrategy bundleUpdateStrategy) - throws FlowSerializationException, UninheritableFlowException, FlowSynchronizationException { - - final FlowManager flowManager = controller.getFlowManager(); - final ProcessGroup root = flowManager.getRootGroup(); - - // handle corner cases involving no proposed flow - if (proposedFlow == null) { - if (root.isEmpty()) { - return; // no sync to perform - } else { - throw new UninheritableFlowException("Proposed configuration is empty, but the controller contains a data flow."); - } - } - - // determine if the controller already had flow sync'd to it - final boolean flowAlreadySynchronized = controller.isFlowSynchronized(); - logger.debug("Synching FlowController with proposed flow: Controller is Already Synchronized = {}", flowAlreadySynchronized); - - // serialize controller state to bytes - final DataFlow existingDataFlow = getExistingDataFlow(controller); - boolean existingFlowEmpty = isFlowEmpty(existingDataFlow.getFlowDocument()); - - logger.trace("Parsing proposed flow bytes as DOM document"); - final Document configuration = proposedFlow.getFlowDocument(); - - // check that the proposed flow is inheritable by the controller - boolean backupAndPurge = false; - if (existingFlowEmpty) { - logger.debug("Checking bundle compatibility"); - - final BundleCompatibilityCheck bundleCompatibilityCheck = new BundleCompatibilityCheck(); - final FlowInheritability bundleInheritability = bundleCompatibilityCheck.checkInheritability(existingDataFlow, proposedFlow, controller); - if (!bundleInheritability.isInheritable()) { - throw new UninheritableFlowException("Proposed flow could not be inherited because it references one or more Bundles that are not available in this NiFi instance: " - + bundleInheritability.getExplanation()); - } - - logger.debug("Bundle Compatibility check passed"); - } else { - logger.debug("Checking flow inheritability"); - final FlowInheritabilityCheck fingerprintCheck = new FlowFingerprintCheck(); - final FlowInheritability inheritability = fingerprintCheck.checkInheritability(existingDataFlow, proposedFlow, controller); - - if (inheritability.isInheritable()) { - logger.debug("Proposed flow is inheritable"); - } else { - if (controller.isInitialized()) { - // Flow has already been initialized so cannot inherit the cluster flow as liberally. - // Since the cluster's flow is not immediately inheritable, we must throw an UninheritableFlowException. - throw new UninheritableFlowException("Proposed configuration is not inheritable by the flow controller because of flow differences: " + inheritability.getExplanation()); - } - - logger.debug("Proposed flow is not directly inheritable. However, the Controller has not been synchronized yet, " + - "so will check if the existing flow can be backed up and replaced by the proposed flow."); - - final FlowInheritabilityCheck connectionMissingCheck = new ConnectionMissingCheck(null); - final FlowInheritability connectionMissingInheritability = connectionMissingCheck.checkInheritability(existingDataFlow, proposedFlow, controller); - if (connectionMissingInheritability.isInheritable()) { - backupAndPurge = true; - existingFlowEmpty = true; // Consider the existing flow as being empty - logger.debug("Proposed flow contains all connections that currently have data queued. Will backup existing flow and replace, provided all other checks pass"); - } else { - throw new UninheritableFlowException("Proposed flow is not inheritable by the flow controller and cannot completely replace the current flow due to: " - + connectionMissingInheritability.getExplanation()); - } - } - } - - logger.debug("Checking missing component inheritability"); - final FlowInheritabilityCheck missingComponentsCheck = new MissingComponentsCheck(); - final FlowInheritability componentInheritability = missingComponentsCheck.checkInheritability(existingDataFlow, proposedFlow, controller); - if (!componentInheritability.isInheritable()) { - throw new UninheritableFlowException("Proposed Flow is not inheritable by the flow controller because of differences in missing components: " + componentInheritability.getExplanation()); - } - logger.debug("Missing Component Inheritability check passed"); - - logger.debug("Checking authorizer inheritability"); - final FlowInheritabilityCheck authorizerCheck = new AuthorizerCheck(); - final FlowInheritability authorizerInheritability = authorizerCheck.checkInheritability(existingDataFlow, proposedFlow, controller); - final Authorizer authorizer = controller.getAuthorizer(); - - if (existingFlowEmpty) { - logger.debug("Existing flow is empty so will not check Authorizer inheritability. Authorizers will be forcibly inherited if necessary."); - } else { - if (!controller.isInitialized() && authorizer instanceof ManagedAuthorizer) { - logger.debug("Authorizations are not inheritable, but Authorizer is a Managed Authorizer and the Controller has not yet been initialized, so it can be forcibly inherited."); - } else { - if (!authorizerInheritability.isInheritable() && authorizerInheritability.getExplanation() != null) { - throw new UninheritableFlowException("Proposed Authorizer is not inheritable by the Flow Controller because NiFi has already started the dataflow " + - "and Authorizer has differences: " + authorizerInheritability.getExplanation()); - } - - logger.debug("Authorizer inheritability check passed"); - } - } - - // attempt to sync controller with proposed flow - try { - if (backupAndPurge) { - logger.warn("Proposed flow cannot be directly inherited. However, all data that is queued in this instance is queued in a connection that exists in the Proposed flow. As a " + - "result, the existing flow will be backed up and replaced with the proposed flow."); - final File backupFile = getFlowBackupFile(); - - try { - flowService.copyCurrentFlow(backupFile); - } catch (final IOException ioe) { - throw new UninheritableFlowException("Could not inherit flow because failed to make a backup of existing flow to " + backupFile.getAbsolutePath(), ioe); - } - - logger.info("Successfully created backup of existing flow to {}. Will now purge local flow and inherit proposed flow", backupFile.getAbsolutePath()); - controller.purge(); - } - - if (!controller.isFlowSynchronized() && !existingFlowEmpty) { - updateThreadCounts(existingDataFlow.getFlowDocument().getDocumentElement(), controller); - } - - if (configuration != null) { - updateFlow(controller, configuration, existingDataFlow, existingFlowEmpty); - } - - inheritSnippets(controller, proposedFlow); - - // if auths are inheritable and we have a policy based authorizer, then inherit - if (authorizer instanceof ManagedAuthorizer) { - final ManagedAuthorizer managedAuthorizer = (ManagedAuthorizer) authorizer; - final String proposedAuthFingerprint = proposedFlow.getAuthorizerFingerprint() == null ? "" : new String(proposedFlow.getAuthorizerFingerprint(), StandardCharsets.UTF_8); - - if (authorizerInheritability.isInheritable()) { - logger.debug("Authorizations are inheritable. Will inherit from proposed fingerprint {}", proposedAuthFingerprint); - managedAuthorizer.inheritFingerprint(proposedAuthFingerprint); - } else if (!Objects.equals(managedAuthorizer.getFingerprint(), proposedAuthFingerprint)) { - // At this point, the flow is not inheritable, but we've made it this far. This can only happen if the existing flow is empty, so we can - // just forcibly inherit the authorizations. - logger.debug("Authorizations are not inheritable. Will force inheritance of proposed fingerprint {}", proposedAuthFingerprint); - managedAuthorizer.forciblyInheritFingerprint(proposedAuthFingerprint); - } - } - - logger.debug("Finished syncing flows"); - } catch (final Exception ex) { - throw new FlowSynchronizationException(ex); - } - } - - private File getFlowBackupFile() { - final File flowConfigurationFile = nifiProperties.getFlowConfigurationFile(); - final String baseFilename = StringUtils.substringBeforeLast(flowConfigurationFile.getName(), ".xml.gz"); - final DateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd-HH-mm-ss"); - final String timestamp = dateFormat.format(new Date()); - final String backupFilename = baseFilename + "-" + timestamp + ".xml.gz"; - final File backupFile = new File(flowConfigurationFile.getParentFile(), backupFilename); - - if (!backupFile.getParentFile().exists() && !backupFile.getParentFile().mkdirs()) { - throw new UninheritableFlowException("Failed to backup existing flow because the configured directory for flow.xml.gz <" + backupFile.getParentFile().getAbsolutePath() - + "> does not exist and could not be created"); - } - - return backupFile; - } - - private DataFlow getExistingDataFlow(final FlowController controller) { - final FlowManager flowManager = controller.getFlowManager(); - final ProcessGroup root = flowManager.getRootGroup(); - - // Determine missing components - final Set missingComponents = new HashSet<>(); - flowManager.getAllControllerServices().stream().filter(ComponentNode::isExtensionMissing).forEach(cs -> missingComponents.add(cs.getIdentifier())); - flowManager.getAllReportingTasks().stream().filter(ComponentNode::isExtensionMissing).forEach(r -> missingComponents.add(r.getIdentifier())); - flowManager.getAllFlowAnalysisRules().stream().filter(ComponentNode::isExtensionMissing).forEach(r -> missingComponents.add(r.getIdentifier())); - flowManager.getAllParameterProviders().stream().filter(ComponentNode::isExtensionMissing).forEach(r -> missingComponents.add(r.getIdentifier())); - flowManager.getAllFlowRegistryClients().stream().filter(ComponentNode::isExtensionMissing).forEach(c -> missingComponents.add(c.getIdentifier())); - root.findAllProcessors().stream().filter(AbstractComponentNode::isExtensionMissing).forEach(p -> missingComponents.add(p.getIdentifier())); - - logger.trace("Exporting snippets from controller"); - final byte[] existingSnippets = controller.getSnippetManager().export(); - - final byte[] existingAuthFingerprint; - final Authorizer authorizer = controller.getAuthorizer(); - if (AuthorizerCapabilityDetection.isManagedAuthorizer(authorizer)) { - final ManagedAuthorizer managedAuthorizer = (ManagedAuthorizer) authorizer; - existingAuthFingerprint = managedAuthorizer.getFingerprint().getBytes(StandardCharsets.UTF_8); - } else { - existingAuthFingerprint = null; - } - - // serialize controller state to bytes - final byte[] existingFlow; - try { - if (controller.isFlowSynchronized()) { - existingFlow = toBytes(controller); - return new StandardDataFlow(existingFlow, existingSnippets, existingAuthFingerprint, missingComponents); - } else { - existingFlow = readFlowFromDisk(); - if (existingFlow == null || existingFlow.length == 0) { - return new StandardDataFlow(existingFlow, existingSnippets, existingAuthFingerprint, missingComponents); - } else { - return new StandardDataFlow(existingFlow, existingSnippets, existingAuthFingerprint, missingComponents); - } - } - } catch (final IOException e) { - throw new FlowSerializationException(e); - } - } - - private void inheritSnippets(final FlowController controller, final DataFlow proposedFlow) { - // clear the snippets that are currently in memory - logger.trace("Clearing existing snippets"); - final SnippetManager snippetManager = controller.getSnippetManager(); - snippetManager.clear(); - - // if proposed flow has any snippets, load them - logger.trace("Loading proposed snippets"); - final byte[] proposedSnippets = proposedFlow.getSnippets(); - if (proposedSnippets != null && proposedSnippets.length > 0) { - for (final StandardSnippet snippet : SnippetManager.parseBytes(proposedSnippets)) { - snippetManager.addSnippet(snippet); - } - } - } - - private void updateFlow( - final FlowController controller, - final Document configuration, - final DataFlow existingFlow, - final boolean existingFlowEmpty - ) throws ReportingTaskInstantiationException, FlowAnalysisRuleInstantiationException { - final boolean flowAlreadySynchronized = controller.isFlowSynchronized(); - final FlowManager flowManager = controller.getFlowManager(); - - final PropertyEncryptor encryptor = controller.getEncryptor(); - - // get the root element - final Element rootElement = (Element) configuration.getElementsByTagName("flowController").item(0); - final FlowEncodingVersion encodingVersion = FlowEncodingVersion.parse(rootElement); - - // set controller config - logger.trace("Updating flow config"); - final Integer maxThreadCount = getInteger(rootElement, "maxThreadCount"); - if (maxThreadCount == null) { - controller.setMaxTimerDrivenThreadCount(getInt(rootElement, "maxTimerDrivenThreadCount")); - } else { - controller.setMaxTimerDrivenThreadCount(maxThreadCount * 2 / 3); - } - - // get the root group XML element - final Element rootGroupElement = (Element) rootElement.getElementsByTagName("rootGroup").item(0); - - // get all the parameter provider elements - final Element parameterProvidersElement = DomUtils.getChild(rootElement, "parameterProviders"); - final List parameterProviderElements = new ArrayList<>(); - if (parameterProvidersElement != null) { - parameterProviderElements.addAll(DomUtils.getChildElementsByTagName(parameterProvidersElement, "parameterProvider")); - } - - // get/create all the parameter provider nodes and DTOs - final Map parameterProviderNodesToDTOs = new HashMap<>(); - for (final Element taskElement : parameterProviderElements) { - final ParameterProviderDTO dto = FlowFromDOMFactory.getParameterProvider(taskElement, encryptor, encodingVersion); - final ParameterProviderNode parameterProvider = getOrCreateParameterProvider(controller, dto, flowAlreadySynchronized, existingFlowEmpty); - parameterProviderNodesToDTOs.put(parameterProvider, dto); - } - - // get all the flow registry client elements - final Element registriesElement = DomUtils.getChild(rootElement, "registries"); - if (registriesElement != null) { - for (final Element flowRegistryElement : DomUtils.getChildElementsByTagName(registriesElement, "flowRegistry")) { - final FlowRegistryClientDTO dto = FlowFromDOMFactory.getFlowRegistryClient(flowRegistryElement, encryptor, encodingVersion); - getOrCreateFlowRegistryClient(controller, dto, flowAlreadySynchronized, existingFlowEmpty); - } - } - - // if this controller isn't initialized or its empty, add the root group, otherwise update - final ProcessGroup rootGroup; - if (!flowAlreadySynchronized || existingFlowEmpty) { - controller.getFlowManager().withParameterContextResolution(() -> { - final Element parameterContextsElement = DomUtils.getChild(rootElement, "parameterContexts"); - if (parameterContextsElement != null) { - final List contextElements = DomUtils.getChildElementsByTagName(parameterContextsElement, "parameterContext"); - for (final Element contextElement : contextElements) { - final ParameterContextDTO parameterContextDto = FlowFromDOMFactory.getParameterContext(contextElement, encryptor); - createParameterContext(parameterContextDto, controller.getFlowManager()); - } - } - }); - - logger.trace("Adding root process group"); - rootGroup = addProcessGroup(controller, /* parent group */ null, rootGroupElement, encodingVersion); - } else { - logger.trace("Updating root process group"); - rootGroup = updateProcessGroup(controller, /* parent group */ null, rootGroupElement, encodingVersion); - } - - rootGroup.findAllRemoteProcessGroups().forEach(RemoteProcessGroup::initialize); - - // get all the reporting task elements - final Element reportingTasksElement = DomUtils.getChild(rootElement, "reportingTasks"); - final List reportingTaskElements = new ArrayList<>(); - if (reportingTasksElement != null) { - reportingTaskElements.addAll(DomUtils.getChildElementsByTagName(reportingTasksElement, "reportingTask")); - } - - // get/create all the reporting task nodes and DTOs, but don't apply their scheduled state yet - final Map reportingTaskNodesToDTOs = new HashMap<>(); - for (final Element taskElement : reportingTaskElements) { - final ReportingTaskDTO dto = FlowFromDOMFactory.getReportingTask(taskElement, encryptor, encodingVersion); - final ReportingTaskNode reportingTask = getOrCreateReportingTask(controller, dto, flowAlreadySynchronized, existingFlowEmpty); - reportingTaskNodesToDTOs.put(reportingTask, dto); - } - - // get all the flow analysis rule elements - final Element flowAnalysisRulesElement = DomUtils.getChild(rootElement, "flowAnalysisRules"); - final List flowAnalysisRuleElements = new ArrayList<>(); - if (flowAnalysisRulesElement != null) { - flowAnalysisRuleElements.addAll(DomUtils.getChildElementsByTagName(flowAnalysisRulesElement, "flowAnalysisRule")); - } - - // get/create all the flow analysis rule nodes and DTOs, but don't apply their state yet - final Map flowAnalysisRuleNodesToDTOs = new HashMap<>(); - for (final Element taskElement : flowAnalysisRuleElements) { - final FlowAnalysisRuleDTO dto = FlowFromDOMFactory.getFlowAnalysisRule(taskElement, encryptor, encodingVersion); - final FlowAnalysisRuleNode flowAnalysisRule = getOrCreateFlowAnalysisRule(controller, dto, flowAlreadySynchronized, existingFlowEmpty); - flowAnalysisRuleNodesToDTOs.put(flowAnalysisRule, dto); - } - - final Element controllerServicesElement = DomUtils.getChild(rootElement, "controllerServices"); - if (controllerServicesElement != null) { - final List serviceElements = DomUtils.getChildElementsByTagName(controllerServicesElement, "controllerService"); - - if (!flowAlreadySynchronized || existingFlowEmpty) { - // If the encoding version is null, we are loading a flow from NiFi 0.x, where Controller - // Services could not be scoped by Process Group. As a result, we want to move the Process Groups - // to the root Group. Otherwise, we want to use a null group, which indicates a Controller-level - // Controller Service. - final ProcessGroup group = (encodingVersion == null) ? rootGroup : null; - final Map controllerServices = ControllerServiceLoader.loadControllerServices( - serviceElements, controller, group, encryptor, encodingVersion); - - // If we are moving controller services to the root group we also need to see if any reporting tasks - // or flow analysis rules reference them, and if so we need to clone the CS and update the task- and rule references - if (group != null) { - // find all the controller service ids referenced by reporting tasks and flow analysis rules - final Set controllerServicesInReportingTasksAndFlowAnalysisRules = Stream.concat( - reportingTaskNodesToDTOs.keySet().stream(), - flowAnalysisRuleNodesToDTOs.keySet().stream() - ) - .flatMap(r -> r.getEffectivePropertyValues().entrySet().stream()) - .filter(e -> e.getKey().getControllerServiceDefinition() != null) - .map(Map.Entry::getValue) - .collect(Collectors.toSet()); - // find all the controller service ids referenced by parameter providers - final Set controllerServicesInParameterProviders = parameterProviderNodesToDTOs.keySet().stream() - .flatMap(r -> r.getEffectivePropertyValues().entrySet().stream()) - .filter(e -> e.getKey().getControllerServiceDefinition() != null) - .map(Map.Entry::getValue) - .collect(Collectors.toSet()); - - // find the controller service nodes for each id referenced by a reporting task or flow analysis rule - final Set controllerServicesToClone = controllerServices.keySet().stream() - .filter(cs -> controllerServicesInReportingTasksAndFlowAnalysisRules.contains(cs.getIdentifier()) - || controllerServicesInParameterProviders.contains(cs.getIdentifier())) - .collect(Collectors.toSet()); - - // clone the controller services and map the original id to the clone - final Map controllerServiceMapping = new HashMap<>(); - for (ControllerServiceNode controllerService : controllerServicesToClone) { - final ControllerServiceNode clone = ControllerServiceLoader.cloneControllerService(controller, controllerService); - flowManager.addRootControllerService(clone); - controllerServiceMapping.put(controllerService.getIdentifier(), clone); - } - - // update the reporting tasks and parameter providers to reference the cloned controller services - updateControllerLevelControllerServices(reportingTaskNodesToDTOs.keySet(), controllerServiceMapping); - updateControllerLevelControllerServices(parameterProviderNodesToDTOs.keySet(), controllerServiceMapping); - - // enable all the cloned controller services - ControllerServiceLoader.enableControllerServices(controllerServiceMapping.values(), controller, autoResumeState); - } - - // enable all the original controller services - ControllerServiceLoader.enableControllerServices(controllerServices, controller, encryptor, autoResumeState, encodingVersion); - } - } - - // Now that root controller services are available, attempt to fetch parameters - for (final ParameterProviderNode parameterProviderNode : parameterProviderNodesToDTOs.keySet()) { - try { - parameterProviderNode.fetchParameters(); - } catch (final Exception e) { - logger.warn("Failed to fetch parameters for provider " + parameterProviderNode.getName(), e); - } - } - - scaleRootGroup(rootGroup, encodingVersion); - - // now that controller services are loaded and enabled we can apply the scheduled state to each reporting task - for (Map.Entry entry : reportingTaskNodesToDTOs.entrySet()) { - applyReportingTaskScheduleState(controller, entry.getValue(), entry.getKey(), flowAlreadySynchronized, existingFlowEmpty); - } - - // now that controller services are loaded and enabled we can apply the state to each flow analysis rule - for (Map.Entry entry : flowAnalysisRuleNodesToDTOs.entrySet()) { - applyFlowAnalysisRuleState(controller, entry.getValue(), entry.getKey(), flowAlreadySynchronized, existingFlowEmpty); - } - } - - private ParameterContext createParameterContext(final ParameterContextDTO dto, final FlowManager flowManager) { - final Map parameters = dto.getParameters().stream() - .map(ParameterEntity::getParameter) - .map(this::createParameter) - .collect(Collectors.toMap(param -> param.getDescriptor().getName(), Function.identity())); - - final List referencedIds = dto.getInheritedParameterContexts() == null ? Collections.emptyList() - : dto.getInheritedParameterContexts().stream() - .map(ParameterContextReferenceEntity::getId) - .collect(Collectors.toList()); - - ParameterProviderConfiguration parameterProviderConfiguration = null; - if (dto.getParameterProviderConfiguration() != null) { - final ParameterProviderConfigurationEntity parameterProviderConfigurationEntity = dto.getParameterProviderConfiguration(); - final ParameterProviderConfigurationDTO configurationDTO = parameterProviderConfigurationEntity.getComponent(); - parameterProviderConfiguration = new StandardParameterProviderConfiguration(configurationDTO.getParameterProviderId(), - configurationDTO.getParameterGroupName(), configurationDTO.getSynchronized()); - } - final ParameterContext context = flowManager.createParameterContext(dto.getId(), dto.getName(), parameters, referencedIds, parameterProviderConfiguration); - context.setDescription(dto.getDescription()); - return context; - } - - private Parameter createParameter(final ParameterDTO dto) { - final ParameterDescriptor parameterDescriptor = new ParameterDescriptor.Builder() - .name(dto.getName()) - .description(dto.getDescription()) - .sensitive(Boolean.TRUE.equals(dto.getSensitive())) - .build(); - - return new Parameter(parameterDescriptor, dto.getValue(), null, dto.getProvided()); - } - - public static String getReferenceId(final ComponentReferenceEntity referenceEntity) { - if (referenceEntity == null) { - return null; - } - final ComponentReferenceDTO dto = referenceEntity.getComponent(); - if (dto == null) { - return null; - } - return dto.getId(); - } - - private void updateControllerLevelControllerServices(final Set componentNodes, final Map controllerServiceMapping) { - for (final ComponentNode componentNode : componentNodes) { - if (componentNode.getProperties() != null) { - componentNode.pauseValidationTrigger(); - try { - final Set> propertyDescriptors = componentNode.getEffectivePropertyValues().entrySet().stream() - .filter(e -> e.getKey().getControllerServiceDefinition() != null) - .filter(e -> controllerServiceMapping.containsKey(e.getValue())) - .collect(Collectors.toSet()); - - final Map controllerServiceProps = new HashMap<>(); - - for (Map.Entry propEntry : propertyDescriptors) { - final PropertyDescriptor propertyDescriptor = propEntry.getKey(); - final ControllerServiceNode clone = controllerServiceMapping.get(propEntry.getValue()); - controllerServiceProps.put(propertyDescriptor.getName(), clone.getIdentifier()); - } - - componentNode.setProperties(controllerServiceProps); - } finally { - componentNode.resumeValidationTrigger(); - } - } - } - } - - void scaleRootGroup(final ProcessGroup rootGroup, final FlowEncodingVersion encodingVersion) { - if (encodingVersion == null || encodingVersion.getMajorVersion() < 1) { - // Calculate new Positions if the encoding version of the flow is older than 1.0. - PositionScaler.scale(rootGroup, 1.5, 1.34); - } - } - - private void updateThreadCounts(final Element rootElement, final FlowController controller) { - logger.trace("Setting controller thread counts"); - final Integer maxThreadCount = getInteger(rootElement, "maxThreadCount"); - if (maxThreadCount == null) { - controller.setMaxTimerDrivenThreadCount(getInt(rootElement, "maxTimerDrivenThreadCount")); - } else { - controller.setMaxTimerDrivenThreadCount(maxThreadCount * 2 / 3); - } - } - - public static boolean isFlowEmpty(final Document flowDocument) { - if (flowDocument == null) { - return true; - } - - final Element rootElement = flowDocument.getDocumentElement(); - if (rootElement == null) { - return true; - } - - final FlowEncodingVersion encodingVersion = FlowEncodingVersion.parse(rootElement); - - final Element reportingTasksElement = DomUtils.getChild(rootElement, "reportingTasks"); - if (reportingTasksElement != null) { - final List taskElements = DomUtils.getChildElementsByTagName(reportingTasksElement, "reportingTask"); - if (!taskElements.isEmpty()) { - return false; - } - } - - final Element flowAnalysisRulesElement = DomUtils.getChild(rootElement, "flowAnalysisRules"); - if (flowAnalysisRulesElement != null) { - final List flowAnalysisRulesElements = DomUtils.getChildElementsByTagName(flowAnalysisRulesElement, "flowAnalysisRule"); - if (!flowAnalysisRulesElements.isEmpty()) { - return false; - } - } - - final Element parameterProvidersElement = DomUtils.getChild(rootElement, "parameterProviders"); - if (parameterProvidersElement != null) { - final List providerElements = DomUtils.getChildElementsByTagName(parameterProvidersElement, "parameterProvider"); - if (!providerElements.isEmpty()) { - return false; - } - } - - final Element controllerServicesElement = DomUtils.getChild(rootElement, "controllerServices"); - if (controllerServicesElement != null) { - final List unrootedControllerServiceElements = DomUtils.getChildElementsByTagName(controllerServicesElement, "controllerService"); - if (!unrootedControllerServiceElements.isEmpty()) { - return false; - } - } - - final Element registriesElement = DomUtils.getChild(rootElement, "registries"); - if (registriesElement != null) { - final List flowRegistryElems = DomUtils.getChildElementsByTagName(registriesElement, "flowRegistry"); - if (!flowRegistryElems.isEmpty()) { - return false; - } - } - - final Element parameterContextsElement = DomUtils.getChild(rootElement, "parameterContexts"); - if (parameterContextsElement != null) { - final List contextList = DomUtils.getChildElementsByTagName(parameterContextsElement, "parameterContext"); - if (!contextList.isEmpty()) { - return false; - } - } - - logger.trace("Parsing process group from DOM"); - final Element rootGroupElement = (Element) rootElement.getElementsByTagName("rootGroup").item(0); - final ProcessGroupDTO rootGroupDto = FlowFromDOMFactory.getProcessGroup(null, rootGroupElement, null, encodingVersion); - return isEmpty(rootGroupDto); - } - - private static boolean isEmpty(final ProcessGroupDTO dto) { - if (dto == null) { - return true; - } - - final FlowSnippetDTO contents = dto.getContents(); - if (contents == null) { - return true; - } - - final String parameterContextId = dto.getParameterContext() == null ? null : dto.getParameterContext().getId(); - - return CollectionUtils.isEmpty(contents.getProcessors()) - && CollectionUtils.isEmpty(contents.getConnections()) - && CollectionUtils.isEmpty(contents.getFunnels()) - && CollectionUtils.isEmpty(contents.getLabels()) - && CollectionUtils.isEmpty(contents.getInputPorts()) - && CollectionUtils.isEmpty(contents.getOutputPorts()) - && CollectionUtils.isEmpty(contents.getProcessGroups()) - && CollectionUtils.isEmpty(contents.getRemoteProcessGroups()) - && CollectionUtils.isEmpty(contents.getControllerServices()) - && parameterContextId == null; - } - - - private byte[] readFlowFromDisk() throws IOException { - final Path flowPath = nifiProperties.getFlowConfigurationFile().toPath(); - if (!Files.exists(flowPath) || Files.size(flowPath) == 0) { - return new byte[0]; - } - - final ByteArrayOutputStream baos = new ByteArrayOutputStream(); - - try (final InputStream in = Files.newInputStream(flowPath, StandardOpenOption.READ); - final InputStream gzipIn = new GZIPInputStream(in)) { - FileUtils.copy(gzipIn, baos); - } - - return baos.toByteArray(); - } - - private ParameterProviderNode getOrCreateParameterProvider(final FlowController controller, final ParameterProviderDTO dto, - final boolean controllerInitialized, final boolean existingFlowEmpty) { - // create a new parameter provider node when the controller is not initialized or the flow is empty - if (!controllerInitialized || existingFlowEmpty) { - BundleCoordinate coordinate; - try { - coordinate = BundleUtils.getCompatibleBundle(extensionManager, dto.getType(), dto.getBundle()); - } catch (final IllegalStateException e) { - final BundleDTO bundleDTO = dto.getBundle(); - if (bundleDTO == null) { - coordinate = BundleCoordinate.UNKNOWN_COORDINATE; - } else { - coordinate = new BundleCoordinate(bundleDTO.getGroup(), bundleDTO.getArtifact(), bundleDTO.getVersion()); - } - } - - final ParameterProviderNode parameterProvider = controller.getFlowManager().createParameterProvider(dto.getType(), dto.getId(), coordinate, false); - parameterProvider.setName(dto.getName()); - parameterProvider.setComments(dto.getComments()); - - parameterProvider.setAnnotationData(dto.getAnnotationData()); - parameterProvider.setProperties(dto.getProperties()); - return parameterProvider; - } else { - // otherwise, return the existing parameter provider node - return controller.getFlowManager().getParameterProvider(dto.getId()); - } - } - - private FlowRegistryClientNode getOrCreateFlowRegistryClient(final FlowController controller, final FlowRegistryClientDTO dto, - final boolean controllerInitialized, final boolean existingFlowEmpty) { - // create a new flow registry client node when the controller is not initialized or the flow is empty - if (!controllerInitialized || existingFlowEmpty) { - BundleCoordinate coordinate; - try { - coordinate = BundleUtils.getCompatibleBundle(extensionManager, dto.getType(), dto.getBundle()); - } catch (final IllegalStateException e) { - final BundleDTO bundleDTO = dto.getBundle(); - if (bundleDTO == null) { - coordinate = BundleCoordinate.UNKNOWN_COORDINATE; - } else { - coordinate = new BundleCoordinate(bundleDTO.getGroup(), bundleDTO.getArtifact(), bundleDTO.getVersion()); - } - } - - final FlowRegistryClientNode registryClient = controller.getFlowManager().createFlowRegistryClient(dto.getType(), dto.getId(), coordinate, Collections.emptySet(), false, true, null); - registryClient.setName(dto.getName()); - registryClient.setDescription(dto.getDescription()); - registryClient.setAnnotationData(dto.getAnnotationData()); - final Set sensitiveDynamicPropertyNames = getSensitiveDynamicPropertyNames(dto.getSensitiveDynamicPropertyNames(), registryClient); - registryClient.setProperties(dto.getProperties(), false, sensitiveDynamicPropertyNames); - return registryClient; - } else { - // otherwise return the existing flow registry client node - return controller.getFlowManager().getFlowRegistryClient(dto.getId()); - } - } - - private ReportingTaskNode getOrCreateReportingTask(final FlowController controller, final ReportingTaskDTO dto, final boolean controllerInitialized, final boolean existingFlowEmpty) - throws ReportingTaskInstantiationException { - // create a new reporting task node when the controller is not initialized or the flow is empty - if (!controllerInitialized || existingFlowEmpty) { - BundleCoordinate coordinate; - try { - coordinate = BundleUtils.getCompatibleBundle(extensionManager, dto.getType(), dto.getBundle()); - } catch (final IllegalStateException e) { - final BundleDTO bundleDTO = dto.getBundle(); - if (bundleDTO == null) { - coordinate = BundleCoordinate.UNKNOWN_COORDINATE; - } else { - coordinate = new BundleCoordinate(bundleDTO.getGroup(), bundleDTO.getArtifact(), bundleDTO.getVersion()); - } - } - - final ReportingTaskNode reportingTask = controller.createReportingTask(dto.getType(), dto.getId(), coordinate, false); - reportingTask.setName(dto.getName()); - reportingTask.setComments(dto.getComments()); - reportingTask.setSchedulingPeriod(dto.getSchedulingPeriod()); - reportingTask.setSchedulingStrategy(SchedulingStrategy.valueOf(dto.getSchedulingStrategy())); - - reportingTask.setAnnotationData(dto.getAnnotationData()); - final Set sensitiveDynamicPropertyNames = getSensitiveDynamicPropertyNames(dto.getSensitiveDynamicPropertyNames(), reportingTask); - reportingTask.setProperties(dto.getProperties(), false, sensitiveDynamicPropertyNames); - return reportingTask; - } else { - // otherwise return the existing reporting task node - return controller.getReportingTaskNode(dto.getId()); - } - } - - private void applyReportingTaskScheduleState(final FlowController controller, final ReportingTaskDTO dto, final ReportingTaskNode reportingTask, - final boolean controllerInitialized, final boolean existingFlowEmpty) { - if (!controllerInitialized || existingFlowEmpty) { - applyNewReportingTaskScheduleState(controller, dto, reportingTask); - } else { - applyExistingReportingTaskScheduleState(controller, dto, reportingTask); - } - } - - private void applyNewReportingTaskScheduleState(final FlowController controller, final ReportingTaskDTO dto, final ReportingTaskNode reportingTask) { - if (autoResumeState) { - if (ScheduledState.RUNNING.name().equals(dto.getState())) { - try { - controller.startReportingTask(reportingTask); - } catch (final Exception e) { - logger.error("Failed to start {} due to {}", reportingTask, e); - if (logger.isDebugEnabled()) { - logger.error("", e); - } - controller.getBulletinRepository().addBulletin(BulletinFactory.createBulletin( - "Reporting Tasks", Severity.ERROR.name(), "Failed to start " + reportingTask + " due to " + e)); - } - } else if (ScheduledState.DISABLED.name().equals(dto.getState())) { - try { - controller.disableReportingTask(reportingTask); - } catch (final Exception e) { - logger.error("Failed to mark {} as disabled due to {}", reportingTask, e); - if (logger.isDebugEnabled()) { - logger.error("", e); - } - controller.getBulletinRepository().addBulletin(BulletinFactory.createBulletin( - "Reporting Tasks", Severity.ERROR.name(), "Failed to mark " + reportingTask + " as disabled due to " + e)); - } - } - } - } - - private void applyExistingReportingTaskScheduleState(final FlowController controller, final ReportingTaskDTO dto, final ReportingTaskNode taskNode) { - if (!taskNode.getScheduledState().name().equals(dto.getState())) { - try { - switch (ScheduledState.valueOf(dto.getState())) { - case DISABLED: - if (taskNode.isRunning()) { - controller.stopReportingTask(taskNode); - } - controller.disableReportingTask(taskNode); - break; - case RUNNING: - if (taskNode.getScheduledState() == ScheduledState.DISABLED) { - controller.enableReportingTask(taskNode); - } - controller.startReportingTask(taskNode); - break; - case STOPPED: - if (taskNode.getScheduledState() == ScheduledState.DISABLED) { - controller.enableReportingTask(taskNode); - } else if (taskNode.getScheduledState() == ScheduledState.RUNNING) { - controller.stopReportingTask(taskNode); - } - break; - } - } catch (final IllegalStateException ise) { - logger.error("Failed to change Scheduled State of {} from {} to {} due to {}", taskNode, taskNode.getScheduledState().name(), dto.getState(), ise.toString()); - logger.error("", ise); - - // create bulletin for the Processor Node - controller.getBulletinRepository().addBulletin(BulletinFactory.createBulletin("Node Reconnection", Severity.ERROR.name(), - "Failed to change Scheduled State of " + taskNode + " from " + taskNode.getScheduledState().name() + " to " + dto.getState() + " due to " + ise.toString())); - - // create bulletin at Controller level. - controller.getBulletinRepository().addBulletin(BulletinFactory.createBulletin("Node Reconnection", Severity.ERROR.name(), - "Failed to change Scheduled State of " + taskNode + " from " + taskNode.getScheduledState().name() + " to " + dto.getState() + " due to " + ise.toString())); - } - } - } - - private FlowAnalysisRuleNode getOrCreateFlowAnalysisRule(final FlowController controller, final FlowAnalysisRuleDTO dto, final boolean controllerInitialized, final boolean existingFlowEmpty) - throws FlowAnalysisRuleInstantiationException { - // create a new flow analysis rule node when the controller is not initialized or the flow is empty - if (!controllerInitialized || existingFlowEmpty) { - BundleCoordinate coordinate; - try { - coordinate = BundleUtils.getCompatibleBundle(extensionManager, dto.getType(), dto.getBundle()); - } catch (final IllegalStateException e) { - final BundleDTO bundleDTO = dto.getBundle(); - if (bundleDTO == null) { - coordinate = BundleCoordinate.UNKNOWN_COORDINATE; - } else { - coordinate = new BundleCoordinate(bundleDTO.getGroup(), bundleDTO.getArtifact(), bundleDTO.getVersion()); - } - } - - final FlowAnalysisRuleNode flowAnalysisRule = controller.createFlowAnalysisRule(dto.getType(), dto.getId(), coordinate, false); - flowAnalysisRule.setName(dto.getName()); - flowAnalysisRule.setComments(dto.getComments()); - - flowAnalysisRule.setEnforcementPolicy(EnforcementPolicy.valueOf(dto.getEnforcementPolicy())); - - final Set sensitiveDynamicPropertyNames = getSensitiveDynamicPropertyNames(dto.getSensitiveDynamicPropertyNames(), flowAnalysisRule); - flowAnalysisRule.setProperties(dto.getProperties(), false, sensitiveDynamicPropertyNames); - - return flowAnalysisRule; - } else { - // otherwise return the existing flow analysis rule node - return controller.getFlowAnalysisRuleNode(dto.getId()); - } - } - - private void applyFlowAnalysisRuleState(final FlowController controller, final FlowAnalysisRuleDTO dto, final FlowAnalysisRuleNode flowAnalysisRule, - final boolean controllerInitialized, final boolean existingFlowEmpty) { - if (!controllerInitialized || existingFlowEmpty) { - applyNewFlowAnalysisRuleState(controller, dto, flowAnalysisRule); - } else { - applyExistingFlowAnalysisRuleState(controller, dto, flowAnalysisRule); - } - } - - private void applyNewFlowAnalysisRuleState(final FlowController controller, final FlowAnalysisRuleDTO dto, final FlowAnalysisRuleNode flowAnalysisRule) { - if (autoResumeState) { - if (FlowAnalysisRuleState.ENABLED.name().equals(dto.getState())) { - try { - controller.enableFlowAnalysisRule(flowAnalysisRule); - } catch (final Exception e) { - logger.error("Failed to enable {} due to {}", flowAnalysisRule, e); - if (logger.isDebugEnabled()) { - logger.error("", e); - } - controller.getBulletinRepository().addBulletin(BulletinFactory.createBulletin( - "Flow Analysis Rules", Severity.ERROR.name(), "Failed to start " + flowAnalysisRule + " due to " + e)); - } - } else if (FlowAnalysisRuleState.DISABLED.name().equals(dto.getState())) { - try { - controller.disableFlowAnalysisRule(flowAnalysisRule); - } catch (final Exception e) { - logger.error("Failed to mark {} as disabled due to {}", flowAnalysisRule, e); - if (logger.isDebugEnabled()) { - logger.error("", e); - } - controller.getBulletinRepository().addBulletin(BulletinFactory.createBulletin( - "Flow Analysis Rules", Severity.ERROR.name(), "Failed to mark " + flowAnalysisRule + " as disabled due to " + e)); - } - } - } - } - - private void applyExistingFlowAnalysisRuleState(final FlowController controller, final FlowAnalysisRuleDTO dto, final FlowAnalysisRuleNode node) { - if (!node.getState().name().equals(dto.getState())) { - try { - switch (FlowAnalysisRuleState.valueOf(dto.getState())) { - case DISABLED: - if (node.isEnabled()) { - controller.disableFlowAnalysisRule(node); - } - break; - case ENABLED: - if (!node.isEnabled()) { - controller.enableFlowAnalysisRule(node); - } - break; - } - } catch (final IllegalStateException ise) { - logger.error("Failed to change State of {} from {} to {} due to {}", node, node.getState().name(), dto.getState(), ise.toString()); - logger.error("", ise); - - // create bulletin for node - controller.getBulletinRepository().addBulletin(BulletinFactory.createBulletin("Node Reconnection", Severity.ERROR.name(), - "Failed to change State of " + node + " from " + node.getState().name() + " to " + dto.getState() + " due to " + ise.toString())); - - // create bulletin at Controller level. - controller.getBulletinRepository().addBulletin(BulletinFactory.createBulletin("Node Reconnection", Severity.ERROR.name(), - "Failed to change State of " + node + " from " + node.getState().name() + " to " + dto.getState() + " due to " + ise.toString())); - } - } - } - - private ControllerServiceState getFinalTransitionState(final ControllerServiceState state) { - switch (state) { - case DISABLED: - case DISABLING: - return ControllerServiceState.DISABLED; - case ENABLED: - case ENABLING: - return ControllerServiceState.ENABLED; - default: - throw new AssertionError(); - } - } - - private ProcessGroup updateProcessGroup(final FlowController controller, final ProcessGroup parentGroup, final Element processGroupElement, - final FlowEncodingVersion encodingVersion) { - - // get the parent group ID - final String parentId = (parentGroup == null) ? null : parentGroup.getIdentifier(); - - final PropertyEncryptor encryptor = controller.getEncryptor(); - - // get the process group - final ProcessGroupDTO processGroupDto = FlowFromDOMFactory.getProcessGroup(parentId, processGroupElement, encryptor, encodingVersion); - final FlowManager flowManager = controller.getFlowManager(); - - // update the process group - if (parentId == null) { - - /* - * Labels are not included in the "inherit flow" algorithm, so we cannot - * blindly update them because they may not exist in the current flow. - * Therefore, we first remove all labels, and then let the updating - * process add labels defined in the new flow. - */ - final ProcessGroup root = flowManager.getRootGroup(); - for (final Label label : root.findAllLabels()) { - label.getProcessGroup().removeLabel(label); - } - } - - // update the process group - final ProcessGroup processGroup = flowManager.getGroup(processGroupDto.getId()); - if (processGroup == null) { - throw new IllegalStateException("No Group with ID " + processGroupDto.getId() + " exists"); - } - - updateProcessGroup(processGroup, processGroupDto, controller.getFlowManager().getParameterContextManager()); - - // determine the scheduled state of all of the Controller Service - final List controllerServiceNodeList = getChildrenByTagName(processGroupElement, "controllerService"); - final Set toDisable = new HashSet<>(); - final Set toEnable = new HashSet<>(); - - for (final Element serviceElement : controllerServiceNodeList) { - final ControllerServiceDTO dto = FlowFromDOMFactory.getControllerService(serviceElement, encryptor, encodingVersion); - final ControllerServiceNode serviceNode = processGroup.getControllerService(dto.getId()); - - // Check if the controller service is in the correct state. We consider it the correct state if - // we are in a transitional state and heading in the right direction or already in the correct state. - // E.g., it is the correct state if it should be 'DISABLED' and it is either DISABLED or DISABLING. - final ControllerServiceState serviceState = getFinalTransitionState(serviceNode.getState()); - final ControllerServiceState clusterState = getFinalTransitionState(ControllerServiceState.valueOf(dto.getState())); - - if (serviceState != clusterState) { - switch (clusterState) { - case DISABLED: - toDisable.add(serviceNode); - break; - case ENABLED: - toEnable.add(serviceNode); - break; - } - } - } - - // Ensure that all services have been validated, so that we don't attempt to enable a service that is still in a 'validating' state - toEnable.forEach(ControllerServiceNode::performValidation); - - final ControllerServiceProvider serviceProvider = controller.getControllerServiceProvider(); - serviceProvider.disableControllerServicesAsync(toDisable); - serviceProvider.enableControllerServices(toEnable); - - // processors & ports cannot be updated - they must be the same. Except for the scheduled state. - final List processorNodeList = getChildrenByTagName(processGroupElement, "processor"); - for (final Element processorElement : processorNodeList) { - final ProcessorDTO dto = FlowFromDOMFactory.getProcessor(processorElement, encryptor, encodingVersion); - final ProcessorNode procNode = processGroup.getProcessor(dto.getId()); - - final ScheduledState procState = getScheduledState(procNode, controller); - updateNonFingerprintedProcessorSettings(procNode, dto); - - if (!procState.name().equals(dto.getState())) { - try { - switch (ScheduledState.valueOf(dto.getState())) { - case DISABLED: - // switch processor do disabled. This means we have to stop it (if it's already stopped, this method does nothing), - // and then we have to disable it. - controller.stopProcessor(procNode.getProcessGroupIdentifier(), procNode.getIdentifier()); - procNode.getProcessGroup().disableProcessor(procNode); - break; - case RUNNING: - // we want to run now. Make sure processor is not disabled and then start it. - procNode.performValidation(); - procNode.getProcessGroup().enableProcessor(procNode); - controller.startProcessor(procNode.getProcessGroupIdentifier(), procNode.getIdentifier(), false); - break; - case STOPPED: - case RUN_ONCE: - if (procState == ScheduledState.DISABLED) { - procNode.getProcessGroup().enableProcessor(procNode); - } else if (procState == ScheduledState.RUNNING || procState == ScheduledState.RUN_ONCE) { - controller.stopProcessor(procNode.getProcessGroupIdentifier(), procNode.getIdentifier()); - } - break; - } - } catch (final IllegalStateException ise) { - logger.error("Failed to change Scheduled State of {} from {} to {} due to {}", procNode, procNode.getScheduledState().name(), dto.getState(), ise.toString()); - logger.error("", ise); - - // create bulletin for the Processor Node - controller.getBulletinRepository().addBulletin(BulletinFactory.createBulletin(procNode, "Node Reconnection", Severity.ERROR.name(), - "Failed to change Scheduled State of " + procNode + " from " + procNode.getScheduledState().name() + " to " + dto.getState() + " due to " + ise.toString())); - - // create bulletin at Controller level. - controller.getBulletinRepository().addBulletin(BulletinFactory.createBulletin("Node Reconnection", Severity.ERROR.name(), - "Failed to change Scheduled State of " + procNode + " from " + procNode.getScheduledState().name() + " to " + dto.getState() + " due to " + ise.toString())); - } - } - } - - final List inputPortList = getChildrenByTagName(processGroupElement, "inputPort"); - for (final Element portElement : inputPortList) { - final PortDTO dto = FlowFromDOMFactory.getPort(portElement); - final Port port = processGroup.getInputPort(dto.getId()); - - final ScheduledState portState = getScheduledState(port, controller); - - if (!portState.name().equals(dto.getState())) { - switch (ScheduledState.valueOf(dto.getState())) { - case DISABLED: - // switch processor do disabled. This means we have to stop it (if it's already stopped, this method does nothing), - // and then we have to disable it. - controller.stopConnectable(port); - port.getProcessGroup().disableInputPort(port); - break; - case RUNNING: - // we want to run now. Make sure processor is not disabled and then start it. - port.getProcessGroup().enableInputPort(port); - controller.startConnectable(port); - break; - case STOPPED: - if (portState == ScheduledState.DISABLED) { - port.getProcessGroup().enableInputPort(port); - } else if (portState == ScheduledState.RUNNING) { - controller.stopConnectable(port); - } - break; - } - } - } - - final List outputPortList = getChildrenByTagName(processGroupElement, "outputPort"); - for (final Element portElement : outputPortList) { - final PortDTO dto = FlowFromDOMFactory.getPort(portElement); - final Port port = processGroup.getOutputPort(dto.getId()); - - final ScheduledState portState = getScheduledState(port, controller); - - if (!portState.name().equals(dto.getState())) { - switch (ScheduledState.valueOf(dto.getState())) { - case DISABLED: - // switch processor do disabled. This means we have to stop it (if it's already stopped, this method does nothing), - // and then we have to disable it. - controller.stopConnectable(port); - port.getProcessGroup().disableOutputPort(port); - break; - case RUNNING: - // we want to run now. Make sure processor is not disabled and then start it. - port.getProcessGroup().enableOutputPort(port); - controller.startConnectable(port); - break; - case STOPPED: - if (portState == ScheduledState.DISABLED) { - port.getProcessGroup().enableOutputPort(port); - } else if (portState == ScheduledState.RUNNING) { - controller.stopConnectable(port); - } - break; - } - } - } - - // Update scheduled state of Remote Group Ports - final List remoteProcessGroupList = getChildrenByTagName(processGroupElement, "remoteProcessGroup"); - for (final Element remoteGroupElement : remoteProcessGroupList) { - final RemoteProcessGroupDTO remoteGroupDto = FlowFromDOMFactory.getRemoteProcessGroup(remoteGroupElement, encryptor); - final RemoteProcessGroup rpg = processGroup.getRemoteProcessGroup(remoteGroupDto.getId()); - - // input ports - final List inputPortElements = getChildrenByTagName(remoteGroupElement, "inputPort"); - for (final Element inputPortElement : inputPortElements) { - final RemoteProcessGroupPortDescriptor portDescriptor = FlowFromDOMFactory.getRemoteProcessGroupPort(inputPortElement); - final String inputPortId = portDescriptor.getId(); - final RemoteGroupPort inputPort = rpg.getInputPort(inputPortId); - if (inputPort == null) { - continue; - } - - final ScheduledState portState = getScheduledState(inputPort, controller); - - if (portDescriptor.isTransmitting()) { - if (portState != ScheduledState.RUNNING && portState != ScheduledState.STARTING) { - controller.startTransmitting(inputPort); - } - } else if (portState != ScheduledState.STOPPED && portState != ScheduledState.STOPPING) { - controller.stopTransmitting(inputPort); - } - } - - // output ports - final List outputPortElements = getChildrenByTagName(remoteGroupElement, "outputPort"); - for (final Element outputPortElement : outputPortElements) { - final RemoteProcessGroupPortDescriptor portDescriptor = FlowFromDOMFactory.getRemoteProcessGroupPort(outputPortElement); - final String outputPortId = portDescriptor.getId(); - final RemoteGroupPort outputPort = rpg.getOutputPort(outputPortId); - if (outputPort == null) { - continue; - } - - final ScheduledState portState = getScheduledState(outputPort, controller); - - if (portDescriptor.isTransmitting()) { - if (portState != ScheduledState.RUNNING && portState != ScheduledState.STARTING) { - controller.startTransmitting(outputPort); - } - } else if (portState != ScheduledState.STOPPED && portState != ScheduledState.STOPPING) { - controller.stopTransmitting(outputPort); - } - } - } - - // add labels - final List labelNodeList = getChildrenByTagName(processGroupElement, "label"); - for (final Element labelElement : labelNodeList) { - final LabelDTO labelDTO = FlowFromDOMFactory.getLabel(labelElement); - final Label label = flowManager.createLabel(labelDTO.getId(), labelDTO.getLabel()); - label.setStyle(labelDTO.getStyle()); - label.setPosition(new Position(labelDTO.getPosition().getX(), labelDTO.getPosition().getY())); - label.setVersionedComponentId(labelDTO.getVersionedComponentId()); - if (labelDTO.getWidth() != null && labelDTO.getHeight() != null) { - label.setSize(new Size(labelDTO.getWidth(), labelDTO.getHeight())); - } - if (labelDTO.getzIndex() != null) { - label.setZIndex(labelDTO.getzIndex()); - } - - processGroup.addLabel(label); - } - - // update nested process groups (recursively) - final List nestedProcessGroupNodeList = getChildrenByTagName(processGroupElement, "processGroup"); - for (final Element nestedProcessGroupElement : nestedProcessGroupNodeList) { - updateProcessGroup(controller, processGroup, nestedProcessGroupElement, encodingVersion); - } - - // update connections - final List connectionNodeList = getChildrenByTagName(processGroupElement, "connection"); - for (final Element connectionElement : connectionNodeList) { - final ConnectionDTO dto = FlowFromDOMFactory.getConnection(connectionElement); - - final Connection connection = processGroup.getConnection(dto.getId()); - connection.setName(dto.getName()); - connection.setProcessGroup(processGroup); - - if (dto.getLabelIndex() != null) { - connection.setLabelIndex(dto.getLabelIndex()); - } - if (dto.getzIndex() != null) { - connection.setZIndex(dto.getzIndex()); - } - - final List bendPoints = new ArrayList<>(); - for (final PositionDTO bend : dto.getBends()) { - bendPoints.add(new Position(bend.getX(), bend.getY())); - } - connection.setBendPoints(bendPoints); - - List newPrioritizers = null; - final List prioritizers = dto.getPrioritizers(); - if (prioritizers != null) { - final List newPrioritizersClasses = new ArrayList<>(prioritizers); - newPrioritizers = new ArrayList<>(); - for (final String className : newPrioritizersClasses) { - try { - newPrioritizers.add(flowManager.createPrioritizer(className)); - } catch (final ClassNotFoundException | InstantiationException | IllegalAccessException e) { - throw new IllegalArgumentException("Unable to set prioritizer " + className + ": " + e); - } - } - } - - if (newPrioritizers != null) { - connection.getFlowFileQueue().setPriorities(newPrioritizers); - } - - if (dto.getBackPressureObjectThreshold() != null) { - connection.getFlowFileQueue().setBackPressureObjectThreshold(dto.getBackPressureObjectThreshold()); - } - - if (dto.getBackPressureDataSizeThreshold() != null && !dto.getBackPressureDataSizeThreshold().trim().isEmpty()) { - connection.getFlowFileQueue().setBackPressureDataSizeThreshold(dto.getBackPressureDataSizeThreshold()); - } - - if (dto.getFlowFileExpiration() != null) { - connection.getFlowFileQueue().setFlowFileExpiration(dto.getFlowFileExpiration()); - } - } - - return processGroup; - } - - /** - * Updates the process group corresponding to the specified DTO. Any field - * in DTO that is null (with the exception of the required ID) - * will be ignored, or in the case of back pressure settings, will obtain - * value from the parent of this process group - * - * @throws IllegalStateException if no process group can be found with the - * ID of DTO or with the ID of the DTO's parentGroupId, or if the DTO's Parent Group ID changes but the - * parent group has incoming or outgoing connections - * - * @throws NullPointerException if the DTO or its ID is null - */ - private void updateProcessGroup(final ProcessGroup group, final ProcessGroupDTO dto, final ParameterContextManager parameterContextManager) { - final String name = dto.getName(); - final PositionDTO position = dto.getPosition(); - final String comments = dto.getComments(); - final String flowfileConcurrencyName = dto.getFlowfileConcurrency(); - final String flowfileOutboundPolicyName = dto.getFlowfileOutboundPolicy(); - final String defaultFlowFileExpiration = dto.getDefaultFlowFileExpiration(); - final Long defaultBackPressureObjectThreshold = dto.getDefaultBackPressureObjectThreshold(); - final String defaultBackPressureDataSizeThreshold = dto.getDefaultBackPressureDataSizeThreshold(); - final String logFileSuffix = dto.getLogFileSuffix(); - - if (name != null) { - group.setName(name); - } - if (position != null) { - group.setPosition(toPosition(position)); - } - if (comments != null) { - group.setComments(comments); - } - - if (flowfileConcurrencyName == null) { - group.setFlowFileConcurrency(FlowFileConcurrency.UNBOUNDED); - } else { - group.setFlowFileConcurrency(FlowFileConcurrency.valueOf(flowfileConcurrencyName)); - } - - if (flowfileOutboundPolicyName == null) { - group.setFlowFileOutboundPolicy(FlowFileOutboundPolicy.STREAM_WHEN_AVAILABLE); - } else { - group.setFlowFileOutboundPolicy(FlowFileOutboundPolicy.valueOf(flowfileOutboundPolicyName)); - } - - final ParameterContextReferenceEntity parameterContextReference = dto.getParameterContext(); - if (parameterContextReference != null && parameterContextReference.getId() != null) { - final String parameterContextId = parameterContextReference.getId(); - final ParameterContext parameterContext = parameterContextManager.getParameterContext(parameterContextId); - if (!Objects.equals(parameterContext, group.getParameterContext())) { - group.setParameterContext(parameterContext); - } - } - - if (defaultFlowFileExpiration != null) { - group.setDefaultFlowFileExpiration(defaultFlowFileExpiration); - } - if (defaultBackPressureObjectThreshold != null) { - group.setDefaultBackPressureObjectThreshold(defaultBackPressureObjectThreshold); - } - if (defaultBackPressureDataSizeThreshold != null) { - group.setDefaultBackPressureDataSizeThreshold(defaultBackPressureDataSizeThreshold); - } - - if (logFileSuffix != null) { - group.setLogFileSuffix(logFileSuffix); - } - } - - private ScheduledState getScheduledState(final T component, final FlowController flowController) { - final ScheduledState componentState = component.getScheduledState(); - if (componentState == ScheduledState.STOPPED) { - if (flowController.isStartAfterInitialization(component)) { - return ScheduledState.RUNNING; - } - } - - return componentState; - } - - private Position toPosition(final PositionDTO dto) { - return new Position(dto.getX(), dto.getY()); - } - - private void updateProcessor(final ProcessorNode procNode, final ProcessorDTO processorDTO, final ProcessGroup processGroup, final FlowController controller) { - - procNode.pauseValidationTrigger(); - try { - final ProcessorConfigDTO config = processorDTO.getConfig(); - procNode.setProcessGroup(processGroup); - procNode.setLossTolerant(config.isLossTolerant()); - procNode.setPenalizationPeriod(config.getPenaltyDuration()); - procNode.setYieldPeriod(config.getYieldDuration()); - procNode.setBulletinLevel(LogLevel.valueOf(config.getBulletinLevel())); - procNode.setRetryCount(config.getRetryCount()); - procNode.setRetriedRelationships(config.getRetriedRelationships()); - - if (config.getBackoffMechanism() != null) { - procNode.setBackoffMechanism(BackoffMechanism.valueOf(config.getBackoffMechanism())); - } - procNode.setMaxBackoffPeriod(config.getMaxBackoffPeriod()); - updateNonFingerprintedProcessorSettings(procNode, processorDTO); - - if (config.getSchedulingStrategy() != null) { - procNode.setSchedulingStrategy(SchedulingStrategy.valueOf(config.getSchedulingStrategy())); - } - - if (config.getExecutionNode() != null) { - procNode.setExecutionNode(ExecutionNode.valueOf(config.getExecutionNode())); - } - - // must set scheduling strategy before these two - procNode.setMaxConcurrentTasks(config.getConcurrentlySchedulableTaskCount()); - procNode.setSchedulingPeriod(config.getSchedulingPeriod()); - if (config.getRunDurationMillis() != null) { - procNode.setRunDuration(config.getRunDurationMillis(), TimeUnit.MILLISECONDS); - } - - procNode.setAnnotationData(config.getAnnotationData()); - - if (config.getAutoTerminatedRelationships() != null) { - final Set relationships = new HashSet<>(); - for (final String rel : config.getAutoTerminatedRelationships()) { - relationships.add(procNode.getRelationship(rel)); - } - procNode.setAutoTerminatedRelationships(relationships); - } - - final Set sensitiveDynamicPropertyNames = getSensitiveDynamicPropertyNames(config.getSensitiveDynamicPropertyNames(), procNode); - procNode.setProperties(config.getProperties(), false, sensitiveDynamicPropertyNames); - - final ScheduledState scheduledState = ScheduledState.valueOf(processorDTO.getState()); - if (ScheduledState.RUNNING.equals(scheduledState)) { - procNode.performValidation(); // ensure that processor has been validated - controller.startProcessor(processGroup.getIdentifier(), procNode.getIdentifier()); - } else if (ScheduledState.DISABLED.equals(scheduledState)) { - processGroup.disableProcessor(procNode); - } else if (ScheduledState.STOPPED.equals(scheduledState)) { - controller.stopProcessor(processGroup.getIdentifier(), procNode.getIdentifier()); - } - } finally { - procNode.resumeValidationTrigger(); - } - } - - private Set getSensitiveDynamicPropertyNames(final Set parsedSensitivePropertyNames, final ComponentNode componentNode) { - final Set sensitivePropertyNames = parsedSensitivePropertyNames == null ? Collections.emptySet() : parsedSensitivePropertyNames; - return sensitivePropertyNames.stream().filter( - propertyName -> { - final PropertyDescriptor propertyDescriptor = componentNode.getPropertyDescriptor(propertyName); - return propertyDescriptor.isDynamic(); - } - ).collect(Collectors.toSet()); - } - - private void updateNonFingerprintedProcessorSettings(final ProcessorNode procNode, final ProcessorDTO processorDTO) { - procNode.setName(processorDTO.getName()); - procNode.setPosition(toPosition(processorDTO.getPosition())); - procNode.setStyle(processorDTO.getStyle()); - procNode.setComments(processorDTO.getConfig().getComments()); - } - - private ProcessGroup addProcessGroup(final FlowController controller, final ProcessGroup parentGroup, final Element processGroupElement, - final FlowEncodingVersion encodingVersion) { - - // get the parent group ID - final String parentId = (parentGroup == null) ? null : parentGroup.getIdentifier(); - final FlowManager flowManager = controller.getFlowManager(); - final PropertyEncryptor encryptor = controller.getEncryptor(); - - // add the process group - final ProcessGroupDTO processGroupDTO = FlowFromDOMFactory.getProcessGroup(parentId, processGroupElement, encryptor, encodingVersion); - final ProcessGroup processGroup = flowManager.createProcessGroup(processGroupDTO.getId()); - processGroup.setComments(processGroupDTO.getComments()); - processGroup.setVersionedComponentId(processGroupDTO.getVersionedComponentId()); - processGroup.setPosition(toPosition(processGroupDTO.getPosition())); - processGroup.setName(processGroupDTO.getName()); - processGroup.setParent(parentGroup); - if (parentGroup == null) { - controller.setRootGroup(processGroup); - } else { - parentGroup.addProcessGroup(processGroup); - } - - final String flowfileConcurrencyName = processGroupDTO.getFlowfileConcurrency(); - final String flowfileOutboundPolicyName = processGroupDTO.getFlowfileOutboundPolicy(); - if (flowfileConcurrencyName == null) { - processGroup.setFlowFileConcurrency(FlowFileConcurrency.UNBOUNDED); - } else { - processGroup.setFlowFileConcurrency(FlowFileConcurrency.valueOf(flowfileConcurrencyName)); - } - - if (flowfileOutboundPolicyName == null) { - processGroup.setFlowFileOutboundPolicy(FlowFileOutboundPolicy.STREAM_WHEN_AVAILABLE); - } else { - processGroup.setFlowFileOutboundPolicy(FlowFileOutboundPolicy.valueOf(flowfileOutboundPolicyName)); - } - - processGroup.setDefaultFlowFileExpiration(processGroupDTO.getDefaultFlowFileExpiration()); - processGroup.setDefaultBackPressureObjectThreshold(processGroupDTO.getDefaultBackPressureObjectThreshold()); - processGroup.setDefaultBackPressureDataSizeThreshold(processGroupDTO.getDefaultBackPressureDataSizeThreshold()); - - processGroup.setLogFileSuffix(processGroupDTO.getLogFileSuffix()); - - final String parameterContextId = getString(processGroupElement, "parameterContextId"); - if (parameterContextId != null) { - final ParameterContext parameterContext = controller.getFlowManager().getParameterContextManager().getParameterContext(parameterContextId); - processGroup.setParameterContext(parameterContext); - } - - addVersionControlInfo(processGroup, processGroupDTO, controller); - addControllerServices(processGroupElement, processGroup, controller, encodingVersion); - addProcessors(processGroupElement, processGroup, controller, encodingVersion); - addInputPorts(processGroupElement, processGroup, controller); - addOutputPorts(processGroupElement, processGroup, controller); - addFunnels(processGroupElement, processGroup, controller); - addLabels(processGroupElement, processGroup, controller); - addNestedProcessGroups(processGroupElement, processGroup, controller, encodingVersion); - addRemoteProcessGroups(processGroupElement, processGroup, controller); - addConnections(processGroupElement, processGroup, controller); - - return processGroup; - } - - private void addNestedProcessGroups(final Element processGroupElement, final ProcessGroup processGroup, final FlowController flowController, final FlowEncodingVersion encodingVersion) { - final List nestedProcessGroupNodeList = getChildrenByTagName(processGroupElement, "processGroup"); - for (final Element nestedProcessGroupElement : nestedProcessGroupNodeList) { - addProcessGroup(flowController, processGroup, nestedProcessGroupElement, encodingVersion); - } - } - - private void addVersionControlInfo(final ProcessGroup processGroup, final ProcessGroupDTO processGroupDTO, final FlowController flowController) { - final VersionControlInformationDTO versionControlInfoDto = processGroupDTO.getVersionControlInformation(); - if (versionControlInfoDto != null) { - final FlowRegistryClientNode flowRegistry = flowController.getFlowManager().getFlowRegistryClient(versionControlInfoDto.getRegistryId()); - final String registryName = flowRegistry == null ? versionControlInfoDto.getRegistryId() : flowRegistry.getName(); - - versionControlInfoDto.setState(VersionedFlowState.SYNC_FAILURE.name()); - versionControlInfoDto.setStateExplanation("Process Group has not yet been synchronized with the Flow Registry"); - final StandardVersionControlInformation versionControlInformation = StandardVersionControlInformation.Builder.fromDto(versionControlInfoDto) - .registryName(registryName) - .build(); - - // pass empty map for the version control mapping because the VersionedComponentId has already been set on the components - processGroup.setVersionControlInformation(versionControlInformation, Collections.emptyMap()); - } - } - - private void addControllerServices(final Element processGroupElement, final ProcessGroup processGroup, final FlowController flowController, final FlowEncodingVersion encodingVersion) { - final List serviceNodeList = getChildrenByTagName(processGroupElement, "controllerService"); - final PropertyEncryptor encryptor = flowController.getEncryptor(); - if (!serviceNodeList.isEmpty()) { - final Map controllerServices = ControllerServiceLoader.loadControllerServices(serviceNodeList, flowController, processGroup, encryptor, encodingVersion); - ControllerServiceLoader.enableControllerServices(controllerServices, flowController, encryptor, autoResumeState, encodingVersion); - } - } - - private void addProcessors(final Element processGroupElement, final ProcessGroup processGroup, final FlowController flowController, final FlowEncodingVersion encodingVersion) { - final List processorNodeList = getChildrenByTagName(processGroupElement, "processor"); - final PropertyEncryptor encryptor = flowController.getEncryptor(); - for (final Element processorElement : processorNodeList) { - final ProcessorDTO processorDTO = FlowFromDOMFactory.getProcessor(processorElement, encryptor, encodingVersion); - - BundleCoordinate coordinate; - try { - coordinate = BundleUtils.getCompatibleBundle(extensionManager, processorDTO.getType(), processorDTO.getBundle()); - } catch (final IllegalStateException e) { - final BundleDTO bundleDTO = processorDTO.getBundle(); - if (bundleDTO == null) { - coordinate = BundleCoordinate.UNKNOWN_COORDINATE; - } else { - coordinate = new BundleCoordinate(bundleDTO.getGroup(), bundleDTO.getArtifact(), bundleDTO.getVersion()); - } - } - - final ProcessorNode procNode = flowController.getFlowManager().createProcessor(processorDTO.getType(), processorDTO.getId(), coordinate, false); - procNode.setVersionedComponentId(processorDTO.getVersionedComponentId()); - processGroup.addProcessor(procNode); - updateProcessor(procNode, processorDTO, processGroup, flowController); - } - } - - private void addInputPorts(final Element processGroupElement, final ProcessGroup processGroup, final FlowController flowController) { - final FlowManager flowManager = flowController.getFlowManager(); - final List inputPortNodeList = getChildrenByTagName(processGroupElement, "inputPort"); - for (final Element inputPortElement : inputPortNodeList) { - final PortDTO portDTO = FlowFromDOMFactory.getPort(inputPortElement); - - final Port port; - if (processGroup.isRootGroup() || Boolean.TRUE.equals(portDTO.getAllowRemoteAccess())) { - port = flowManager.createPublicInputPort(portDTO.getId(), portDTO.getName()); - } else { - port = flowManager.createLocalInputPort(portDTO.getId(), portDTO.getName()); - } - - port.setVersionedComponentId(portDTO.getVersionedComponentId()); - port.setPosition(toPosition(portDTO.getPosition())); - port.setComments(portDTO.getComments()); - port.setProcessGroup(processGroup); - - final Set userControls = portDTO.getUserAccessControl(); - if (userControls != null && !userControls.isEmpty()) { - if (!(port instanceof PublicPort)) { - throw new IllegalStateException("Attempting to add User Access Controls to " + port.getIdentifier() + ", but it is not a RootGroupPort"); - } - ((PublicPort) port).setUserAccessControl(userControls); - } - final Set groupControls = portDTO.getGroupAccessControl(); - if (groupControls != null && !groupControls.isEmpty()) { - if (!(port instanceof PublicPort)) { - throw new IllegalStateException("Attempting to add Group Access Controls to " + port.getIdentifier() + ", but it is not a RootGroupPort"); - } - ((PublicPort) port).setGroupAccessControl(groupControls); - } - - processGroup.addInputPort(port); - if (portDTO.getConcurrentlySchedulableTaskCount() != null) { - port.setMaxConcurrentTasks(portDTO.getConcurrentlySchedulableTaskCount()); - } - - final ScheduledState scheduledState = ScheduledState.valueOf(portDTO.getState()); - if (ScheduledState.RUNNING.equals(scheduledState)) { - flowController.startConnectable(port); - } else if (ScheduledState.DISABLED.equals(scheduledState)) { - processGroup.disableInputPort(port); - } - } - } - - private void addOutputPorts(final Element processGroupElement, final ProcessGroup processGroup, final FlowController flowController) { - final FlowManager flowManager = flowController.getFlowManager(); - final List outputPortNodeList = getChildrenByTagName(processGroupElement, "outputPort"); - for (final Element outputPortElement : outputPortNodeList) { - final PortDTO portDTO = FlowFromDOMFactory.getPort(outputPortElement); - - final Port port; - if (processGroup.isRootGroup() || Boolean.TRUE.equals(portDTO.getAllowRemoteAccess())) { - port = flowManager.createPublicOutputPort(portDTO.getId(), portDTO.getName()); - } else { - port = flowManager.createLocalOutputPort(portDTO.getId(), portDTO.getName()); - } - - port.setVersionedComponentId(portDTO.getVersionedComponentId()); - port.setPosition(toPosition(portDTO.getPosition())); - port.setComments(portDTO.getComments()); - port.setProcessGroup(processGroup); - - final Set userControls = portDTO.getUserAccessControl(); - if (userControls != null && !userControls.isEmpty()) { - if (!(port instanceof PublicPort)) { - throw new IllegalStateException("Attempting to add User Access Controls to " + port.getIdentifier() + ", but it is not a RootGroupPort"); - } - ((PublicPort) port).setUserAccessControl(userControls); - } - final Set groupControls = portDTO.getGroupAccessControl(); - if (groupControls != null && !groupControls.isEmpty()) { - if (!(port instanceof PublicPort)) { - throw new IllegalStateException("Attempting to add Group Access Controls to " + port.getIdentifier() + ", but it is not a RootGroupPort"); - } - ((PublicPort) port).setGroupAccessControl(groupControls); - } - - processGroup.addOutputPort(port); - if (portDTO.getConcurrentlySchedulableTaskCount() != null) { - port.setMaxConcurrentTasks(portDTO.getConcurrentlySchedulableTaskCount()); - } - - final ScheduledState scheduledState = ScheduledState.valueOf(portDTO.getState()); - if (ScheduledState.RUNNING.equals(scheduledState)) { - flowController.startConnectable(port); - } else if (ScheduledState.DISABLED.equals(scheduledState)) { - processGroup.disableOutputPort(port); - } - } - } - - private void addFunnels(final Element processGroupElement, final ProcessGroup processGroup, final FlowController controller) { - final List funnelNodeList = getChildrenByTagName(processGroupElement, "funnel"); - for (final Element funnelElement : funnelNodeList) { - final FunnelDTO funnelDTO = FlowFromDOMFactory.getFunnel(funnelElement); - final Funnel funnel = controller.getFlowManager().createFunnel(funnelDTO.getId()); - funnel.setVersionedComponentId(funnelDTO.getVersionedComponentId()); - funnel.setPosition(toPosition(funnelDTO.getPosition())); - - // Since this is called during startup, we want to add the funnel without enabling it - // and then tell the controller to enable it. This way, if the controller is not fully - // initialized, the starting of the funnel is delayed until the controller is ready. - processGroup.addFunnel(funnel, false); - controller.startConnectable(funnel); - } - } - - private void addLabels(final Element processGroupElement, final ProcessGroup processGroup, final FlowController controller) { - final List labelNodeList = getChildrenByTagName(processGroupElement, "label"); - for (final Element labelElement : labelNodeList) { - final LabelDTO labelDTO = FlowFromDOMFactory.getLabel(labelElement); - final Label label = controller.getFlowManager().createLabel(labelDTO.getId(), labelDTO.getLabel()); - label.setVersionedComponentId(labelDTO.getVersionedComponentId()); - label.setStyle(labelDTO.getStyle()); - - label.setPosition(toPosition(labelDTO.getPosition())); - label.setSize(new Size(labelDTO.getWidth(), labelDTO.getHeight())); - final Long zIndex = labelDTO.getzIndex(); - if (zIndex != null) { - label.setZIndex(zIndex); - } - processGroup.addLabel(label); - } - } - - private void addRemoteProcessGroups(final Element processGroupElement, final ProcessGroup processGroup, final FlowController controller) { - final List remoteProcessGroupNodeList = getChildrenByTagName(processGroupElement, "remoteProcessGroup"); - final PropertyEncryptor encryptor = controller.getEncryptor(); - for (final Element remoteProcessGroupElement : remoteProcessGroupNodeList) { - final RemoteProcessGroupDTO remoteGroupDto = FlowFromDOMFactory.getRemoteProcessGroup(remoteProcessGroupElement, encryptor); - final RemoteProcessGroup remoteGroup = controller.getFlowManager().createRemoteProcessGroup(remoteGroupDto.getId(), remoteGroupDto.getTargetUris()); - remoteGroup.setVersionedComponentId(remoteGroupDto.getVersionedComponentId()); - remoteGroup.setComments(remoteGroupDto.getComments()); - remoteGroup.setPosition(toPosition(remoteGroupDto.getPosition())); - final String name = remoteGroupDto.getName(); - if (name != null && !name.trim().isEmpty()) { - remoteGroup.setName(name); - } - remoteGroup.setProcessGroup(processGroup); - remoteGroup.setCommunicationsTimeout(remoteGroupDto.getCommunicationsTimeout()); - - if (remoteGroupDto.getYieldDuration() != null) { - remoteGroup.setYieldDuration(remoteGroupDto.getYieldDuration()); - } - - final String transportProtocol = remoteGroupDto.getTransportProtocol(); - if (transportProtocol != null && !transportProtocol.trim().isEmpty()) { - remoteGroup.setTransportProtocol(SiteToSiteTransportProtocol.valueOf(transportProtocol.toUpperCase())); - } - - if (remoteGroupDto.getProxyHost() != null) { - remoteGroup.setProxyHost(remoteGroupDto.getProxyHost()); - } - - if (remoteGroupDto.getProxyPort() != null) { - remoteGroup.setProxyPort(remoteGroupDto.getProxyPort()); - } - - if (remoteGroupDto.getProxyUser() != null) { - remoteGroup.setProxyUser(remoteGroupDto.getProxyUser()); - } - - if (remoteGroupDto.getProxyPassword() != null) { - remoteGroup.setProxyPassword(remoteGroupDto.getProxyPassword()); - } - - if (StringUtils.isBlank(remoteGroupDto.getLocalNetworkInterface())) { - remoteGroup.setNetworkInterface(null); - } else { - remoteGroup.setNetworkInterface(remoteGroupDto.getLocalNetworkInterface()); - } - - final Set inputPorts = new HashSet<>(); - for (final Element portElement : getChildrenByTagName(remoteProcessGroupElement, "inputPort")) { - inputPorts.add(FlowFromDOMFactory.getRemoteProcessGroupPort(portElement)); - } - remoteGroup.setInputPorts(inputPorts, false); - - final Set outputPorts = new HashSet<>(); - for (final Element portElement : getChildrenByTagName(remoteProcessGroupElement, "outputPort")) { - outputPorts.add(FlowFromDOMFactory.getRemoteProcessGroupPort(portElement)); - } - remoteGroup.setOutputPorts(outputPorts, false); - processGroup.addRemoteProcessGroup(remoteGroup); - - for (final RemoteProcessGroupPortDescriptor remoteGroupPortDTO : outputPorts) { - final RemoteGroupPort port = remoteGroup.getOutputPort(remoteGroupPortDTO.getId()); - if (Boolean.TRUE.equals(remoteGroupPortDTO.isTransmitting())) { - controller.startTransmitting(port); - } - } - for (final RemoteProcessGroupPortDescriptor remoteGroupPortDTO : inputPorts) { - final RemoteGroupPort port = remoteGroup.getInputPort(remoteGroupPortDTO.getId()); - if (Boolean.TRUE.equals(remoteGroupPortDTO.isTransmitting())) { - controller.startTransmitting(port); - } - } - } - } - - private void addConnections(final Element processGroupElement, final ProcessGroup processGroup, final FlowController controller) { - final FlowManager flowManager = controller.getFlowManager(); - - final List connectionNodeList = getChildrenByTagName(processGroupElement, "connection"); - for (final Element connectionElement : connectionNodeList) { - final ConnectionDTO dto = FlowFromDOMFactory.getConnection(connectionElement); - - final Connectable source; - final ConnectableDTO sourceDto = dto.getSource(); - if (ConnectableType.REMOTE_OUTPUT_PORT.name().equals(sourceDto.getType())) { - final RemoteProcessGroup remoteGroup = processGroup.getRemoteProcessGroup(sourceDto.getGroupId()); - source = remoteGroup.getOutputPort(sourceDto.getId()); - } else { - final ProcessGroup sourceGroup = flowManager.getGroup(sourceDto.getGroupId()); - if (sourceGroup == null) { - throw new RuntimeException("Found Invalid ProcessGroup ID for Source: " + dto.getSource().getGroupId()); - } - - source = sourceGroup.getConnectable(sourceDto.getId()); - } - if (source == null) { - throw new RuntimeException("Found Invalid Connectable ID for Source: " + dto.getSource().getId()); - } - - final Connectable destination; - final ConnectableDTO destinationDto = dto.getDestination(); - if (ConnectableType.REMOTE_INPUT_PORT.name().equals(destinationDto.getType())) { - final RemoteProcessGroup remoteGroup = processGroup.getRemoteProcessGroup(destinationDto.getGroupId()); - destination = remoteGroup.getInputPort(destinationDto.getId()); - } else { - final ProcessGroup destinationGroup = flowManager.getGroup(destinationDto.getGroupId()); - if (destinationGroup == null) { - throw new RuntimeException("Found Invalid ProcessGroup ID for Destination: " + dto.getDestination().getGroupId()); - } - - destination = destinationGroup.getConnectable(destinationDto.getId()); - } - if (destination == null) { - throw new RuntimeException("Found Invalid Connectable ID for Destination: " + dto.getDestination().getId()); - } - - final Connection connection = flowManager.createConnection(dto.getId(), dto.getName(), source, destination, dto.getSelectedRelationships()); - connection.setVersionedComponentId(dto.getVersionedComponentId()); - connection.setProcessGroup(processGroup); - - final List bendPoints = new ArrayList<>(); - for (final PositionDTO bend : dto.getBends()) { - bendPoints.add(new Position(bend.getX(), bend.getY())); - } - connection.setBendPoints(bendPoints); - - final Long zIndex = dto.getzIndex(); - if (zIndex != null) { - connection.setZIndex(zIndex); - } - - if (dto.getLabelIndex() != null) { - connection.setLabelIndex(dto.getLabelIndex()); - } - - List newPrioritizers = null; - final List prioritizers = dto.getPrioritizers(); - if (prioritizers != null) { - final List newPrioritizersClasses = new ArrayList<>(prioritizers); - newPrioritizers = new ArrayList<>(); - for (final String className : newPrioritizersClasses) { - try { - newPrioritizers.add(flowManager.createPrioritizer(className)); - } catch (final ClassNotFoundException | InstantiationException | IllegalAccessException e) { - throw new IllegalArgumentException("Unable to set prioritizer " + className + ": " + e); - } - } - } - if (newPrioritizers != null) { - connection.getFlowFileQueue().setPriorities(newPrioritizers); - } - - if (dto.getBackPressureObjectThreshold() != null) { - connection.getFlowFileQueue().setBackPressureObjectThreshold(dto.getBackPressureObjectThreshold()); - } - if (dto.getBackPressureDataSizeThreshold() != null) { - connection.getFlowFileQueue().setBackPressureDataSizeThreshold(dto.getBackPressureDataSizeThreshold()); - } - if (dto.getFlowFileExpiration() != null) { - connection.getFlowFileQueue().setFlowFileExpiration(dto.getFlowFileExpiration()); - } - - if (dto.getLoadBalanceStrategy() != null) { - connection.getFlowFileQueue().setLoadBalanceStrategy(LoadBalanceStrategy.valueOf(dto.getLoadBalanceStrategy()), dto.getLoadBalancePartitionAttribute()); - } - - if (dto.getLoadBalanceCompression() != null) { - connection.getFlowFileQueue().setLoadBalanceCompression(LoadBalanceCompression.valueOf(dto.getLoadBalanceCompression())); - } - - processGroup.addConnection(connection); - } - } - - - private byte[] toBytes(final FlowController flowController) throws FlowSerializationException { - final ByteArrayOutputStream result = new ByteArrayOutputStream(); - final StandardFlowSerializer flowSerializer = new StandardFlowSerializer(); - flowController.serialize(flowSerializer, result); - return result.toByteArray(); - } - - private static String getString(final Element element, final String childElementName) { - final List nodeList = getChildrenByTagName(element, childElementName); - if (nodeList == null || nodeList.isEmpty()) { - return ""; - } - final Element childElement = nodeList.get(0); - return childElement.getTextContent(); - } - - private static int getInt(final Element element, final String childElementName) { - return Integer.parseInt(getString(element, childElementName)); - } - - private static Integer getInteger(final Element element, final String childElementName) { - final String value = getString(element, childElementName); - return (value == null || value.trim().equals("") ? null : Integer.parseInt(value)); - } - - private static List getChildrenByTagName(final Element element, final String tagName) { - final List matches = new ArrayList<>(); - final NodeList nodeList = element.getChildNodes(); - for (int i = 0; i < nodeList.getLength(); i++) { - final Node node = nodeList.item(i); - if (!(node instanceof Element)) { - continue; - } - - final Element child = (Element) nodeList.item(i); - if (child.getNodeName().equals(tagName)) { - matches.add(child); - } - } - - return matches; - } -} diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/inheritance/BundleCompatibilityCheck.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/inheritance/BundleCompatibilityCheck.java index c0163141b0..077bb95d1b 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/inheritance/BundleCompatibilityCheck.java +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/inheritance/BundleCompatibilityCheck.java @@ -40,14 +40,6 @@ public class BundleCompatibilityCheck implements FlowInheritabilityCheck { @Override public FlowInheritability checkInheritability(final DataFlow existingFlow, final DataFlow proposedFlow, final FlowController flowController) { - if (proposedFlow.isXml()) { - return checkInheritability(proposedFlow.getFlowDocument(), flowController); - } else { - return checkVersionedFlowInheritability(proposedFlow, flowController); - } - } - - private FlowInheritability checkVersionedFlowInheritability(final DataFlow proposedFlow, final FlowController flowController) { return checkBundles(proposedFlow, flowController.getExtensionManager()); } diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/inheritance/ConnectionMissingCheck.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/inheritance/ConnectionMissingCheck.java index 6832a03f78..5465b36eff 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/inheritance/ConnectionMissingCheck.java +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/inheritance/ConnectionMissingCheck.java @@ -22,24 +22,17 @@ import org.apache.nifi.controller.FlowController; import org.apache.nifi.controller.flow.FlowManager; import org.apache.nifi.controller.flow.VersionedDataflow; import org.apache.nifi.controller.repository.FlowFileRepository; -import org.apache.nifi.controller.serialization.FlowEncodingVersion; -import org.apache.nifi.controller.serialization.FlowFromDOMFactory; import org.apache.nifi.controller.serialization.FlowSynchronizationException; import org.apache.nifi.flow.ComponentType; import org.apache.nifi.flow.VersionedComponent; import org.apache.nifi.registry.flow.diff.DifferenceType; import org.apache.nifi.registry.flow.diff.FlowComparison; import org.apache.nifi.registry.flow.diff.FlowDifference; -import org.apache.nifi.web.api.dto.ConnectionDTO; -import org.apache.nifi.web.api.dto.ProcessGroupDTO; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import org.w3c.dom.Document; -import org.w3c.dom.Element; import java.io.IOException; import java.util.Collections; -import java.util.HashSet; import java.util.Set; /** @@ -57,11 +50,7 @@ public class ConnectionMissingCheck implements FlowInheritabilityCheck { @Override public FlowInheritability checkInheritability(final DataFlow existingFlow, final DataFlow proposedFlow, final FlowController flowController) { - if (proposedFlow.isXml()) { - return checkInheritability(proposedFlow.getFlowDocument(), flowController); - } else { - return checkInheritability(existingFlow.getVersionedDataflow(), proposedFlow.getVersionedDataflow(), flowController); - } + return checkInheritability(existingFlow.getVersionedDataflow(), proposedFlow.getVersionedDataflow(), flowController); } private FlowInheritability checkInheritability(final VersionedDataflow existingFlow, final VersionedDataflow proposedFlow, final FlowController flowController) { @@ -111,47 +100,4 @@ public class ConnectionMissingCheck implements FlowInheritabilityCheck { return FlowInheritability.inheritable(); } - - private FlowInheritability checkInheritability(final Document flowDocument, final FlowController flowController) { - final Element rootGroupElement = (Element) flowDocument.getDocumentElement().getElementsByTagName("rootGroup").item(0); - final FlowEncodingVersion encodingVersion = FlowEncodingVersion.parse(flowDocument.getDocumentElement()); - - final ProcessGroupDTO rootGroupDto = FlowFromDOMFactory.getProcessGroup(null, rootGroupElement, null, encodingVersion); - final Set connectionIds = findAllConnectionIds(rootGroupDto); - - final FlowFileRepository flowFileRepository = flowController.getRepositoryContextFactory().getFlowFileRepository(); - - final Set queuesWithFlowFiles; - try { - queuesWithFlowFiles = flowFileRepository.findQueuesWithFlowFiles(flowController.createSwapManager()); - } catch (final IOException ioe) { - throw new FlowSynchronizationException("Failed to determine which connections have FlowFiles queued", ioe); - } - - logger.debug("The following {} Connections/Queues have data queued up currently: {}", queuesWithFlowFiles.size(), queuesWithFlowFiles); - - for (final String queueId : queuesWithFlowFiles) { - if (!connectionIds.contains(queueId)) { - return FlowInheritability.notInheritable("Proposed Flow does not contain a Connection with ID " + queueId + " but this instance has data queued in that connection"); - } - } - - return FlowInheritability.inheritable(); - } - - private Set findAllConnectionIds(final ProcessGroupDTO group) { - final Set connectionIds = new HashSet<>(); - findAllConnectionIds(group, connectionIds); - return connectionIds; - } - - private void findAllConnectionIds(final ProcessGroupDTO group, final Set ids) { - for (final ConnectionDTO connectionDTO : group.getContents().getConnections()) { - ids.add(connectionDTO.getId()); - } - - for (final ProcessGroupDTO childGroup : group.getContents().getProcessGroups()) { - findAllConnectionIds(childGroup, ids); - } - } } diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/inheritance/FlowFingerprintCheck.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/inheritance/FlowFingerprintCheck.java deleted file mode 100644 index 079786d2e0..0000000000 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/inheritance/FlowFingerprintCheck.java +++ /dev/null @@ -1,99 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.nifi.controller.inheritance; - -import org.apache.nifi.cluster.protocol.DataFlow; -import org.apache.nifi.controller.FlowController; -import org.apache.nifi.encrypt.PropertyEncryptor; -import org.apache.nifi.encrypt.SensitiveValueEncoder; -import org.apache.nifi.fingerprint.FingerprintFactory; -import org.apache.nifi.nar.ExtensionManager; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -public class FlowFingerprintCheck implements FlowInheritabilityCheck { - private static final Logger logger = LoggerFactory.getLogger(FlowFingerprintCheck.class); - - @Override - public FlowInheritability checkInheritability(final DataFlow existingFlow, final DataFlow proposedFlow, final FlowController flowController) { - if (existingFlow == null) { - return FlowInheritability.inheritable(); - } - - final byte[] existingFlowBytes = existingFlow.getFlow(); - final byte[] proposedFlowBytes = proposedFlow.getFlow(); - - final PropertyEncryptor encryptor = flowController.getEncryptor(); - final ExtensionManager extensionManager = flowController.getExtensionManager(); - final SensitiveValueEncoder sensitiveValueEncoder = flowController.getSensitiveValueEncoder(); - - final FingerprintFactory fingerprintFactory = new FingerprintFactory(encryptor, extensionManager, sensitiveValueEncoder); - final String existingFlowFingerprintBeforeHash = fingerprintFactory.createFingerprint(existingFlowBytes, flowController); - if (existingFlowFingerprintBeforeHash.trim().isEmpty()) { - return null; // no existing flow, so equivalent to proposed flow - } - - if (proposedFlow == null || proposedFlowBytes.length == 0) { - return FlowInheritability.notInheritable("Proposed Flow was empty but Current Flow is not"); // existing flow is not empty and proposed flow is empty (we could orphan flowfiles) - } - - final String proposedFlowFingerprintBeforeHash = fingerprintFactory.createFingerprint(proposedFlow.getFlowDocument(), flowController); - if (proposedFlowFingerprintBeforeHash.trim().isEmpty()) { - return FlowInheritability.notInheritable("Proposed Flow was empty but Current Flow is not"); // existing flow is not empty and proposed flow is empty (we could orphan flowfiles) - } - - if (logger.isTraceEnabled()) { - logger.trace("Local Fingerprint Before Hash = {}", new Object[] {existingFlowFingerprintBeforeHash}); - logger.trace("Proposed Fingerprint Before Hash = {}", new Object[] {proposedFlowFingerprintBeforeHash}); - } - - final boolean inheritable = existingFlowFingerprintBeforeHash.equals(proposedFlowFingerprintBeforeHash); - if (!inheritable) { - final String discrepancy = findFirstDiscrepancy(existingFlowFingerprintBeforeHash, proposedFlowFingerprintBeforeHash, "Flows"); - return FlowInheritability.notInheritable(discrepancy); - } - - return FlowInheritability.inheritable(); - } - - private String findFirstDiscrepancy(final String existing, final String proposed, final String comparisonDescription) { - final int shortestFileLength = Math.min(existing.length(), proposed.length()); - for (int i = 0; i < shortestFileLength; i++) { - if (existing.charAt(i) != proposed.charAt(i)) { - final String formattedExistingDelta = formatFlowDiscrepancy(existing, i, 100); - final String formattedProposedDelta = formatFlowDiscrepancy(proposed, i, 100); - return String.format("Found difference in %s:\nLocal Fingerprint: %s\nCluster Fingerprint: %s", comparisonDescription, formattedExistingDelta, formattedProposedDelta); - } - } - - // existing must startWith proposed or proposed must startWith existing - if (existing.length() > proposed.length()) { - final String formattedExistingDelta = existing.substring(proposed.length(), Math.min(existing.length(), proposed.length() + 200)); - return String.format("Found difference in %s:\nLocal Fingerprint contains additional configuration from Cluster Fingerprint: %s", comparisonDescription, formattedExistingDelta); - } else if (proposed.length() > existing.length()) { - final String formattedProposedDelta = proposed.substring(existing.length(), Math.min(proposed.length(), existing.length() + 200)); - return String.format("Found difference in %s:\nCluster Fingerprint contains additional configuration from Local Fingerprint: %s", comparisonDescription, formattedProposedDelta); - } - - return "Unable to find any discrepancies between fingerprints. Please contact the NiFi support team"; - } - - private String formatFlowDiscrepancy(final String flowFingerprint, final int deltaIndex, final int deltaPad) { - return flowFingerprint.substring(Math.max(0, deltaIndex - deltaPad), Math.min(flowFingerprint.length(), deltaIndex + deltaPad)); - } - -} diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/serialization/FlowFromDOMFactory.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/serialization/FlowFromDOMFactory.java deleted file mode 100644 index 67dfc1d5ac..0000000000 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/serialization/FlowFromDOMFactory.java +++ /dev/null @@ -1,783 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.nifi.controller.serialization; - -import org.apache.nifi.connectable.Size; -import org.apache.nifi.controller.ScheduledState; -import org.apache.nifi.controller.service.ControllerServiceState; -import org.apache.nifi.encrypt.EncryptionException; -import org.apache.nifi.encrypt.PropertyEncryptor; -import org.apache.nifi.groups.RemoteProcessGroupPortDescriptor; -import org.apache.nifi.parameter.ExpressionLanguageAwareParameterParser; -import org.apache.nifi.parameter.ParameterParser; -import org.apache.nifi.parameter.ParameterTokenList; -import org.apache.nifi.remote.StandardRemoteProcessGroupPortDescriptor; -import org.apache.nifi.scheduling.ExecutionNode; -import org.apache.nifi.scheduling.SchedulingStrategy; -import org.apache.nifi.util.DomUtils; -import org.apache.nifi.web.api.dto.BundleDTO; -import org.apache.nifi.web.api.dto.ConnectableDTO; -import org.apache.nifi.web.api.dto.ConnectionDTO; -import org.apache.nifi.web.api.dto.ControllerServiceDTO; -import org.apache.nifi.web.api.dto.FlowAnalysisRuleDTO; -import org.apache.nifi.web.api.dto.FlowRegistryClientDTO; -import org.apache.nifi.web.api.dto.FlowSnippetDTO; -import org.apache.nifi.web.api.dto.FunnelDTO; -import org.apache.nifi.web.api.dto.LabelDTO; -import org.apache.nifi.web.api.dto.ParameterContextDTO; -import org.apache.nifi.web.api.dto.ParameterDTO; -import org.apache.nifi.web.api.dto.ParameterProviderConfigurationDTO; -import org.apache.nifi.web.api.dto.ParameterProviderDTO; -import org.apache.nifi.web.api.dto.PortDTO; -import org.apache.nifi.web.api.dto.PositionDTO; -import org.apache.nifi.web.api.dto.ProcessGroupDTO; -import org.apache.nifi.web.api.dto.ProcessorConfigDTO; -import org.apache.nifi.web.api.dto.ProcessorDTO; -import org.apache.nifi.web.api.dto.RemoteProcessGroupDTO; -import org.apache.nifi.web.api.dto.ReportingTaskDTO; -import org.apache.nifi.web.api.dto.VersionControlInformationDTO; -import org.apache.nifi.web.api.entity.ParameterContextReferenceEntity; -import org.apache.nifi.web.api.entity.ParameterEntity; -import org.apache.nifi.web.api.entity.ParameterProviderConfigurationEntity; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.w3c.dom.Element; -import org.w3c.dom.NodeList; - -import java.util.ArrayList; -import java.util.Collections; -import java.util.HashMap; -import java.util.HashSet; -import java.util.LinkedHashMap; -import java.util.LinkedHashSet; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.concurrent.TimeUnit; - -public class FlowFromDOMFactory { - private static final Logger logger = LoggerFactory.getLogger(FlowFromDOMFactory.class); - private static final String DEPRECATED_FLOW_REGISTRY_CLIENT_TYPE = "org.apache.nifi.registry.flow.NifiRegistryFlowRegistryClient"; - - public static BundleDTO getBundle(final Element bundleElement) { - if (bundleElement == null) { - return null; - } - - final Element groupElement = DomUtils.getChild(bundleElement, "group"); - final Element artifactElement = DomUtils.getChild(bundleElement, "artifact"); - final Element versionElement = DomUtils.getChild(bundleElement, "version"); - - return new BundleDTO(groupElement.getTextContent(), artifactElement.getTextContent(), versionElement.getTextContent()); - } - - public static PositionDTO getPosition(final Element positionElement) { - if (positionElement == null) { - throw new IllegalArgumentException("Invalid Flow: Found no 'position' element"); - } - return new PositionDTO(Double.parseDouble(positionElement.getAttribute("x")), Double.parseDouble(positionElement.getAttribute("y"))); - } - - public static Size getSize(final Element sizeElement) { - if (sizeElement == null) { - throw new IllegalArgumentException("Invalid Flow: Found no 'size' element"); - } - - return new Size(Double.parseDouble(sizeElement.getAttribute("width")), Double.parseDouble(sizeElement.getAttribute("height"))); - } - - public static Map getStyle(final Element stylesElement) { - final Map styles = new HashMap<>(); - if (stylesElement == null) { - return styles; - } - - for (final Element styleElement : getChildrenByTagName(stylesElement, "style")) { - final String styleName = styleElement.getAttribute("name"); - final String styleValue = styleElement.getTextContent(); - styles.put(styleName, styleValue); - } - - return styles; - } - - public static ControllerServiceDTO getControllerService(final Element element, final PropertyEncryptor encryptor, final FlowEncodingVersion flowEncodingVersion) { - final ControllerServiceDTO dto = new ControllerServiceDTO(); - - dto.setId(getString(element, "id")); - dto.setVersionedComponentId(getString(element, "versionedComponentId")); - dto.setName(getString(element, "name")); - dto.setComments(getString(element, "comment")); - dto.setBulletinLevel(getString(element, "bulletinLevel")); - dto.setType(getString(element, "class")); - dto.setBundle(getBundle(DomUtils.getChild(element, "bundle"))); - - final boolean enabled = getBoolean(element, "enabled"); - dto.setState(enabled ? ControllerServiceState.ENABLED.name() : ControllerServiceState.DISABLED.name()); - - dto.setSensitiveDynamicPropertyNames(getSensitivePropertyNames(element)); - dto.setProperties(getProperties(element, encryptor, flowEncodingVersion)); - dto.setAnnotationData(getString(element, "annotationData")); - - return dto; - } - - public static ReportingTaskDTO getReportingTask(final Element element, final PropertyEncryptor encryptor, final FlowEncodingVersion flowEncodingVersion) { - final ReportingTaskDTO dto = new ReportingTaskDTO(); - - dto.setId(getString(element, "id")); - dto.setName(getString(element, "name")); - dto.setComments(getString(element, "comment")); - dto.setType(getString(element, "class")); - dto.setBundle(getBundle(DomUtils.getChild(element, "bundle"))); - dto.setSchedulingPeriod(getString(element, "schedulingPeriod")); - dto.setState(getString(element, "scheduledState")); - dto.setSchedulingStrategy(getString(element, "schedulingStrategy")); - - dto.setSensitiveDynamicPropertyNames(getSensitivePropertyNames(element)); - dto.setProperties(getProperties(element, encryptor, flowEncodingVersion)); - dto.setAnnotationData(getString(element, "annotationData")); - - return dto; - } - - public static FlowAnalysisRuleDTO getFlowAnalysisRule(Element element, PropertyEncryptor encryptor, FlowEncodingVersion flowEncodingVersion) { - final FlowAnalysisRuleDTO dto = new FlowAnalysisRuleDTO(); - - dto.setId(getString(element, "id")); - dto.setName(getString(element, "name")); - dto.setComments(getString(element, "comment")); - dto.setType(getString(element, "class")); - dto.setBundle(getBundle(DomUtils.getChild(element, "bundle"))); - - dto.setEnforcementPolicy(getString(element, "enforcementPolicy")); - dto.setState(getString(element, "state")); - - dto.setSensitiveDynamicPropertyNames(getSensitivePropertyNames(element)); - dto.setProperties(getProperties(element, encryptor, flowEncodingVersion)); - - return dto; - } - - public static FlowRegistryClientDTO getFlowRegistryClient(final Element element, final PropertyEncryptor encryptor, final FlowEncodingVersion flowEncodingVersion) { - final FlowRegistryClientDTO dto = new FlowRegistryClientDTO(); - - if (isOldStyleRegistryClient(element)) { - return getFlowRegistryClientFromOldStyleConfig(element); - } - - dto.setId(getString(element, "id")); - dto.setName(getString(element, "name")); - dto.setDescription(getString(element, "description")); - dto.setUri(getString(element, "uri")); - - dto.setType(getString(element, "class")); - dto.setBundle(getBundle(DomUtils.getChild(element, "bundle"))); - - dto.setSensitiveDynamicPropertyNames(getSensitivePropertyNames(element)); - dto.setProperties(getProperties(element, encryptor, flowEncodingVersion)); - dto.setAnnotationData(getString(element, "annotationData")); - - return dto; - } - - private static FlowRegistryClientDTO getFlowRegistryClientFromOldStyleConfig(final Element element) { - final String id = getString(element, "id"); - final String name = getString(element, "name"); - final String url = getString(element, "url"); - final String description = getString(element, "description"); - - final FlowRegistryClientDTO dto = new FlowRegistryClientDTO(); - dto.setId(id); - dto.setName(name); - dto.setDescription(description); - dto.setUri(url); - - dto.setType(DEPRECATED_FLOW_REGISTRY_CLIENT_TYPE); - dto.setBundle(new BundleDTO("org.apache.nifi", "nifi-flow-registry-client-nar", "1.18.0")); - - dto.setSensitiveDynamicPropertyNames(Collections.emptySet()); - dto.setProperties(Collections.singletonMap("url", url)); - dto.setAnnotationData(null); - - return dto; - } - - private static boolean isOldStyleRegistryClient(final Element element) { - final String id = getString(element, "id"); - final String identifier = getString(element, "identifier"); - final List bundleElements = getChildrenByTagName(element, "bundle"); - - return id != null && identifier == null && bundleElements.isEmpty(); - } - - public static ParameterProviderDTO getParameterProvider(final Element element, final PropertyEncryptor encryptor, final FlowEncodingVersion flowEncodingVersion) { - final ParameterProviderDTO dto = new ParameterProviderDTO(); - - dto.setId(getString(element, "id")); - dto.setName(getString(element, "name")); - dto.setComments(getString(element, "comment")); - dto.setType(getString(element, "class")); - dto.setBundle(getBundle(DomUtils.getChild(element, "bundle"))); - - dto.setProperties(getProperties(element, encryptor, flowEncodingVersion)); - dto.setAnnotationData(getString(element, "annotationData")); - - return dto; - } - - public static ParameterContextDTO getParameterContext(final Element element, final PropertyEncryptor encryptor) { - final ParameterContextDTO dto = new ParameterContextDTO(); - - dto.setId(getString(element, "id")); - dto.setName(getString(element, "name")); - dto.setDescription(getString(element, "description")); - - final Set parameterDtos = new LinkedHashSet<>(); - final List parameterElements = FlowFromDOMFactory.getChildrenByTagName(element, "parameter"); - for (final Element parameterElement : parameterElements) { - final ParameterDTO parameterDto = new ParameterDTO(); - - parameterDto.setName(getString(parameterElement, "name")); - parameterDto.setDescription(getString(parameterElement, "description")); - parameterDto.setSensitive(getBoolean(parameterElement, "sensitive")); - parameterDto.setProvided(getBoolean(parameterElement, "provided")); - - final String value = decrypt(getString(parameterElement, "value"), encryptor); - parameterDto.setValue(value); - - final ParameterEntity parameterEntity = new ParameterEntity(); - parameterEntity.setParameter(parameterDto); - parameterDtos.add(parameterEntity); - } - final List inheritedParameterContextIds = FlowFromDOMFactory.getChildrenByTagName(element, "inheritedParameterContextId"); - final List parameterContexts = new ArrayList<>(); - for (final Element inheritedParameterContextElement : inheritedParameterContextIds) { - final ParameterContextReferenceEntity parameterContextReference = new ParameterContextReferenceEntity(); - parameterContextReference.setId(inheritedParameterContextElement.getTextContent()); - parameterContexts.add(parameterContextReference); - } - dto.setInheritedParameterContexts(parameterContexts); - - final ParameterProviderConfigurationEntity parameterProviderConfiguration = getParameterProviderConfiguration(element); - if (parameterProviderConfiguration != null) { - dto.setParameterProviderConfiguration(parameterProviderConfiguration); - } - - dto.setParameters(parameterDtos); - - return dto; - } - - private static ParameterProviderConfigurationEntity getParameterProviderConfiguration(final Element parameterContextElement) { - final String parameterProviderId = getString(parameterContextElement, "parameterProviderId"); - if (parameterProviderId != null) { - final ParameterProviderConfigurationEntity entity = new ParameterProviderConfigurationEntity(); - entity.setId(parameterProviderId); - final ParameterProviderConfigurationDTO dto = new ParameterProviderConfigurationDTO(); - final String parameterGroupName = getString(parameterContextElement, "parameterGroupName"); - final Boolean isSynchronized = getBoolean(parameterContextElement, "isSynchronized"); - dto.setParameterProviderId(parameterProviderId); - dto.setParameterGroupName(parameterGroupName); - dto.setSynchronized(isSynchronized); - entity.setComponent(dto); - - return entity; - } - return null; - } - - public static ProcessGroupDTO getProcessGroup(final String parentId, final Element element, final PropertyEncryptor encryptor, final FlowEncodingVersion encodingVersion) { - final ProcessGroupDTO dto = new ProcessGroupDTO(); - final String groupId = getString(element, "id"); - dto.setId(groupId); - dto.setVersionedComponentId(getString(element, "versionedComponentId")); - dto.setParentGroupId(parentId); - dto.setName(getString(element, "name")); - dto.setPosition(getPosition(DomUtils.getChild(element, "position"))); - dto.setComments(getString(element, "comment")); - dto.setFlowfileConcurrency(getString(element, "flowfileConcurrency")); - dto.setFlowfileOutboundPolicy(getString(element, "flowfileOutboundPolicy")); - dto.setDefaultFlowFileExpiration(getString(element, "defaultFlowFileExpiration")); - dto.setDefaultBackPressureObjectThreshold(getLong(element, "defaultBackPressureObjectThreshold")); - dto.setDefaultBackPressureDataSizeThreshold(getString(element, "defaultBackPressureDataSizeThreshold")); - dto.setLogFileSuffix(getString(element, "logFileSuffix")); - - final Element versionControlInfoElement = DomUtils.getChild(element, "versionControlInformation"); - dto.setVersionControlInformation(getVersionControlInformation(versionControlInfoElement)); - - final String parameterContextId = getString(element, "parameterContextId"); - final ParameterContextReferenceEntity parameterContextReference = new ParameterContextReferenceEntity(); - parameterContextReference.setId(parameterContextId); - dto.setParameterContext(parameterContextReference); - - final Set processors = new HashSet<>(); - final Set connections = new HashSet<>(); - final Set funnels = new HashSet<>(); - final Set inputPorts = new HashSet<>(); - final Set outputPorts = new HashSet<>(); - final Set labels = new HashSet<>(); - final Set processGroups = new HashSet<>(); - final Set remoteProcessGroups = new HashSet<>(); - final Set controllerServices = new HashSet<>(); - - NodeList nodeList = DomUtils.getChildNodesByTagName(element, "processor"); - for (int i = 0; i < nodeList.getLength(); i++) { - processors.add(getProcessor((Element) nodeList.item(i), encryptor, encodingVersion)); - } - - nodeList = DomUtils.getChildNodesByTagName(element, "funnel"); - for (int i = 0; i < nodeList.getLength(); i++) { - funnels.add(getFunnel((Element) nodeList.item(i))); - } - - nodeList = DomUtils.getChildNodesByTagName(element, "inputPort"); - for (int i = 0; i < nodeList.getLength(); i++) { - inputPorts.add(getPort((Element) nodeList.item(i))); - } - - nodeList = DomUtils.getChildNodesByTagName(element, "outputPort"); - for (int i = 0; i < nodeList.getLength(); i++) { - outputPorts.add(getPort((Element) nodeList.item(i))); - } - - nodeList = DomUtils.getChildNodesByTagName(element, "label"); - for (int i = 0; i < nodeList.getLength(); i++) { - labels.add(getLabel((Element) nodeList.item(i))); - } - - nodeList = DomUtils.getChildNodesByTagName(element, "processGroup"); - for (int i = 0; i < nodeList.getLength(); i++) { - processGroups.add(getProcessGroup(groupId, (Element) nodeList.item(i), encryptor, encodingVersion)); - } - - nodeList = DomUtils.getChildNodesByTagName(element, "remoteProcessGroup"); - for (int i = 0; i < nodeList.getLength(); i++) { - remoteProcessGroups.add(getRemoteProcessGroup((Element) nodeList.item(i), encryptor)); - } - - nodeList = DomUtils.getChildNodesByTagName(element, "connection"); - for (int i = 0; i < nodeList.getLength(); i++) { - connections.add(getConnection((Element) nodeList.item(i))); - } - - nodeList = DomUtils.getChildNodesByTagName(element, "controllerService"); - for (int i=0; i < nodeList.getLength(); i++) { - controllerServices.add(getControllerService((Element) nodeList.item(i), encryptor, encodingVersion)); - } - - final FlowSnippetDTO groupContents = new FlowSnippetDTO(); - groupContents.setConnections(connections); - groupContents.setFunnels(funnels); - groupContents.setInputPorts(inputPorts); - groupContents.setLabels(labels); - groupContents.setOutputPorts(outputPorts); - groupContents.setProcessGroups(processGroups); - groupContents.setProcessors(processors); - groupContents.setRemoteProcessGroups(remoteProcessGroups); - groupContents.setControllerServices(controllerServices); - - dto.setContents(groupContents); - return dto; - } - - private static VersionControlInformationDTO getVersionControlInformation(final Element versionControlInfoElement) { - if (versionControlInfoElement == null) { - return null; - } - - final VersionControlInformationDTO dto = new VersionControlInformationDTO(); - dto.setRegistryId(getString(versionControlInfoElement, "registryId")); - dto.setBucketId(getString(versionControlInfoElement, "bucketId")); - dto.setBucketName(getString(versionControlInfoElement, "bucketName")); - dto.setFlowId(getString(versionControlInfoElement, "flowId")); - dto.setFlowName(getString(versionControlInfoElement, "flowName")); - dto.setFlowDescription(getString(versionControlInfoElement, "flowDescription")); - dto.setVersion(getInt(versionControlInfoElement, "version")); - dto.setStorageLocation(getString(versionControlInfoElement, "storageLocation")); - return dto; - } - - public static ConnectionDTO getConnection(final Element element) { - final ConnectionDTO dto = new ConnectionDTO(); - dto.setId(getString(element, "id")); - dto.setName(getString(element, "name")); - dto.setLabelIndex(getOptionalInt(element, "labelIndex")); - dto.setzIndex(getOptionalLong(element, "zIndex")); - dto.setVersionedComponentId(getString(element, "versionedComponentId")); - - final List bends = new ArrayList<>(); - final Element bendPointsElement = DomUtils.getChild(element, "bendPoints"); - if (bendPointsElement != null) { - for (final Element bendPointElement : getChildrenByTagName(bendPointsElement, "bendPoint")) { - final PositionDTO bend = getPosition(bendPointElement); - bends.add(bend); - } - } - dto.setBends(bends); - - final ConnectableDTO sourceConnectable = new ConnectableDTO(); - dto.setSource(sourceConnectable); - sourceConnectable.setId(getString(element, "sourceId")); - sourceConnectable.setGroupId(getString(element, "sourceGroupId")); - sourceConnectable.setType(getString(element, "sourceType")); - - final ConnectableDTO destConnectable = new ConnectableDTO(); - dto.setDestination(destConnectable); - destConnectable.setId(getString(element, "destinationId")); - destConnectable.setGroupId(getString(element, "destinationGroupId")); - destConnectable.setType(getString(element, "destinationType")); - - final Set relationships = new HashSet<>(); - final List relationshipNodeList = getChildrenByTagName(element, "relationship"); - for (final Element relationshipElem : relationshipNodeList) { - relationships.add(relationshipElem.getTextContent()); - } - dto.setSelectedRelationships(relationships); - - dto.setBackPressureObjectThreshold(getLong(element, "maxWorkQueueSize")); - - final String maxDataSize = getString(element, "maxWorkQueueDataSize"); - if (maxDataSize != null && !maxDataSize.trim().isEmpty()) { - dto.setBackPressureDataSizeThreshold(maxDataSize); - } - - String expiration = getString(element, "flowFileExpiration"); - if (expiration == null) { - expiration = "0 sec"; - } - dto.setFlowFileExpiration(expiration); - - final List prioritizerClasses = new ArrayList<>(); - final List prioritizerNodeList = getChildrenByTagName(element, "queuePrioritizerClass"); - for (final Element prioritizerElement : prioritizerNodeList) { - prioritizerClasses.add(prioritizerElement.getTextContent().trim()); - } - dto.setPrioritizers(prioritizerClasses); - - dto.setLoadBalanceStrategy(getString(element, "loadBalanceStrategy")); - dto.setLoadBalancePartitionAttribute(getString(element, "partitioningAttribute")); - dto.setLoadBalanceCompression(getString(element, "loadBalanceCompression")); - - return dto; - } - - public static RemoteProcessGroupDTO getRemoteProcessGroup(final Element element, final PropertyEncryptor encryptor) { - final RemoteProcessGroupDTO dto = new RemoteProcessGroupDTO(); - dto.setId(getString(element, "id")); - dto.setVersionedComponentId(getString(element, "versionedComponentId")); - dto.setName(getString(element, "name")); - dto.setTargetUri(getString(element, "url")); - dto.setTargetUris(getString(element, "urls")); - dto.setTransmitting(getBoolean(element, "transmitting")); - dto.setPosition(getPosition(DomUtils.getChild(element, "position"))); - dto.setCommunicationsTimeout(getString(element, "timeout")); - dto.setComments(getString(element, "comment")); - dto.setYieldDuration(getString(element, "yieldPeriod")); - dto.setTransportProtocol(getString(element, "transportProtocol")); - dto.setProxyHost(getString(element, "proxyHost")); - dto.setProxyPort(getOptionalInt(element, "proxyPort")); - dto.setProxyUser(getString(element, "proxyUser")); - dto.setLocalNetworkInterface(getString(element, "networkInterface")); - - final String rawPassword = getString(element, "proxyPassword"); - final String proxyPassword = encryptor == null ? rawPassword : decrypt(rawPassword, encryptor); - dto.setProxyPassword(proxyPassword); - - return dto; - } - - public static LabelDTO getLabel(final Element element) { - final LabelDTO dto = new LabelDTO(); - dto.setId(getString(element, "id")); - dto.setVersionedComponentId(getString(element, "versionedComponentId")); - dto.setLabel(getString(element, "value")); - dto.setPosition(getPosition(DomUtils.getChild(element, "position"))); - final Size size = getSize(DomUtils.getChild(element, "size")); - dto.setWidth(size.getWidth()); - dto.setHeight(size.getHeight()); - dto.setzIndex(getLong(element, "zIndex")); - dto.setStyle(getStyle(DomUtils.getChild(element, "styles"))); - - return dto; - } - - public static FunnelDTO getFunnel(final Element element) { - final FunnelDTO dto = new FunnelDTO(); - dto.setId(getString(element, "id")); - dto.setVersionedComponentId(getString(element, "versionedComponentId")); - dto.setPosition(getPosition(DomUtils.getChild(element, "position"))); - - return dto; - } - - public static PortDTO getPort(final Element element) { - final PortDTO portDTO = new PortDTO(); - portDTO.setId(getString(element, "id")); - portDTO.setVersionedComponentId(getString(element, "versionedComponentId")); - portDTO.setPosition(getPosition(DomUtils.getChild(element, "position"))); - portDTO.setName(getString(element, "name")); - portDTO.setComments(getString(element, "comments")); - portDTO.setAllowRemoteAccess(getBoolean(element, "allowRemoteAccess")); - final ScheduledState scheduledState = getScheduledState(element); - portDTO.setState(scheduledState.toString()); - - final List maxTasksElements = getChildrenByTagName(element, "maxConcurrentTasks"); - if (!maxTasksElements.isEmpty()) { - portDTO.setConcurrentlySchedulableTaskCount(Integer.parseInt(maxTasksElements.get(0).getTextContent())); - } - - final List userAccessControls = getChildrenByTagName(element, "userAccessControl"); - if (userAccessControls != null && !userAccessControls.isEmpty()) { - final Set users = new HashSet<>(); - portDTO.setUserAccessControl(users); - for (final Element userElement : userAccessControls) { - users.add(userElement.getTextContent()); - } - } - - final List groupAccessControls = getChildrenByTagName(element, "groupAccessControl"); - if (groupAccessControls != null && !groupAccessControls.isEmpty()) { - final Set groups = new HashSet<>(); - portDTO.setGroupAccessControl(groups); - for (final Element groupElement : groupAccessControls) { - groups.add(groupElement.getTextContent()); - } - } - - return portDTO; - } - - public static RemoteProcessGroupPortDescriptor getRemoteProcessGroupPort(final Element element) { - final StandardRemoteProcessGroupPortDescriptor descriptor = new StandardRemoteProcessGroupPortDescriptor(); - - // What we have serialized is the ID of the Remote Process Group, followed by a dash ('-'), followed by - // the actual ID of the port; we want to get rid of the remote process group id. - String id = getString(element, "id"); - if (id.length() > 37) { - id = id.substring(37); - } - - descriptor.setId(id); - - final String targetId = getString(element, "targetId"); - descriptor.setTargetId(targetId == null ? id : targetId); - - descriptor.setVersionedComponentId(getString(element, "versionedComponentId")); - descriptor.setName(getString(element, "name")); - descriptor.setComments(getString(element, "comments")); - descriptor.setConcurrentlySchedulableTaskCount(getInt(element, "maxConcurrentTasks")); - descriptor.setUseCompression(getBoolean(element, "useCompression")); - descriptor.setBatchCount(getOptionalInt(element, "batchCount")); - descriptor.setBatchSize(getString(element, "batchSize")); - descriptor.setBatchDuration(getString(element, "batchDuration")); - descriptor.setTransmitting("RUNNING".equalsIgnoreCase(getString(element, "scheduledState"))); - - return descriptor; - } - - public static ProcessorDTO getProcessor(final Element element, final PropertyEncryptor encryptor, final FlowEncodingVersion flowEncodingVersion) { - final ProcessorDTO dto = new ProcessorDTO(); - - dto.setId(getString(element, "id")); - dto.setVersionedComponentId(getString(element, "versionedComponentId")); - dto.setName(getString(element, "name")); - dto.setType(getString(element, "class")); - dto.setBundle(getBundle(DomUtils.getChild(element, "bundle"))); - dto.setPosition(getPosition(DomUtils.getChild(element, "position"))); - dto.setStyle(getStyle(DomUtils.getChild(element, "styles"))); - - final ProcessorConfigDTO configDto = new ProcessorConfigDTO(); - dto.setConfig(configDto); - configDto.setComments(getString(element, "comment")); - configDto.setConcurrentlySchedulableTaskCount(getInt(element, "maxConcurrentTasks")); - final String schedulingPeriod = getString(element, "schedulingPeriod"); - configDto.setSchedulingPeriod(schedulingPeriod); - configDto.setPenaltyDuration(getString(element, "penalizationPeriod")); - configDto.setYieldDuration(getString(element, "yieldPeriod")); - configDto.setBulletinLevel(getString(element, "bulletinLevel")); - configDto.setLossTolerant(getBoolean(element, "lossTolerant")); - if (getString(element, "retryCount") != null) { - configDto.setRetryCount(getInt(element, "retryCount")); - } else { - configDto.setRetryCount(10); - } - configDto.setMaxBackoffPeriod(getString(element, "maxBackoffPeriod")); - configDto.setBackoffMechanism(getString(element, "backoffMechanism")); - - final Set retriedRelationships = new HashSet<>(); - final List retriedRelationshipList = getChildrenByTagName(element, "retriedRelationship"); - for (final Element retriedRelationship : retriedRelationshipList) { - retriedRelationships.add(retriedRelationship.getTextContent()); - } - configDto.setRetriedRelationships(retriedRelationships); - - final ScheduledState scheduledState = getScheduledState(element); - dto.setState(scheduledState.toString()); - - // handle scheduling strategy - final String schedulingStrategyName = getString(element, "schedulingStrategy"); - if (schedulingStrategyName == null || schedulingStrategyName.trim().isEmpty()) { - configDto.setSchedulingStrategy(SchedulingStrategy.TIMER_DRIVEN.name()); - } else { - configDto.setSchedulingStrategy(schedulingStrategyName.trim()); - } - - // handle execution node - final String executionNode = getString(element, "executionNode"); - if (executionNode == null || executionNode.trim().isEmpty()) { - configDto.setExecutionNode(ExecutionNode.ALL.name()); - } else { - configDto.setExecutionNode(executionNode.trim()); - } - - final Long runDurationNanos = getOptionalLong(element, "runDurationNanos"); - if (runDurationNanos != null) { - configDto.setRunDurationMillis(TimeUnit.NANOSECONDS.toMillis(runDurationNanos)); - } - - configDto.setSensitiveDynamicPropertyNames(getSensitivePropertyNames(element)); - configDto.setProperties(getProperties(element, encryptor, flowEncodingVersion)); - configDto.setAnnotationData(getString(element, "annotationData")); - - final Set autoTerminatedRelationships = new HashSet<>(); - final List autoTerminateList = getChildrenByTagName(element, "autoTerminatedRelationship"); - for (final Element autoTerminateElement : autoTerminateList) { - autoTerminatedRelationships.add(autoTerminateElement.getTextContent()); - } - configDto.setAutoTerminatedRelationships(autoTerminatedRelationships); - - return dto; - } - - private static Set getSensitivePropertyNames(final Element element) { - final Set sensitivePropertyNames = new LinkedHashSet<>(); - - final List propertyElements = getChildrenByTagName(element, "property"); - for (final Element propertyElement : propertyElements) { - final String rawPropertyValue = getString(propertyElement, "value"); - if (isValueSensitive(rawPropertyValue)) { - final String name = getString(propertyElement, "name"); - sensitivePropertyNames.add(name); - } - } - - return sensitivePropertyNames; - } - - private static LinkedHashMap getProperties(final Element element, final PropertyEncryptor encryptor, final FlowEncodingVersion flowEncodingVersion) { - final LinkedHashMap properties = new LinkedHashMap<>(); - final List propertyNodeList = getChildrenByTagName(element, "property"); - - final ParameterParser parameterParser = new ExpressionLanguageAwareParameterParser(); - - for (final Element propertyElement : propertyNodeList) { - final String name = getString(propertyElement, "name"); - - final String rawPropertyValue = getString(propertyElement, "value"); - final String value = encryptor == null ? rawPropertyValue : decrypt(rawPropertyValue, encryptor); - - if (flowEncodingVersion == null || (flowEncodingVersion.getMajorVersion() <= 1 && flowEncodingVersion.getMinorVersion() < 4)) { - // Version 1.4 introduced the #{paramName} syntax for referencing parameters. If the version is less than 1.4, we must escpae any - // #{...} reference that we find. - final ParameterTokenList parameterTokenList = parameterParser.parseTokens(value); - final String escaped = parameterTokenList.escape(); - properties.put(name, escaped); - } else { - properties.put(name, value); - } - } - - return properties; - } - - private static String getString(final Element element, final String childElementName) { - final List nodeList = getChildrenByTagName(element, childElementName); - if (nodeList == null || nodeList.isEmpty()) { - return null; - } - final Element childElement = nodeList.get(0); - return childElement.getTextContent(); - } - - private static Integer getOptionalInt(final Element element, final String childElementName) { - final List nodeList = getChildrenByTagName(element, childElementName); - if (nodeList == null || nodeList.isEmpty()) { - return null; - } - final Element childElement = nodeList.get(0); - final String val = childElement.getTextContent(); - if (val == null) { - return null; - } - return Integer.parseInt(val); - } - - private static Long getOptionalLong(final Element element, final String childElementName) { - final List nodeList = getChildrenByTagName(element, childElementName); - if (nodeList == null || nodeList.isEmpty()) { - return null; - } - final Element childElement = nodeList.get(0); - final String val = childElement.getTextContent(); - if (val == null) { - return null; - } - return Long.parseLong(val); - } - - private static int getInt(final Element element, final String childElementName) { - return Integer.parseInt(getString(element, childElementName)); - } - - private static Long getLong(final Element element, final String childElementName) { - // missing element must be handled gracefully, e.g. flow definition from a previous version without this element - String longString = getString(element, childElementName); - return longString == null ? null : Long.parseLong(longString); - } - - private static boolean getBoolean(final Element element, final String childElementName) { - return Boolean.parseBoolean(getString(element, childElementName)); - } - - private static ScheduledState getScheduledState(final Element element) { - return ScheduledState.valueOf(getString(element, "scheduledState")); - } - - private static List getChildrenByTagName(final Element element, final String childElementName) { - return DomUtils.getChildElementsByTagName(element, childElementName); - } - - private static String decrypt(final String value, final PropertyEncryptor encryptor) { - if (isValueSensitive(value)) { - try { - return encryptor.decrypt(value.substring(FlowSerializer.ENC_PREFIX.length(), value.length() - FlowSerializer.ENC_SUFFIX.length())); - } catch (EncryptionException e) { - final String moreDescriptiveMessage = "There was a problem decrypting a sensitive flow configuration value. " + - "Check that the nifi.sensitive.props.key value in nifi.properties matches the value used to encrypt the flow.xml.gz file"; - logger.error(moreDescriptiveMessage, e); - throw new EncryptionException(moreDescriptiveMessage, e); - } - } else { - return value; - } - } - - private static boolean isValueSensitive(final String value) { - return value != null && value.startsWith(FlowSerializer.ENC_PREFIX) && value.endsWith(FlowSerializer.ENC_SUFFIX); - } -} diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/serialization/RunningComponentSetFilter.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/serialization/RunningComponentSetFilter.java index cb0db371ca..3fa49c3cac 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/serialization/RunningComponentSetFilter.java +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/serialization/RunningComponentSetFilter.java @@ -53,7 +53,9 @@ public class RunningComponentSetFilter implements ComponentSetFilter { public RunningComponentSetFilter(final VersionedDataflow dataflow) { dataflow.getControllerServices().forEach(service -> controllerServices.put(service.getInstanceIdentifier(), service)); dataflow.getReportingTasks().forEach(task -> reportingTasks.put(task.getInstanceIdentifier(), task)); - dataflow.getFlowAnalysisRules().forEach(rule -> flowAnalysisRules.put(rule.getInstanceIdentifier(), rule)); + if (dataflow.getFlowAnalysisRules() != null) { + dataflow.getFlowAnalysisRules().forEach(rule -> flowAnalysisRules.put(rule.getInstanceIdentifier(), rule)); + } flatten(dataflow.getRootGroup()); } diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/serialization/StandardFlowSerializer.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/serialization/StandardFlowSerializer.java deleted file mode 100644 index 4339d3040a..0000000000 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/serialization/StandardFlowSerializer.java +++ /dev/null @@ -1,716 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.nifi.controller.serialization; - -import org.apache.nifi.bundle.BundleCoordinate; -import org.apache.nifi.components.PropertyDescriptor; -import org.apache.nifi.connectable.ConnectableType; -import org.apache.nifi.connectable.Connection; -import org.apache.nifi.connectable.Funnel; -import org.apache.nifi.connectable.Port; -import org.apache.nifi.connectable.Position; -import org.apache.nifi.connectable.Size; -import org.apache.nifi.controller.FlowAnalysisRuleNode; -import org.apache.nifi.controller.FlowController; -import org.apache.nifi.controller.ParameterProviderNode; -import org.apache.nifi.controller.ProcessorNode; -import org.apache.nifi.controller.ReportingTaskNode; -import org.apache.nifi.controller.flow.FlowManager; -import org.apache.nifi.controller.label.Label; -import org.apache.nifi.controller.service.ControllerServiceNode; -import org.apache.nifi.controller.service.ControllerServiceState; -import org.apache.nifi.encrypt.PropertyEncryptor; -import org.apache.nifi.flowfile.FlowFilePrioritizer; -import org.apache.nifi.groups.ProcessGroup; -import org.apache.nifi.groups.RemoteProcessGroup; -import org.apache.nifi.parameter.Parameter; -import org.apache.nifi.parameter.ParameterContext; -import org.apache.nifi.parameter.ParameterContextManager; -import org.apache.nifi.parameter.ParameterDescriptor; -import org.apache.nifi.parameter.ParameterProviderConfiguration; -import org.apache.nifi.processor.Relationship; -import org.apache.nifi.registry.flow.FlowRegistryClientNode; -import org.apache.nifi.registry.flow.VersionControlInformation; -import org.apache.nifi.remote.PublicPort; -import org.apache.nifi.remote.RemoteGroupPort; -import org.apache.nifi.util.CharacterFilterUtils; -import org.apache.nifi.util.StringUtils; -import org.apache.nifi.xml.processing.ProcessingException; -import org.apache.nifi.xml.processing.parsers.StandardDocumentProvider; -import org.apache.nifi.xml.processing.transform.StandardTransformProvider; -import org.w3c.dom.DOMException; -import org.w3c.dom.Document; -import org.w3c.dom.Element; - -import javax.xml.transform.dom.DOMSource; -import javax.xml.transform.stream.StreamResult; -import java.io.BufferedOutputStream; -import java.io.OutputStream; -import java.util.Map; -import java.util.Optional; -import java.util.concurrent.TimeUnit; -import java.util.stream.Collectors; - -/** -* Serializes a Flow Controller as XML to an output stream. -* -* NOT THREAD-SAFE. -*/ -public class StandardFlowSerializer implements FlowSerializer { - - private static final String MAX_ENCODING_VERSION = "1.4"; - - public StandardFlowSerializer() { - } - - - @Override - public Document transform(final FlowController controller, final ScheduledStateLookup scheduledStateLookup) throws FlowSerializationException { - final PropertyEncryptor encryptor = controller.getEncryptor(); - try { - // create a new, empty document - final StandardDocumentProvider documentProvider = new StandardDocumentProvider(); - documentProvider.setNamespaceAware(true); - final Document doc = documentProvider.newDocument(); - - // populate document with controller state - final Element rootNode = doc.createElement("flowController"); - rootNode.setAttribute("encoding-version", MAX_ENCODING_VERSION); - doc.appendChild(rootNode); - addTextElement(rootNode, "maxTimerDrivenThreadCount", controller.getMaxTimerDrivenThreadCount()); - - final Element registriesElement = doc.createElement("registries"); - rootNode.appendChild(registriesElement); - addFlowRegistryClients(registriesElement, controller.getFlowManager(), encryptor); - - final Element parameterContextsElement = doc.createElement("parameterContexts"); - rootNode.appendChild(parameterContextsElement); - addParameterContexts(parameterContextsElement, controller.getFlowManager().getParameterContextManager(), encryptor); - - addProcessGroup(rootNode, controller.getFlowManager().getRootGroup(), "rootGroup", scheduledStateLookup, encryptor); - - // Add root-level controller services - final Element controllerServicesNode = doc.createElement("controllerServices"); - rootNode.appendChild(controllerServicesNode); - for (final ControllerServiceNode serviceNode : controller.getFlowManager().getRootControllerServices()) { - addControllerService(controllerServicesNode, serviceNode, encryptor); - } - - final Element reportingTasksNode = doc.createElement("reportingTasks"); - rootNode.appendChild(reportingTasksNode); - for (final ReportingTaskNode taskNode : controller.getAllReportingTasks()) { - addReportingTask(reportingTasksNode, taskNode, encryptor); - } - - final Element flowAnalysisRulesNode = doc.createElement("flowAnalysisRules"); - rootNode.appendChild(flowAnalysisRulesNode); - for (final FlowAnalysisRuleNode flowAnalysisRuleNode : controller.getAllFlowAnalysisRules()) { - addFlowAnalysisRule(flowAnalysisRulesNode, flowAnalysisRuleNode, encryptor); - } - - final Element parameterProvidersNode = doc.createElement("parameterProviders"); - rootNode.appendChild(parameterProvidersNode); - for (final ParameterProviderNode providerNode : controller.getFlowManager().getAllParameterProviders()) { - addParameterProvider(parameterProvidersNode, providerNode, encryptor); - } - - return doc; - } catch (final ProcessingException | DOMException | IllegalArgumentException e) { - throw new FlowSerializationException(e); - } - } - - @Override - public void serialize(final Document flowConfiguration, final OutputStream os) throws FlowSerializationException { - try { - final DOMSource domSource = new DOMSource(flowConfiguration); - final StreamResult streamResult = new StreamResult(new BufferedOutputStream(os)); - - // configure the transformer and convert the DOM - final StandardTransformProvider transformProvider = new StandardTransformProvider(); - transformProvider.setIndent(true); - - // transform the document to byte stream - transformProvider.transform(domSource, streamResult); - - } catch (final DOMException | IllegalArgumentException | ProcessingException e) { - throw new FlowSerializationException(e); - } - } - - private void addParameterContexts(final Element parentElement, final ParameterContextManager parameterContextManager, - final PropertyEncryptor encryptor) { - for (final ParameterContext parameterContext : parameterContextManager.getParameterContexts()) { - final Element parameterContextElement = parentElement.getOwnerDocument().createElement("parameterContext"); - parentElement.appendChild(parameterContextElement); - - addStringElement(parameterContextElement, "id", parameterContext.getIdentifier()); - addStringElement(parameterContextElement, "name", parameterContext.getName()); - addStringElement(parameterContextElement, "description", parameterContext.getDescription()); - - for(final ParameterContext childContext : parameterContext.getInheritedParameterContexts()) { - addStringElement(parameterContextElement, "inheritedParameterContextId", childContext.getIdentifier()); - } - if (parameterContext.getParameterProviderConfiguration() != null) { - final ParameterProviderConfiguration parameterProviderConfiguration = parameterContext.getParameterProviderConfiguration(); - addStringElement(parameterContextElement, "parameterProviderId", parameterProviderConfiguration.getParameterProviderId()); - if (parameterProviderConfiguration.getParameterGroupName() != null) { - addStringElement(parameterContextElement, "parameterGroupName", parameterProviderConfiguration.getParameterGroupName()); - } - addStringElement(parameterContextElement, "isSynchronized", String.valueOf(parameterProviderConfiguration.isSynchronized())); - } - - for (final Parameter parameter : parameterContext.getParameters().values()) { - addParameter(parameterContextElement, parameter, encryptor); - } - } - } - - private void addParameter(final Element parentElement, final Parameter parameter, final PropertyEncryptor encryptor) { - final Element parameterElement = parentElement.getOwnerDocument().createElement("parameter"); - parentElement.appendChild(parameterElement); - - final ParameterDescriptor descriptor = parameter.getDescriptor(); - addStringElement(parameterElement, "name", descriptor.getName()); - addStringElement(parameterElement, "description", descriptor.getDescription()); - addStringElement(parameterElement, "sensitive", String.valueOf(descriptor.isSensitive())); - addStringElement(parameterElement, "provided", String.valueOf(parameter.isProvided())); - - if (parameter.getValue() != null) { - if (descriptor.isSensitive()) { - final String parameterValue = parameter.getValue(); - addStringElement(parameterElement, "value", parameterValue == null ? null : ENC_PREFIX + encryptor.encrypt(parameterValue) + ENC_SUFFIX); - } else { - addStringElement(parameterElement, "value", parameter.getValue()); - } - } - } - - private void addFlowRegistryClients(final Element parentElement, final FlowManager flowManager, final PropertyEncryptor encryptor) { - for (final String registryId : flowManager.getAllFlowRegistryClients().stream().map(FlowRegistryClientNode::getIdentifier).collect(Collectors.toSet())) { - final FlowRegistryClientNode flowRegistry = flowManager.getFlowRegistryClient(registryId); - - final Element registryElement = parentElement.getOwnerDocument().createElement("flowRegistry"); - parentElement.appendChild(registryElement); - - addTextElement(registryElement, "id", flowRegistry.getIdentifier()); - addTextElement(registryElement, "name", flowRegistry.getName()); - addTextElement(registryElement, "description", flowRegistry.getDescription()); - addTextElement(registryElement, "class", flowRegistry.getCanonicalClassName()); - addBundle(registryElement, flowRegistry.getBundleCoordinate()); - addConfiguration(registryElement, flowRegistry.getRawPropertyValues(), flowRegistry.getAnnotationData(), encryptor); - } - } - - private void addStringElement(final Element parentElement, final String elementName, final String value) { - final Element childElement = parentElement.getOwnerDocument().createElement(elementName); - childElement.setTextContent(CharacterFilterUtils.filterInvalidXmlCharacters(value)); - parentElement.appendChild(childElement); - } - - private void addSize(final Element parentElement, final Size size) { - final Element element = parentElement.getOwnerDocument().createElement("size"); - element.setAttribute("width", String.valueOf(size.getWidth())); - element.setAttribute("height", String.valueOf(size.getHeight())); - parentElement.appendChild(element); - } - - private void addPosition(final Element parentElement, final Position position) { - addPosition(parentElement, position, "position"); - } - - private void addPosition(final Element parentElement, final Position position, final String elementName) { - final Element element = parentElement.getOwnerDocument().createElement(elementName); - element.setAttribute("x", String.valueOf(position.getX())); - element.setAttribute("y", String.valueOf(position.getY())); - parentElement.appendChild(element); - } - - private void addProcessGroup(final Element parentElement, final ProcessGroup group, final String elementName, final ScheduledStateLookup scheduledStateLookup, - final PropertyEncryptor encryptor) { - final Document doc = parentElement.getOwnerDocument(); - final Element element = doc.createElement(elementName); - parentElement.appendChild(element); - addTextElement(element, "id", group.getIdentifier()); - addTextElement(element, "versionedComponentId", group.getVersionedComponentId()); - addTextElement(element, "name", group.getName()); - addPosition(element, group.getPosition()); - addTextElement(element, "comment", group.getComments()); - addTextElement(element, "flowfileConcurrency", group.getFlowFileConcurrency().name()); - addTextElement(element, "flowfileOutboundPolicy", group.getFlowFileOutboundPolicy().name()); - addTextElement(element, "defaultFlowFileExpiration", group.getDefaultFlowFileExpiration()); - addTextElement(element, "defaultBackPressureObjectThreshold", group.getDefaultBackPressureObjectThreshold()); - addTextElement(element, "defaultBackPressureDataSizeThreshold", group.getDefaultBackPressureDataSizeThreshold()); - addTextElement(element, "logFileSuffix", group.getLogFileSuffix()); - - final VersionControlInformation versionControlInfo = group.getVersionControlInformation(); - if (versionControlInfo != null) { - final Element versionControlInfoElement = doc.createElement("versionControlInformation"); - addTextElement(versionControlInfoElement, "registryId", versionControlInfo.getRegistryIdentifier()); - addTextElement(versionControlInfoElement, "bucketId", versionControlInfo.getBucketIdentifier()); - addTextElement(versionControlInfoElement, "bucketName", versionControlInfo.getBucketName()); - addTextElement(versionControlInfoElement, "flowId", versionControlInfo.getFlowIdentifier()); - addTextElement(versionControlInfoElement, "flowName", versionControlInfo.getFlowName()); - addTextElement(versionControlInfoElement, "flowDescription", versionControlInfo.getFlowDescription()); - addTextElement(versionControlInfoElement, "version", versionControlInfo.getVersion()); - addTextElement(versionControlInfoElement, "storageLocation", versionControlInfo.getStorageLocation()); - element.appendChild(versionControlInfoElement); - } - - for (final ProcessorNode processor : group.getProcessors()) { - addProcessor(element, processor, scheduledStateLookup, encryptor); - } - - for (final Port port : group.getInputPorts()) { - if (port instanceof PublicPort) { - addPublicPort(element, (PublicPort) port, "inputPort", scheduledStateLookup); - } else { - addPort(element, port, "inputPort", scheduledStateLookup); - } - } - - for (final Port port : group.getOutputPorts()) { - if (port instanceof PublicPort) { - addPublicPort(element, (PublicPort) port, "outputPort", scheduledStateLookup); - } else { - addPort(element, port, "outputPort", scheduledStateLookup); - } - } - - for (final Label label : group.getLabels()) { - addLabel(element, label); - } - - for (final Funnel funnel : group.getFunnels()) { - addFunnel(element, funnel); - } - - for (final ProcessGroup childGroup : group.getProcessGroups()) { - addProcessGroup(element, childGroup, "processGroup", scheduledStateLookup, encryptor); - } - - for (final RemoteProcessGroup remoteRef : group.getRemoteProcessGroups()) { - addRemoteProcessGroup(element, remoteRef, scheduledStateLookup, encryptor); - } - - for (final Connection connection : group.getConnections()) { - addConnection(element, connection); - } - - for (final ControllerServiceNode service : group.getControllerServices(false)) { - addControllerService(element, service, encryptor); - } - - final ParameterContext parameterContext = group.getParameterContext(); - if (parameterContext != null) { - addTextElement(element, "parameterContextId", parameterContext.getIdentifier()); - } - } - - private static void addBundle(final Element parentElement, final BundleCoordinate coordinate) { - // group - final Element groupElement = parentElement.getOwnerDocument().createElement("group"); - groupElement.setTextContent(CharacterFilterUtils.filterInvalidXmlCharacters(coordinate.getGroup())); - - // artifact - final Element artifactElement = parentElement.getOwnerDocument().createElement("artifact"); - artifactElement.setTextContent(CharacterFilterUtils.filterInvalidXmlCharacters(coordinate.getId())); - - // version - final Element versionElement = parentElement.getOwnerDocument().createElement("version"); - versionElement.setTextContent(CharacterFilterUtils.filterInvalidXmlCharacters(coordinate.getVersion())); - - // bundle - final Element bundleElement = parentElement.getOwnerDocument().createElement("bundle"); - bundleElement.appendChild(groupElement); - bundleElement.appendChild(artifactElement); - bundleElement.appendChild(versionElement); - - parentElement.appendChild(bundleElement); - } - - private void addStyle(final Element parentElement, final Map style) { - final Element element = parentElement.getOwnerDocument().createElement("styles"); - - for (final Map.Entry entry : style.entrySet()) { - final Element styleElement = parentElement.getOwnerDocument().createElement("style"); - styleElement.setAttribute("name", entry.getKey()); - styleElement.setTextContent(entry.getValue()); - element.appendChild(styleElement); - } - - parentElement.appendChild(element); - } - - private void addLabel(final Element parentElement, final Label label) { - final Document doc = parentElement.getOwnerDocument(); - final Element element = doc.createElement("label"); - parentElement.appendChild(element); - addTextElement(element, "id", label.getIdentifier()); - addTextElement(element, "versionedComponentId", label.getVersionedComponentId()); - addTextElement(element, "zIndex", label.getZIndex()); - - addPosition(element, label.getPosition()); - addSize(element, label.getSize()); - addStyle(element, label.getStyle()); - - addTextElement(element, "value", label.getValue()); - parentElement.appendChild(element); - } - - private void addFunnel(final Element parentElement, final Funnel funnel) { - final Document doc = parentElement.getOwnerDocument(); - final Element element = doc.createElement("funnel"); - parentElement.appendChild(element); - addTextElement(element, "id", funnel.getIdentifier()); - addTextElement(element, "versionedComponentId", funnel.getVersionedComponentId()); - addPosition(element, funnel.getPosition()); - } - - private void addRemoteProcessGroup(final Element parentElement, final RemoteProcessGroup remoteRef, final ScheduledStateLookup scheduledStateLookup, - final PropertyEncryptor encryptor) { - final Document doc = parentElement.getOwnerDocument(); - final Element element = doc.createElement("remoteProcessGroup"); - parentElement.appendChild(element); - addTextElement(element, "id", remoteRef.getIdentifier()); - addTextElement(element, "versionedComponentId", remoteRef.getVersionedComponentId()); - addTextElement(element, "name", remoteRef.getName()); - addPosition(element, remoteRef.getPosition()); - addTextElement(element, "comment", remoteRef.getComments()); - addTextElement(element, "url", remoteRef.getTargetUri()); - addTextElement(element, "urls", remoteRef.getTargetUris()); - addTextElement(element, "timeout", remoteRef.getCommunicationsTimeout()); - addTextElement(element, "yieldPeriod", remoteRef.getYieldDuration()); - addTextElement(element, "transmitting", String.valueOf(remoteRef.isTransmitting())); - addTextElement(element, "transportProtocol", remoteRef.getTransportProtocol().name()); - addTextElement(element, "proxyHost", remoteRef.getProxyHost()); - if (remoteRef.getProxyPort() != null) { - addTextElement(element, "proxyPort", remoteRef.getProxyPort()); - } - addTextElement(element, "proxyUser", remoteRef.getProxyUser()); - if (!StringUtils.isEmpty(remoteRef.getProxyPassword())) { - final String value = ENC_PREFIX + encryptor.encrypt(remoteRef.getProxyPassword()) + ENC_SUFFIX; - addTextElement(element, "proxyPassword", value); - } - if (remoteRef.getNetworkInterface() != null) { - addTextElement(element, "networkInterface", remoteRef.getNetworkInterface()); - } - - for (final RemoteGroupPort port : remoteRef.getInputPorts()) { - if (port.hasIncomingConnection()) { - addRemoteGroupPort(element, port, "inputPort", scheduledStateLookup); - } - } - - for (final RemoteGroupPort port : remoteRef.getOutputPorts()) { - if (!port.getConnections().isEmpty()) { - addRemoteGroupPort(element, port, "outputPort", scheduledStateLookup); - } - } - - parentElement.appendChild(element); - } - - private void addRemoteGroupPort(final Element parentElement, final RemoteGroupPort port, final String elementName, final ScheduledStateLookup scheduledStateLookup) { - final Document doc = parentElement.getOwnerDocument(); - final Element element = doc.createElement(elementName); - parentElement.appendChild(element); - addTextElement(element, "id", port.getIdentifier()); - addTextElement(element, "versionedComponentId", port.getVersionedComponentId()); - addTextElement(element, "name", port.getName()); - addPosition(element, port.getPosition()); - addTextElement(element, "comments", port.getComments()); - addTextElement(element, "scheduledState", scheduledStateLookup.getScheduledState(port).name()); - addTextElement(element, "targetId", port.getTargetIdentifier()); - addTextElement(element, "maxConcurrentTasks", port.getMaxConcurrentTasks()); - addTextElement(element, "useCompression", String.valueOf(port.isUseCompression())); - final Integer batchCount = port.getBatchCount(); - if (batchCount != null && batchCount > 0) { - addTextElement(element, "batchCount", batchCount); - } - final String batchSize = port.getBatchSize(); - if (batchSize != null && batchSize.length() > 0) { - addTextElement(element, "batchSize", batchSize); - } - final String batchDuration = port.getBatchDuration(); - if (batchDuration != null && batchDuration.length() > 0) { - addTextElement(element, "batchDuration", batchDuration); - } - - parentElement.appendChild(element); - } - - private void addPort(final Element parentElement, final Port port, final String elementName, final ScheduledStateLookup scheduledStateLookup) { - final Document doc = parentElement.getOwnerDocument(); - final Element element = doc.createElement(elementName); - parentElement.appendChild(element); - addTextElement(element, "id", port.getIdentifier()); - addTextElement(element, "versionedComponentId", port.getVersionedComponentId()); - addTextElement(element, "name", port.getName()); - addPosition(element, port.getPosition()); - addTextElement(element, "comments", port.getComments()); - addTextElement(element, "scheduledState", scheduledStateLookup.getScheduledState(port).name()); - - parentElement.appendChild(element); - } - - private void addPublicPort(final Element parentElement, final PublicPort port, final String elementName, final ScheduledStateLookup scheduledStateLookup) { - final Document doc = parentElement.getOwnerDocument(); - final Element element = doc.createElement(elementName); - parentElement.appendChild(element); - addTextElement(element, "id", port.getIdentifier()); - addTextElement(element, "versionedComponentId", port.getVersionedComponentId()); - addTextElement(element, "name", port.getName()); - addPosition(element, port.getPosition()); - addTextElement(element, "comments", port.getComments()); - addTextElement(element, "scheduledState", scheduledStateLookup.getScheduledState(port).name()); - addTextElement(element, "maxConcurrentTasks", String.valueOf(port.getMaxConcurrentTasks())); - addTextElement(element, "allowRemoteAccess", Boolean.TRUE.toString()); - for (final String user : port.getUserAccessControl()) { - addTextElement(element, "userAccessControl", user); - } - for (final String group : port.getGroupAccessControl()) { - addTextElement(element, "groupAccessControl", group); - } - - parentElement.appendChild(element); - } - - private void addProcessor(final Element parentElement, final ProcessorNode processor, final ScheduledStateLookup scheduledStateLookup, - final PropertyEncryptor encryptor) { - final Document doc = parentElement.getOwnerDocument(); - final Element element = doc.createElement("processor"); - parentElement.appendChild(element); - addTextElement(element, "id", processor.getIdentifier()); - addTextElement(element, "versionedComponentId", processor.getVersionedComponentId()); - addTextElement(element, "name", processor.getName()); - - addPosition(element, processor.getPosition()); - addStyle(element, processor.getStyle()); - - addTextElement(element, "comment", processor.getComments()); - addTextElement(element, "class", processor.getCanonicalClassName()); - - addBundle(element, processor.getBundleCoordinate()); - - addTextElement(element, "maxConcurrentTasks", processor.getMaxConcurrentTasks()); - addTextElement(element, "schedulingPeriod", processor.getSchedulingPeriod()); - addTextElement(element, "penalizationPeriod", processor.getPenalizationPeriod()); - addTextElement(element, "yieldPeriod", processor.getYieldPeriod()); - addTextElement(element, "bulletinLevel", processor.getBulletinLevel().toString()); - addTextElement(element, "lossTolerant", String.valueOf(processor.isLossTolerant())); - addTextElement(element, "scheduledState", scheduledStateLookup.getScheduledState(processor).name()); - addTextElement(element, "schedulingStrategy", processor.getSchedulingStrategy().name()); - addTextElement(element, "executionNode", processor.getExecutionNode().name()); - addTextElement(element, "runDurationNanos", processor.getRunDuration(TimeUnit.NANOSECONDS)); - addTextElement(element, "retryCount", processor.getRetryCount()); - addTextElement(element, "backoffMechanism", processor.getBackoffMechanism().name()); - addTextElement(element, "maxBackoffPeriod", processor.getMaxBackoffPeriod()); - - for (final String relationship : processor.getRetriedRelationships()) { - addTextElement(element, "retriedRelationship", relationship); - } - - addConfiguration(element, processor.getRawPropertyValues(), processor.getAnnotationData(), encryptor); - - for (final Relationship rel : processor.getAutoTerminatedRelationships()) { - addTextElement(element, "autoTerminatedRelationship", rel.getName()); - } - } - - private static void addConfiguration(final Element element, final Map properties, final String annotationData, final PropertyEncryptor encryptor) { - final Document doc = element.getOwnerDocument(); - for (final Map.Entry entry : properties.entrySet()) { - final PropertyDescriptor descriptor = entry.getKey(); - String value = entry.getValue(); - - if (value == null) { - value = descriptor.getDefaultValue(); - } - - if (value != null && descriptor.isSensitive()) { - value = ENC_PREFIX + encryptor.encrypt(value) + ENC_SUFFIX; - } - - final Element propElement = doc.createElement("property"); - addTextElement(propElement, "name", descriptor.getName()); - if (value != null) { - addTextElement(propElement, "value", value); - } - - element.appendChild(propElement); - } - - if (annotationData != null) { - addTextElement(element, "annotationData", annotationData); - } - } - - private void addConnection(final Element parentElement, final Connection connection) { - final Document doc = parentElement.getOwnerDocument(); - final Element element = doc.createElement("connection"); - parentElement.appendChild(element); - addTextElement(element, "id", connection.getIdentifier()); - addTextElement(element, "versionedComponentId", connection.getVersionedComponentId()); - addTextElement(element, "name", connection.getName()); - - final Element bendPointsElement = doc.createElement("bendPoints"); - element.appendChild(bendPointsElement); - for (final Position bendPoint : connection.getBendPoints()) { - addPosition(bendPointsElement, bendPoint, "bendPoint"); - } - - addTextElement(element, "labelIndex", connection.getLabelIndex()); - addTextElement(element, "zIndex", connection.getZIndex()); - - final String sourceId = connection.getSource().getIdentifier(); - final ConnectableType sourceType = connection.getSource().getConnectableType(); - final String sourceGroupId; - if (sourceType == ConnectableType.REMOTE_OUTPUT_PORT) { - sourceGroupId = ((RemoteGroupPort) connection.getSource()).getRemoteProcessGroup().getIdentifier(); - } else { - sourceGroupId = connection.getSource().getProcessGroup().getIdentifier(); - } - - final ConnectableType destinationType = connection.getDestination().getConnectableType(); - final String destinationId = connection.getDestination().getIdentifier(); - final String destinationGroupId; - if (destinationType == ConnectableType.REMOTE_INPUT_PORT) { - destinationGroupId = ((RemoteGroupPort) connection.getDestination()).getRemoteProcessGroup().getIdentifier(); - } else { - destinationGroupId = connection.getDestination().getProcessGroup().getIdentifier(); - } - - addTextElement(element, "sourceId", sourceId); - addTextElement(element, "sourceGroupId", sourceGroupId); - addTextElement(element, "sourceType", sourceType.toString()); - - addTextElement(element, "destinationId", destinationId); - addTextElement(element, "destinationGroupId", destinationGroupId); - addTextElement(element, "destinationType", destinationType.toString()); - - for (final Relationship relationship : connection.getRelationships()) { - addTextElement(element, "relationship", relationship.getName()); - } - - addTextElement(element, "maxWorkQueueSize", connection.getFlowFileQueue().getBackPressureObjectThreshold()); - addTextElement(element, "maxWorkQueueDataSize", connection.getFlowFileQueue().getBackPressureDataSizeThreshold()); - - addTextElement(element, "flowFileExpiration", connection.getFlowFileQueue().getFlowFileExpiration()); - for (final FlowFilePrioritizer comparator : connection.getFlowFileQueue().getPriorities()) { - final String className = comparator.getClass().getCanonicalName(); - addTextElement(element, "queuePrioritizerClass", className); - } - - addTextElement(element, "loadBalanceStrategy", connection.getFlowFileQueue().getLoadBalanceStrategy().name()); - addTextElement(element, "partitioningAttribute", connection.getFlowFileQueue().getPartitioningAttribute()); - addTextElement(element, "loadBalanceCompression", connection.getFlowFileQueue().getLoadBalanceCompression().name()); - - parentElement.appendChild(element); - } - - public void addControllerService(final Element element, final ControllerServiceNode serviceNode, final PropertyEncryptor encryptor) { - final Element serviceElement = element.getOwnerDocument().createElement("controllerService"); - addTextElement(serviceElement, "id", serviceNode.getIdentifier()); - addTextElement(serviceElement, "versionedComponentId", serviceNode.getVersionedComponentId()); - addTextElement(serviceElement, "name", serviceNode.getName()); - addTextElement(serviceElement, "comment", serviceNode.getComments()); - addTextElement(serviceElement, "bulletinLevel", serviceNode.getBulletinLevel().toString()); - addTextElement(serviceElement, "class", serviceNode.getCanonicalClassName()); - - addBundle(serviceElement, serviceNode.getBundleCoordinate()); - - final ControllerServiceState state = serviceNode.getState(); - final boolean enabled = (state == ControllerServiceState.ENABLED || state == ControllerServiceState.ENABLING); - addTextElement(serviceElement, "enabled", String.valueOf(enabled)); - - addConfiguration(serviceElement, serviceNode.getRawPropertyValues(), serviceNode.getAnnotationData(), encryptor); - - element.appendChild(serviceElement); - } - - public static void addReportingTask(final Element element, final ReportingTaskNode taskNode, final PropertyEncryptor encryptor) { - final Element taskElement = element.getOwnerDocument().createElement("reportingTask"); - addTextElement(taskElement, "id", taskNode.getIdentifier()); - addTextElement(taskElement, "name", taskNode.getName()); - addTextElement(taskElement, "comment", taskNode.getComments()); - addTextElement(taskElement, "class", taskNode.getCanonicalClassName()); - - addBundle(taskElement, taskNode.getBundleCoordinate()); - - addTextElement(taskElement, "schedulingPeriod", taskNode.getSchedulingPeriod()); - addTextElement(taskElement, "scheduledState", taskNode.getScheduledState().name()); - addTextElement(taskElement, "schedulingStrategy", taskNode.getSchedulingStrategy().name()); - - addConfiguration(taskElement, taskNode.getRawPropertyValues(), taskNode.getAnnotationData(), encryptor); - - element.appendChild(taskElement); - } - - private void addFlowAnalysisRule(Element flowAnalysisRulesNode, FlowAnalysisRuleNode flowAnalysisRuleNode, final PropertyEncryptor encryptor) { - final Element taskElement = flowAnalysisRulesNode.getOwnerDocument().createElement("flowAnalysisRule"); - addTextElement(taskElement, "id", flowAnalysisRuleNode.getIdentifier()); - addTextElement(taskElement, "name", flowAnalysisRuleNode.getName()); - addTextElement(taskElement, "comment", flowAnalysisRuleNode.getComments()); - addTextElement(taskElement, "class", flowAnalysisRuleNode.getCanonicalClassName()); - - addBundle(taskElement, flowAnalysisRuleNode.getBundleCoordinate()); - - addTextElement(taskElement, "enforcementPolicy", flowAnalysisRuleNode.getEnforcementPolicy().name()); - addTextElement(taskElement, "state", flowAnalysisRuleNode.getState().name()); - - addConfiguration(taskElement, flowAnalysisRuleNode.getRawPropertyValues(), flowAnalysisRuleNode.getAnnotationData(), encryptor); - - flowAnalysisRulesNode.appendChild(taskElement); - } - - public static void addParameterProvider(final Element element, final ParameterProviderNode providerNode, final PropertyEncryptor encryptor) { - final Element taskElement = element.getOwnerDocument().createElement("parameterProvider"); - addTextElement(taskElement, "id", providerNode.getIdentifier()); - addTextElement(taskElement, "name", providerNode.getName()); - addTextElement(taskElement, "comment", providerNode.getComments()); - addTextElement(taskElement, "class", providerNode.getCanonicalClassName()); - - addBundle(taskElement, providerNode.getBundleCoordinate()); - - addConfiguration(taskElement, providerNode.getRawPropertyValues(), providerNode.getAnnotationData(), encryptor); - - element.appendChild(taskElement); - } - - private static void addTextElement(final Element element, final String name, final long value) { - addTextElement(element, name, String.valueOf(value)); - } - - private static void addTextElement(final Element element, final String name, final String value) { - final Document doc = element.getOwnerDocument(); - final Element toAdd = doc.createElement(name); - toAdd.setTextContent(CharacterFilterUtils.filterInvalidXmlCharacters(value)); // value should already be filtered, but just in case ensure there are no invalid xml characters - element.appendChild(toAdd); - } - - private static void addTextElement(final Element element, final String name, final Optional value) { - if (!value.isPresent()) { - return; - } - - final Document doc = element.getOwnerDocument(); - final Element toAdd = doc.createElement(name); - toAdd.setTextContent(CharacterFilterUtils.filterInvalidXmlCharacters(value.get())); // value should already be filtered, but just in case ensure there are no invalid xml characters - element.appendChild(toAdd); - } -} \ No newline at end of file diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/serialization/StandardFlowSynchronizer.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/serialization/StandardFlowSynchronizer.java deleted file mode 100644 index 0b8d142e0c..0000000000 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/serialization/StandardFlowSynchronizer.java +++ /dev/null @@ -1,64 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.nifi.controller.serialization; - -import org.apache.nifi.cluster.protocol.DataFlow; -import org.apache.nifi.controller.FlowController; -import org.apache.nifi.controller.MissingBundleException; -import org.apache.nifi.controller.UninheritableFlowException; -import org.apache.nifi.controller.XmlFlowSynchronizer; -import org.apache.nifi.groups.BundleUpdateStrategy; -import org.apache.nifi.services.FlowService; - -public class StandardFlowSynchronizer implements FlowSynchronizer { - private final XmlFlowSynchronizer xmlFlowSynchronizer; - private final VersionedFlowSynchronizer versionedFlowSynchronizer; - - public StandardFlowSynchronizer(final XmlFlowSynchronizer xmlFlowSynchronizer, final VersionedFlowSynchronizer versionedFlowSynchronizer) { - this.xmlFlowSynchronizer = xmlFlowSynchronizer; - this.versionedFlowSynchronizer = versionedFlowSynchronizer; - } - - @Override - public void sync(final FlowController controller, final DataFlow dataFlow, final FlowService flowService, final BundleUpdateStrategy bundleUpdateStrategy) - throws FlowSerializationException, UninheritableFlowException, FlowSynchronizationException, MissingBundleException { - - final FlowSynchronizer synchronizer = isXml(dataFlow) ? xmlFlowSynchronizer : versionedFlowSynchronizer; - synchronizer.sync(controller, dataFlow, flowService, bundleUpdateStrategy); - } - - public static boolean isFlowEmpty(final DataFlow dataFlow) { - if (dataFlow == null || dataFlow.getFlow() == null || dataFlow.getFlow().length == 0) { - return true; - } - - if (isXml(dataFlow)) { - return XmlFlowSynchronizer.isFlowEmpty(dataFlow.getFlowDocument()); - } else { - return VersionedFlowSynchronizer.isFlowEmpty(dataFlow); - } - } - - private static boolean isXml(final DataFlow dataFlow) { - if (dataFlow == null || dataFlow.getFlow() == null || dataFlow.getFlow().length == 0) { - return true; - } - - return dataFlow.isXml(); - } -} diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/serialization/VersionedFlowSynchronizer.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/serialization/VersionedFlowSynchronizer.java index f53fddd21c..8e8537f2a2 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/serialization/VersionedFlowSynchronizer.java +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/serialization/VersionedFlowSynchronizer.java @@ -66,6 +66,7 @@ import org.apache.nifi.controller.inheritance.BundleCompatibilityCheck; import org.apache.nifi.controller.inheritance.ConnectionMissingCheck; import org.apache.nifi.controller.inheritance.FlowInheritability; import org.apache.nifi.controller.inheritance.FlowInheritabilityCheck; +import org.apache.nifi.controller.inheritance.MissingComponentsCheck; import org.apache.nifi.controller.reporting.ReportingTaskInstantiationException; import org.apache.nifi.controller.service.ControllerServiceNode; import org.apache.nifi.encrypt.EncryptionException; @@ -163,56 +164,78 @@ public class VersionedFlowSynchronizer implements FlowSynchronizer { final boolean flowAlreadySynchronized = controller.isFlowSynchronized(); logger.info("Synchronizing FlowController with proposed flow: Controller Already Synchronized = {}", flowAlreadySynchronized); + final DataFlow existingDataFlow = getExistingDataFlow(controller); + boolean existingFlowEmpty = isFlowEmpty(existingDataFlow); + // If bundle update strategy is configured to allow for compatible bundles, update any components to use compatible bundles if // the exact bundle does not exist. - if (bundleUpdateStrategy == BundleUpdateStrategy.USE_SPECIFIED_OR_COMPATIBLE_OR_GHOST) { + if (!existingFlowEmpty && bundleUpdateStrategy == BundleUpdateStrategy.USE_SPECIFIED_OR_COMPATIBLE_OR_GHOST) { mapCompatibleBundles(proposedFlow, controller.getExtensionManager()); } // serialize controller state to bytes - final DataFlow existingDataFlow = getExistingDataFlow(controller); checkFlowInheritability(existingDataFlow, proposedFlow, controller, bundleUpdateStrategy); - final FlowComparison flowComparison = compareFlows(existingDataFlow, proposedFlow, controller.getEncryptor()); - final Set flowDifferences = flowComparison.getDifferences(); - if (flowDifferences.isEmpty()) { - logger.debug("No differences between current flow and proposed flow. Will not create backup of existing flow."); - } else if (isExistingFlowEmpty(controller)) { - logger.debug("Currently loaded dataflow is empty. Will not create backup of existing flow."); - } else { - backupExistingFlow(); + logger.debug("Checking missing component inheritability"); + final FlowInheritabilityCheck missingComponentsCheck = new MissingComponentsCheck(); + final FlowInheritability componentInheritability = missingComponentsCheck.checkInheritability(existingDataFlow, proposedFlow, controller); + if (!componentInheritability.isInheritable()) { + throw new UninheritableFlowException("Proposed Flow is not inheritable by the flow controller because of differences in missing components: " + componentInheritability.getExplanation()); } - final AffectedComponentSet affectedComponents = determineAffectedComponents(flowComparison, controller); - final AffectedComponentSet activeSet = affectedComponents.toActiveSet(); + FlowComparison flowComparison = null; + AffectedComponentSet affectedComponents = null; + AffectedComponentSet activeSet = null; - // Stop the active components, and then wait for all components to be stopped. - logger.info("In order to inherit proposed dataflow, will stop any components that will be affected by the update"); - if (logger.isDebugEnabled()) { - logger.debug("Will stop the following components:"); - logger.debug(activeSet.toString()); - final String differencesToString = flowDifferences.stream() - .map(FlowDifference::toString) - .collect(Collectors.joining("\n")); - logger.debug("This Active Set was determined from the following Flow Differences:\n{}", differencesToString); + if (!existingFlowEmpty) { + flowComparison = compareFlows(existingDataFlow, proposedFlow, controller.getEncryptor()); + final Set flowDifferences = flowComparison.getDifferences(); + + if (flowDifferences.isEmpty()) { + logger.debug("No differences between current flow and proposed flow. Will not create backup of existing flow."); + } else if (isExistingFlowEmpty(controller)) { + logger.debug("Currently loaded dataflow is empty. Will not create backup of existing flow."); + } else { + backupExistingFlow(); + } + + affectedComponents = determineAffectedComponents(flowComparison, controller); + activeSet = affectedComponents.toActiveSet(); + + // Stop the active components, and then wait for all components to be stopped. + logger.info("In order to inherit proposed dataflow, will stop any components that will be affected by the update"); + if (logger.isDebugEnabled()) { + logger.debug("Will stop the following components:"); + logger.debug(activeSet.toString()); + final String differencesToString = flowDifferences.stream() + .map(FlowDifference::toString) + .collect(Collectors.joining("\n")); + logger.debug("This Active Set was determined from the following Flow Differences:\n{}", + differencesToString); + } + + activeSet.stop(); } - activeSet.stop(); - try { // Ensure that the proposed flow doesn't remove any Connections for which there is currently data queued - verifyNoConnectionsWithDataRemoved(existingDataFlow, proposedFlow, controller, flowComparison); + if (!existingFlowEmpty) { + verifyNoConnectionsWithDataRemoved(existingDataFlow, proposedFlow, controller, flowComparison); + } synchronizeFlow(controller, existingDataFlow, proposedFlow, affectedComponents); } finally { // We have to call toExistingSet() here because some of the components that existed in the active set may no longer exist, // so attempting to start them will fail. - final AffectedComponentSet startable = activeSet.toExistingSet().toStartableSet(); - final ComponentSetFilter runningComponentFilter = new RunningComponentSetFilter(proposedFlow.getVersionedDataflow()); - final ComponentSetFilter stoppedComponentFilter = runningComponentFilter.reverse(); - startable.removeComponents(stoppedComponentFilter); - startable.start(); + if (!existingFlowEmpty) { + final AffectedComponentSet startable = activeSet.toExistingSet().toStartableSet(); + + final ComponentSetFilter runningComponentFilter = new RunningComponentSetFilter(proposedFlow.getVersionedDataflow()); + final ComponentSetFilter stoppedComponentFilter = runningComponentFilter.reverse(); + startable.removeComponents(stoppedComponentFilter); + startable.start(); + } } final long millis = System.currentTimeMillis() - start; @@ -661,6 +684,11 @@ public class VersionedFlowSynchronizer implements FlowSynchronizer { final VersionedDataflow dataflow, final AffectedComponentSet affectedComponentSet ) throws FlowAnalysisRuleInstantiationException { + // Guard state in order to be able to read flow.json from before adding the flow analysis rules + if (dataflow.getFlowAnalysisRules() == null) { + return; + } + for (final VersionedFlowAnalysisRule versionedFlowAnalysisRule : dataflow.getFlowAnalysisRules()) { final FlowAnalysisRuleNode existing = controller.getFlowAnalysisRuleNode(versionedFlowAnalysisRule.getInstanceIdentifier()); if (existing == null) { @@ -1201,13 +1229,15 @@ public class VersionedFlowSynchronizer implements FlowSynchronizer { private DataFlow getExistingDataFlow(final FlowController controller) { final FlowManager flowManager = controller.getFlowManager(); + final ProcessGroup root = flowManager.getRootGroup(); // Determine missing components final Set missingComponents = new HashSet<>(); flowManager.getAllControllerServices().stream().filter(ComponentNode::isExtensionMissing).forEach(cs -> missingComponents.add(cs.getIdentifier())); flowManager.getAllReportingTasks().stream().filter(ComponentNode::isExtensionMissing).forEach(r -> missingComponents.add(r.getIdentifier())); flowManager.getAllParameterProviders().stream().filter(ComponentNode::isExtensionMissing).forEach(r -> missingComponents.add(r.getIdentifier())); - flowManager.findAllProcessors(AbstractComponentNode::isExtensionMissing).forEach(p -> missingComponents.add(p.getIdentifier())); + flowManager.getAllFlowRegistryClients().stream().filter(ComponentNode::isExtensionMissing).forEach(c -> missingComponents.add(c.getIdentifier())); + root.findAllProcessors().stream().filter(AbstractComponentNode::isExtensionMissing).forEach(p -> missingComponents.add(p.getIdentifier())); logger.trace("Exporting snippets from controller"); final byte[] existingSnippets = controller.getSnippetManager().export(); diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/service/ControllerServiceLoader.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/service/ControllerServiceLoader.java deleted file mode 100644 index e330f0fc6e..0000000000 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/service/ControllerServiceLoader.java +++ /dev/null @@ -1,188 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.nifi.controller.service; - -import org.apache.nifi.bundle.BundleCoordinate; -import org.apache.nifi.components.PropertyDescriptor; -import org.apache.nifi.controller.FlowController; -import org.apache.nifi.controller.serialization.FlowEncodingVersion; -import org.apache.nifi.controller.serialization.FlowFromDOMFactory; -import org.apache.nifi.encrypt.PropertyEncryptor; -import org.apache.nifi.groups.ProcessGroup; -import org.apache.nifi.logging.LogLevel; -import org.apache.nifi.util.BundleUtils; -import org.apache.nifi.web.api.dto.BundleDTO; -import org.apache.nifi.web.api.dto.ControllerServiceDTO; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.w3c.dom.Element; - -import java.nio.charset.StandardCharsets; -import java.util.Collection; -import java.util.Collections; -import java.util.HashMap; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.UUID; -import java.util.stream.Collectors; - -public class ControllerServiceLoader { - - private static final Logger logger = LoggerFactory.getLogger(ControllerServiceLoader.class); - - public static Map loadControllerServices(final List serviceElements, final FlowController controller, - final ProcessGroup parentGroup, final PropertyEncryptor encryptor, final FlowEncodingVersion encodingVersion) { - - final Map nodeMap = new HashMap<>(); - for (final Element serviceElement : serviceElements) { - final ControllerServiceNode serviceNode = createControllerService(controller, serviceElement, encryptor, encodingVersion); - if (parentGroup == null) { - controller.getFlowManager().addRootControllerService(serviceNode); - } else { - parentGroup.addControllerService(serviceNode); - } - - // We need to clone the node because it will be used in a separate thread below, and - // Element is not thread-safe. - nodeMap.put(serviceNode, (Element) serviceElement.cloneNode(true)); - } - for (final Map.Entry entry : nodeMap.entrySet()) { - configureControllerService(entry.getKey(), entry.getValue(), encryptor, encodingVersion); - } - - return nodeMap; - } - - public static void enableControllerServices(final Map nodeMap, final FlowController controller, - final PropertyEncryptor encryptor, final boolean autoResumeState, final FlowEncodingVersion encodingVersion) { - // Start services - if (autoResumeState) { - final Set nodesToEnable = new HashSet<>(); - - for (final ControllerServiceNode node : nodeMap.keySet()) { - final Element controllerServiceElement = nodeMap.get(node); - - final ControllerServiceDTO dto; - synchronized (controllerServiceElement.getOwnerDocument()) { - dto = FlowFromDOMFactory.getControllerService(controllerServiceElement, encryptor, encodingVersion); - } - - final ControllerServiceState state = ControllerServiceState.valueOf(dto.getState()); - if (state == ControllerServiceState.ENABLED) { - nodesToEnable.add(node); - logger.debug("Will enable Controller Service {}", node); - } else { - logger.debug("Will not enable Controller Service {} because its state is set to {}", node, state); - } - } - - enableControllerServices(nodesToEnable, controller, autoResumeState); - } else { - logger.debug("Will not enable the following Controller Services because 'auto-resume state' flag is false: {}", nodeMap.keySet()); - } - } - - public static void enableControllerServices(final Collection nodesToEnable, final FlowController controller, final boolean autoResumeState) { - // Start services - if (autoResumeState) { - logger.debug("Enabling Controller Services {}", nodesToEnable); - nodesToEnable.forEach(ControllerServiceNode::performValidation); // validate services before attempting to enable them - - controller.getControllerServiceProvider().enableControllerServices(nodesToEnable); - } else { - logger.debug("Will not enable the following Controller Services because 'auto-resume state' flag is false: {}", nodesToEnable); - } - } - - public static ControllerServiceNode cloneControllerService(final FlowController flowController, final ControllerServiceNode controllerService) { - // create a new id for the clone seeded from the original id so that it is consistent in a cluster - final UUID id = UUID.nameUUIDFromBytes(controllerService.getIdentifier().getBytes(StandardCharsets.UTF_8)); - - final ControllerServiceNode clone = flowController.getFlowManager().createControllerService(controllerService.getCanonicalClassName(), id.toString(), - controllerService.getBundleCoordinate(), Collections.emptySet(), false, true, null); - clone.setName(controllerService.getName()); - clone.setComments(controllerService.getComments()); - clone.setBulletinLevel(controllerService.getBulletinLevel()); - - if (controllerService.getProperties() != null) { - Map properties = new HashMap<>(); - for (Map.Entry propEntry : controllerService.getRawPropertyValues().entrySet()) { - properties.put(propEntry.getKey().getName(), propEntry.getValue()); - } - clone.setProperties(properties); - } - - return clone; - } - - private static ControllerServiceNode createControllerService(final FlowController flowController, final Element controllerServiceElement, final PropertyEncryptor encryptor, - final FlowEncodingVersion encodingVersion) { - final ControllerServiceDTO dto = FlowFromDOMFactory.getControllerService(controllerServiceElement, encryptor, encodingVersion); - - BundleCoordinate coordinate; - try { - coordinate = BundleUtils.getCompatibleBundle(flowController.getExtensionManager(), dto.getType(), dto.getBundle()); - } catch (final IllegalStateException e) { - final BundleDTO bundleDTO = dto.getBundle(); - if (bundleDTO == null) { - coordinate = BundleCoordinate.UNKNOWN_COORDINATE; - } else { - coordinate = new BundleCoordinate(bundleDTO.getGroup(), bundleDTO.getArtifact(), bundleDTO.getVersion()); - } - } - - final ControllerServiceNode node = flowController.getFlowManager().createControllerService(dto.getType(), dto.getId(), coordinate, Collections.emptySet(), false, true, null); - node.setName(dto.getName()); - node.setComments(dto.getComments()); - - if (dto.getBulletinLevel() != null) { - node.setBulletinLevel(LogLevel.valueOf(dto.getBulletinLevel())); - } else { - // this situation exists for backward compatibility with nifi 1.16 and earlier where controller services do not have bulletinLevels set in flow.xml/flow.json - // and bulletinLevels are at the WARN level by default - node.setBulletinLevel(LogLevel.WARN); - } - - node.setVersionedComponentId(dto.getVersionedComponentId()); - return node; - } - - private static void configureControllerService(final ControllerServiceNode node, final Element controllerServiceElement, final PropertyEncryptor encryptor, - final FlowEncodingVersion encodingVersion) { - final ControllerServiceDTO dto = FlowFromDOMFactory.getControllerService(controllerServiceElement, encryptor, encodingVersion); - node.pauseValidationTrigger(); - try { - node.setAnnotationData(dto.getAnnotationData()); - final Set sensitiveDynamicPropertyNames = getSensitiveDynamicPropertyNames(dto.getSensitiveDynamicPropertyNames(), node); - node.setProperties(dto.getProperties(), false, sensitiveDynamicPropertyNames); - } finally { - node.resumeValidationTrigger(); - } - } - - private static Set getSensitiveDynamicPropertyNames(final Set parsedSensitivePropertyNames, final ControllerServiceNode controllerServiceNode) { - final Set sensitivePropertyNames = parsedSensitivePropertyNames == null ? Collections.emptySet() : parsedSensitivePropertyNames; - return sensitivePropertyNames.stream().filter( - propertyName -> { - final PropertyDescriptor propertyDescriptor = controllerServiceNode.getPropertyDescriptor(propertyName); - return propertyDescriptor.isDynamic(); - } - ).collect(Collectors.toSet()); - } -} diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/fingerprint/FingerprintException.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/fingerprint/FingerprintException.java deleted file mode 100644 index 7c3b9bc5f1..0000000000 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/fingerprint/FingerprintException.java +++ /dev/null @@ -1,40 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.nifi.fingerprint; - -/** - */ -public class FingerprintException extends RuntimeException { - - private static final long serialVersionUID = 189234798327894327L; - - public FingerprintException(Throwable cause) { - super(cause); - } - - public FingerprintException(String message, Throwable cause) { - super(message, cause); - } - - public FingerprintException(String message) { - super(message); - } - - public FingerprintException() { - } - -} diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/fingerprint/FingerprintFactory.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/fingerprint/FingerprintFactory.java deleted file mode 100644 index 1a2402483e..0000000000 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/fingerprint/FingerprintFactory.java +++ /dev/null @@ -1,1050 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.nifi.fingerprint; - -import org.apache.commons.lang3.StringUtils; -import org.apache.nifi.bundle.BundleCoordinate; -import org.apache.nifi.components.ConfigurableComponent; -import org.apache.nifi.components.PropertyDescriptor; -import org.apache.nifi.controller.FlowController; -import org.apache.nifi.controller.serialization.FlowEncodingVersion; -import org.apache.nifi.controller.serialization.FlowFromDOMFactory; -import org.apache.nifi.encrypt.PropertyEncryptor; -import org.apache.nifi.encrypt.SensitiveValueEncoder; -import org.apache.nifi.nar.ExtensionManager; -import org.apache.nifi.util.BundleUtils; -import org.apache.nifi.util.DomUtils; -import org.apache.nifi.util.LoggingXmlParserErrorHandler; -import org.apache.nifi.web.api.dto.BundleDTO; -import org.apache.nifi.web.api.dto.ControllerServiceDTO; -import org.apache.nifi.web.api.dto.FlowAnalysisRuleDTO; -import org.apache.nifi.web.api.dto.FlowRegistryClientDTO; -import org.apache.nifi.web.api.dto.ParameterProviderDTO; -import org.apache.nifi.web.api.dto.ReportingTaskDTO; -import org.apache.nifi.xml.processing.ProcessingException; -import org.apache.nifi.xml.processing.parsers.StandardDocumentProvider; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.w3c.dom.Document; -import org.w3c.dom.Element; -import org.w3c.dom.Node; -import org.w3c.dom.NodeList; -import org.xml.sax.ErrorHandler; - -import javax.xml.XMLConstants; -import javax.xml.validation.Schema; -import javax.xml.validation.SchemaFactory; -import java.io.ByteArrayInputStream; -import java.util.ArrayList; -import java.util.Collections; -import java.util.Comparator; -import java.util.List; -import java.util.Map; -import java.util.SortedMap; -import java.util.TreeMap; -import java.util.stream.Stream; - -/** - *

Creates a fingerprint of a flow.xml. The order of elements or attributes in the flow.xml does not influence the fingerprint generation. - * - *

Only items in the flow.xml that influence the processing of data are incorporated into the fingerprint. - * Examples of items involved in the fingerprint are: processor IDs, processor relationships, and processor properties. - * Examples of items not involved in the fingerprint are: items in the processor "comments" tabs, position information, flow controller settings, and counters. - * - *

The determination for making items into the fingerprint is whether we can - * easily change the setting in order to inherit the cluster's flow. - * For example, if the component has to be stopped to apply the change and started again, - * then the item should be included in a fingerprint. - */ -public class FingerprintFactory { - - /* - * Developer Note: This class should be changed with care and coordinated - * with all classes that use fingerprinting. Improper coordination may - * lead to orphaning flow files, especially when flows are reloaded in a - * clustered environment. - */ - // no fingerprint value - public static final String NO_VALUE = "NO_VALUE"; - - static final String FLOW_CONFIG_XSD = "/FlowConfiguration.xsd"; - private static final String ENCRYPTED_VALUE_PREFIX = "enc{"; - private static final String ENCRYPTED_VALUE_SUFFIX = "}"; - private final PropertyEncryptor encryptor; - private final Schema schema; - private final ExtensionManager extensionManager; - private final SensitiveValueEncoder sensitiveValueEncoder; - - private static final Logger logger = LoggerFactory.getLogger(FingerprintFactory.class); - - public FingerprintFactory(final PropertyEncryptor encryptor, final ExtensionManager extensionManager, final SensitiveValueEncoder sensitiveValueEncoder) { - this.encryptor = encryptor; - this.extensionManager = extensionManager; - this.sensitiveValueEncoder = sensitiveValueEncoder; - - final SchemaFactory schemaFactory = SchemaFactory.newInstance(XMLConstants.W3C_XML_SCHEMA_NS_URI); - try { - schema = schemaFactory.newSchema(FingerprintFactory.class.getResource(FLOW_CONFIG_XSD)); - } catch (final Exception e) { - throw new RuntimeException("Failed to parse schema for file flow configuration.", e); - } - } - - /** - * Creates a fingerprint of a flow. The order of elements or attributes in the flow does not influence the fingerprint generation. - * This method does not accept a FlowController, which means that Processors cannot be created in order to verify default property - * values, etc. As a result, if Flow A and Flow B are fingerprinted and Flow B, for instance, contains a property with a default value - * that is not present in Flow A, then the two will have different fingerprints. - * - * @param flowBytes the flow represented as bytes - * @return a generated fingerprint - * @throws FingerprintException if the fingerprint failed to be generated - */ - public synchronized String createFingerprint(final byte[] flowBytes) throws FingerprintException { - return createFingerprint(flowBytes, null); - } - - /** - * Creates a fingerprint of a flow. The order of elements or attributes in the flow does not influence the fingerprint generation. - * - * @param flowBytes the flow represented as bytes - * @param controller the controller - * @return a generated fingerprint - * @throws FingerprintException if the fingerprint failed to be generated - */ - public synchronized String createFingerprint(final byte[] flowBytes, final FlowController controller) throws FingerprintException { - return createFingerprint(parseFlow(flowBytes), controller); - } - - /** - * Creates a fingerprint from an XML document representing the flow.xml. - * - * @param flowDoc the DOM - * @return the fingerprint - */ - public synchronized String createFingerprint(final Document flowDoc, final FlowController controller) { - if (flowDoc == null) { - return ""; - } - - // builder to hold fingerprint state - final StringBuilder fingerprintBuilder = new StringBuilder(); - - // add flow controller fingerprint - final Element flowControllerElem = flowDoc.getDocumentElement(); - if (flowControllerElem == null) { - logger.warn("Unable to create fingerprint because no 'flowController' element found in XML."); - return ""; - } - - final FlowEncodingVersion encodingVersion = FlowEncodingVersion.parse(flowControllerElem); - - addFlowControllerFingerprint(fingerprintBuilder, flowControllerElem, controller, encodingVersion); - - return fingerprintBuilder.toString(); - } - - /** - * Parse the given flow.xml bytes into a Document instance. - * - * @param flow a flow - * @return the DOM - * @throws FingerprintException if the flow could not be parsed - */ - private Document parseFlow(final byte[] flow) throws FingerprintException { - if (flow == null || flow.length == 0) { - return null; - } - - try { - final ErrorHandler errorHandler = new LoggingXmlParserErrorHandler("Flow Configuration", logger); - final StandardDocumentProvider documentProvider = new StandardDocumentProvider(); - documentProvider.setSchema(schema); - documentProvider.setNamespaceAware(true); - documentProvider.setErrorHandler(errorHandler); - - return documentProvider.parse(new ByteArrayInputStream(flow)); - } catch (final ProcessingException e) { - throw new FingerprintException("Flow Parsing failed", e); - } - } - - private StringBuilder addFlowControllerFingerprint(final StringBuilder builder, final Element flowControllerElem, final FlowController controller, final FlowEncodingVersion encodingVersion) { - // registries - final Element registriesElement = DomUtils.getChild(flowControllerElem, "registries"); - if (registriesElement == null) { - builder.append("NO_VALUE"); - } else { - final List flowRegistryElems = DomUtils.getChildElementsByTagName(registriesElement, "flowRegistry"); - if (flowRegistryElems.isEmpty()) { - builder.append("NO_VALUE"); - } else { - final List registryClientDtos = new ArrayList<>(); - for (final Element flowRegistryElement : flowRegistryElems) { - registryClientDtos.add(FlowFromDOMFactory.getFlowRegistryClient(flowRegistryElement, encryptor, encodingVersion)); - } - - registryClientDtos.sort(Comparator.comparing(FlowRegistryClientDTO::getId)); - - for (final FlowRegistryClientDTO registryClientDto : registryClientDtos) { - addFlowRegistryFingerprint(builder, registryClientDto); - } - } - } - - final Element contextsElement = DomUtils.getChild(flowControllerElem, "parameterContexts"); - if (contextsElement == null) { - builder.append("NO_PARAMETER_CONTEXTS"); - } else { - final List parameterContextElements = DomUtils.getChildElementsByTagName(contextsElement, "parameterContext"); - if (parameterContextElements.isEmpty()) { - builder.append("NO_PARAMETER_CONTEXTS"); - } else { - orderByChildElement(parameterContextElements, "id"); - - for (final Element parameterContextElement : parameterContextElements) { - addParameterContext(builder, parameterContextElement); - } - } - } - - // root group - final Element rootGroupElem = (Element) DomUtils.getChildNodesByTagName(flowControllerElem, "rootGroup").item(0); - addProcessGroupFingerprint(builder, rootGroupElem, encodingVersion); - - final Element controllerServicesElem = DomUtils.getChild(flowControllerElem, "controllerServices"); - if (controllerServicesElem != null) { - final List serviceDtos = new ArrayList<>(); - for (final Element serviceElem : DomUtils.getChildElementsByTagName(controllerServicesElem, "controllerService")) { - final ControllerServiceDTO dto = FlowFromDOMFactory.getControllerService(serviceElem, encryptor, encodingVersion); - serviceDtos.add(dto); - } - - Collections.sort(serviceDtos, new Comparator() { - @Override - public int compare(final ControllerServiceDTO o1, final ControllerServiceDTO o2) { - if (o1 == null && o2 == null) { - return 0; - } - if (o1 == null && o2 != null) { - return 1; - } - if (o1 != null && o2 == null) { - return -1; - } - - return o1.getId().compareTo(o2.getId()); - } - }); - - for (final ControllerServiceDTO dto : serviceDtos) { - addControllerServiceFingerprint(builder, dto); - } - } - - final Element reportingTasksElem = DomUtils.getChild(flowControllerElem, "reportingTasks"); - if (reportingTasksElem != null) { - final List reportingTaskDtos = new ArrayList<>(); - for (final Element taskElem : DomUtils.getChildElementsByTagName(reportingTasksElem, "reportingTask")) { - final ReportingTaskDTO dto = FlowFromDOMFactory.getReportingTask(taskElem, encryptor, encodingVersion); - reportingTaskDtos.add(dto); - } - - Collections.sort(reportingTaskDtos, new Comparator() { - @Override - public int compare(final ReportingTaskDTO o1, final ReportingTaskDTO o2) { - if (o1 == null && o2 == null) { - return 0; - } - if (o1 == null && o2 != null) { - return 1; - } - if (o1 != null && o2 == null) { - return -1; - } - - return o1.getId().compareTo(o2.getId()); - } - }); - - for (final ReportingTaskDTO dto : reportingTaskDtos) { - addReportingTaskFingerprint(builder, dto); - } - } - - final Element flowAnalysisRulesElem = DomUtils.getChild(flowControllerElem, "flowAnalysisRules"); - if (flowAnalysisRulesElem != null) { - final List flowAnalysisRuleDtos = new ArrayList<>(); - for (final Element ruleElem : DomUtils.getChildElementsByTagName(flowAnalysisRulesElem, "flowAnalysisRule")) { - final FlowAnalysisRuleDTO dto = FlowFromDOMFactory.getFlowAnalysisRule(ruleElem, encryptor, encodingVersion); - flowAnalysisRuleDtos.add(dto); - } - - Collections.sort(flowAnalysisRuleDtos, new Comparator() { - @Override - public int compare(final FlowAnalysisRuleDTO o1, final FlowAnalysisRuleDTO o2) { - if (o1 == null && o2 == null) { - return 0; - } - if (o1 == null && o2 != null) { - return 1; - } - if (o1 != null && o2 == null) { - return -1; - } - - return o1.getId().compareTo(o2.getId()); - } - }); - - for (final FlowAnalysisRuleDTO dto : flowAnalysisRuleDtos) { - addFlowAnalysisRuleFingerprint(builder, dto); - } - } - - final Element parameterProvidersElem = DomUtils.getChild(flowControllerElem, "parameterProviders"); - if (parameterProvidersElem != null) { - final List parameterProviderDtos = new ArrayList<>(); - for (final Element taskElem : DomUtils.getChildElementsByTagName(parameterProvidersElem, "parameterProvider")) { - final ParameterProviderDTO dto = FlowFromDOMFactory.getParameterProvider(taskElem, encryptor, encodingVersion); - parameterProviderDtos.add(dto); - } - - Collections.sort(parameterProviderDtos, Comparator.comparing(ParameterProviderDTO::getId)); - - for (final ParameterProviderDTO dto : parameterProviderDtos) { - addParameterProviderFingerprint(builder, dto); - } - } - - return builder; - } - - private void orderByChildElement(final List toSort, final String childTagName) { - toSort.sort((a, b) -> { - final String valueA = DomUtils.getChildText(a, childTagName); - final String valueB = DomUtils.getChildText(b, childTagName); - return valueA.compareTo(valueB); - }); - } - - private StringBuilder addParameterContext(final StringBuilder builder, final Element parameterContextElement) { - appendFirstValue(builder, DomUtils.getChildNodesByTagName(parameterContextElement, "id")); - appendFirstValue(builder, DomUtils.getChildNodesByTagName(parameterContextElement, "name")); - appendFirstValue(builder, DomUtils.getChildNodesByTagName(parameterContextElement, "description")); - - final List parameterElements = DomUtils.getChildElementsByTagName(parameterContextElement, "parameter"); - if (parameterElements == null || parameterElements.isEmpty()) { - builder.append("NO_PARAMETERS"); - } else { - orderByChildElement(parameterElements, "name"); - - for (final Element parameterElement : parameterElements) { - addParameter(builder, parameterElement); - } - } - - final List inheritedParameterContexts = DomUtils.getChildElementsByTagName(parameterContextElement, "inheritedParameterContextId"); - if (inheritedParameterContexts == null || inheritedParameterContexts.isEmpty()) { - builder.append("NO_INHERITED_PARAMETER_CONTEXT_IDS"); - } else { - for (final Element inheritedParameterContextId : inheritedParameterContexts) { - builder.append(inheritedParameterContextId.getTextContent()); - } - } - final String parameterProviderId = DomUtils.getChildText(parameterContextElement, "parameterProviderId"); - builder.append(parameterProviderId == null ? "NO_PARAMETER_PROVIDER_ID" : parameterProviderId); - final String parameterGroupName = DomUtils.getChildText(parameterContextElement, "parameterGroupName"); - builder.append(parameterGroupName == null ? "NO_PARAMETER_GROUP_NAME" : parameterGroupName); - final String isSynchronized = DomUtils.getChildText(parameterContextElement, "isSynchronized"); - builder.append(isSynchronized == null ? "NO_PARAMETER_IS_SYNCHRONIZED" : isSynchronized); - - return builder; - } - - private void addParameter(final StringBuilder builder, final Element parameterElement) { - Stream.of("name", "description", "sensitive", "provided").forEach(elementName -> appendFirstValue(builder, DomUtils.getChildNodesByTagName(parameterElement, elementName))); - - final String value = DomUtils.getChildText(parameterElement, "value"); - if (value == null) { - builder.append("NO_VALUE"); - return; - } - - // append value - if (isEncrypted(value)) { - // Get a secure, deterministic, loggable representation of this value - builder.append(getLoggableRepresentationOfSensitiveValue(value)); - } else { - builder.append(getValue(value, NO_VALUE)); - } - - } - - private void addFlowRegistryFingerprint(final StringBuilder builder, final FlowRegistryClientDTO dto) { - builder.append(dto.getId()); - builder.append(dto.getName()); - builder.append(dto.getDeprecated()); - builder.append(dto.getUri()); - - builder.append(dto.getType()); - addBundleFingerprint(builder, dto.getBundle()); - - // get the temp instance of the FlowRegistryClient so that we know the default property values - final BundleCoordinate coordinate = getCoordinate(dto.getType(), dto.getBundle()); - final ConfigurableComponent configurableComponent = extensionManager.getTempComponent(dto.getType(), coordinate); - if (configurableComponent == null) { - logger.warn("Unable to get FlowRegistryClient of type {}; its default properties will be fingerprinted instead of being ignored.", dto.getType()); - } - addPropertiesFingerprint(builder, configurableComponent, dto.getProperties()); - } - - StringBuilder addProcessGroupFingerprint(final StringBuilder builder, final Element processGroupElem, final FlowEncodingVersion encodingVersion) throws FingerprintException { - // id - appendFirstValue(builder, DomUtils.getChildNodesByTagName(processGroupElem, "id")); - appendFirstValue(builder, DomUtils.getChildNodesByTagName(processGroupElem, "versionedComponentId")); - appendFirstValue(builder, DomUtils.getChildNodesByTagName(processGroupElem, "parameterContextId")); - appendFirstValue(builder, DomUtils.getChildNodesByTagName(processGroupElem, "flowfileConcurrency")); - appendFirstValue(builder, DomUtils.getChildNodesByTagName(processGroupElem, "flowfileOutboundPolicy")); - appendFirstValue(builder, DomUtils.getChildNodesByTagName(processGroupElem, "defaultFlowFileExpiration")); - appendFirstValue(builder, DomUtils.getChildNodesByTagName(processGroupElem, "defaultBackPressureObjectThreshold")); - appendFirstValue(builder, DomUtils.getChildNodesByTagName(processGroupElem, "defaultBackPressureDataSizeThreshold")); - appendFirstValue(builder, DomUtils.getChildNodesByTagName(processGroupElem, "logFileSuffix")); - - final Element versionControlInfo = DomUtils.getChild(processGroupElem, "versionControlInformation"); - if (versionControlInfo == null) { - builder.append("NO_VERSION_CONTROL_INFORMATION"); - } else { - appendFirstValue(builder, DomUtils.getChildNodesByTagName(versionControlInfo, "registryId")); - appendFirstValue(builder, DomUtils.getChildNodesByTagName(versionControlInfo, "bucketId")); - appendFirstValue(builder, DomUtils.getChildNodesByTagName(versionControlInfo, "flowId")); - appendFirstValue(builder, DomUtils.getChildNodesByTagName(versionControlInfo, "version")); - } - - // processors - final List processorElems = DomUtils.getChildElementsByTagName(processGroupElem, "processor"); - processorElems.sort(getIdsComparator()); - for (final Element processorElem : processorElems) { - addFlowFileProcessorFingerprint(builder, processorElem); - } - - // input ports - final NodeList inputPortElems = DomUtils.getChildNodesByTagName(processGroupElem, "inputPort"); - final List sortedInputPortElems = sortElements(inputPortElems, getIdsComparator()); - for (final Element inputPortElem : sortedInputPortElems) { - addPortFingerprint(builder, inputPortElem); - } - - // labels - final NodeList labelElems = DomUtils.getChildNodesByTagName(processGroupElem, "label"); - final List sortedLabels = sortElements(labelElems, getIdsComparator()); - for (final Element labelElem : sortedLabels) { - addLabelFingerprint(builder, labelElem); - } - - // output ports - final NodeList outputPortElems = DomUtils.getChildNodesByTagName(processGroupElem, "outputPort"); - final List sortedOutputPortElems = sortElements(outputPortElems, getIdsComparator()); - for (final Element outputPortElem : sortedOutputPortElems) { - addPortFingerprint(builder, outputPortElem); - } - - // process groups - final NodeList nestedProcessGroupElems = DomUtils.getChildNodesByTagName(processGroupElem, "processGroup"); - final List sortedNestedProcessGroupElems = sortElements(nestedProcessGroupElems, getIdsComparator()); - for (final Element nestedProcessGroupElem : sortedNestedProcessGroupElems) { - addProcessGroupFingerprint(builder, nestedProcessGroupElem, encodingVersion); - } - - // remote process groups - final NodeList remoteProcessGroupElems = DomUtils.getChildNodesByTagName(processGroupElem, "remoteProcessGroup"); - final List sortedRemoteProcessGroupElems = sortElements(remoteProcessGroupElems, getIdsComparator()); - for (final Element remoteProcessGroupElem : sortedRemoteProcessGroupElems) { - addRemoteProcessGroupFingerprint(builder, remoteProcessGroupElem); - } - - // connections - final NodeList connectionElems = DomUtils.getChildNodesByTagName(processGroupElem, "connection"); - final List sortedConnectionElems = sortElements(connectionElems, getIdsComparator()); - for (final Element connectionElem : sortedConnectionElems) { - addConnectionFingerprint(builder, connectionElem); - } - - // funnel - final NodeList funnelElems = DomUtils.getChildNodesByTagName(processGroupElem, "funnel"); - final List sortedFunnelElems = sortElements(funnelElems, getIdsComparator()); - for (final Element funnelElem : sortedFunnelElems) { - addFunnelFingerprint(builder, funnelElem); - } - - final NodeList controllerServiceElems = DomUtils.getChildNodesByTagName(processGroupElem, "controllerService"); - final List sortedControllerServiceElems = sortElements(controllerServiceElems, getIdsComparator()); - for (final Element controllerServiceElem : sortedControllerServiceElems) { - final ControllerServiceDTO dto = FlowFromDOMFactory.getControllerService(controllerServiceElem, encryptor, encodingVersion); - addControllerServiceFingerprint(builder, dto); - } - - // add variables - final NodeList variableElems = DomUtils.getChildNodesByTagName(processGroupElem, "variable"); - final List sortedVarList = sortElements(variableElems, getVariableNameComparator()); - for (final Element varElem : sortedVarList) { - addVariableFingerprint(builder, varElem); - } - - return builder; - } - - private void addVariableFingerprint(final StringBuilder builder, final Element variableElement) { - final String variableName = variableElement.getAttribute("name"); - final String variableValue = variableElement.getAttribute("value"); - builder.append(variableName).append("=").append(variableValue); - } - - private StringBuilder addFlowFileProcessorFingerprint(final StringBuilder builder, final Element processorElem) throws FingerprintException { - // id - appendFirstValue(builder, DomUtils.getChildNodesByTagName(processorElem, "id")); - appendFirstValue(builder, DomUtils.getChildNodesByTagName(processorElem, "versionedComponentId")); - // class - final NodeList childNodes = DomUtils.getChildNodesByTagName(processorElem, "class"); - final String className = childNodes.item(0).getTextContent(); - appendFirstValue(builder, childNodes); - // annotation data - appendFirstValue(builder, DomUtils.getChildNodesByTagName(processorElem, "annotationData")); - - // get the bundle details if possible - final BundleDTO bundle = FlowFromDOMFactory.getBundle(DomUtils.getChild(processorElem, "bundle")); - addBundleFingerprint(builder, bundle); - - // max concurrent tasks - appendFirstValue(builder, DomUtils.getChildNodesByTagName(processorElem, "maxConcurrentTasks")); - // scheduling period - appendFirstValue(builder, DomUtils.getChildNodesByTagName(processorElem, "schedulingPeriod")); - // penalization period - appendFirstValue(builder, DomUtils.getChildNodesByTagName(processorElem, "penalizationPeriod")); - // yield period - appendFirstValue(builder, DomUtils.getChildNodesByTagName(processorElem, "yieldPeriod")); - // bulletin level - appendFirstValue(builder, DomUtils.getChildNodesByTagName(processorElem, "bulletinLevel")); - // loss tolerant - appendFirstValue(builder, DomUtils.getChildNodesByTagName(processorElem, "lossTolerant")); - // scheduling strategy - appendFirstValue(builder, DomUtils.getChildNodesByTagName(processorElem, "schedulingStrategy")); - // execution node - appendFirstValue(builder, DomUtils.getChildNodesByTagName(processorElem, "executionNode")); - // run duration nanos - appendFirstValue(builder, DomUtils.getChildNodesByTagName(processorElem, "runDurationNanos")); - // retry count - appendFirstValue(builder, DomUtils.getChildNodesByTagName(processorElem, "retryCount")); - // backoff mechanism - appendFirstValue(builder, DomUtils.getChildNodesByTagName(processorElem, "backoffMechanism")); - // max backoff period - appendFirstValue(builder, DomUtils.getChildNodesByTagName(processorElem, "maxBackoffPeriod")); - - // get the temp instance of the Processor so that we know the default property values - final BundleCoordinate coordinate = getCoordinate(className, bundle); - final ConfigurableComponent configurableComponent = extensionManager.getTempComponent(className, coordinate); - if (configurableComponent == null) { - logger.warn("Unable to get Processor of type {}; its default properties will be fingerprinted instead of being ignored.", className); - } - - // properties - final NodeList propertyElems = DomUtils.getChildNodesByTagName(processorElem, "property"); - final List sortedPropertyElems = sortElements(propertyElems, getProcessorPropertiesComparator()); - for (final Element propertyElem : sortedPropertyElems) { - final String propName = DomUtils.getChildElementsByTagName(propertyElem, "name").get(0).getTextContent(); - String propValue = getFirstValue(DomUtils.getChildNodesByTagName(propertyElem, "value"), null); - addPropertyFingerprint(builder, configurableComponent, propName, propValue); - } - - final NodeList autoTerminateElems = DomUtils.getChildNodesByTagName(processorElem, "autoTerminatedRelationship"); - final List sortedAutoTerminateElems = sortElements(autoTerminateElems, getElementTextComparator()); - for (final Element autoTerminateElem : sortedAutoTerminateElems) { - builder.append(autoTerminateElem.getTextContent()); - } - - final NodeList retriedRelationshipsElems = DomUtils.getChildNodesByTagName(processorElem, "retriedRelationships"); - final List sortedRetriedRelationshipsElems = sortElements(retriedRelationshipsElems, getElementTextComparator()); - for (final Element retriedRelationshipElem : sortedRetriedRelationshipsElems) { - builder.append(retriedRelationshipElem.getTextContent()); - } - - return builder; - } - - private StringBuilder addPropertyFingerprint(final StringBuilder builder, final ConfigurableComponent component, final String propName, final String propValue) throws FingerprintException { - // If we have a component to use, first determine if the value given is the default value for the specified property. - // If so, we do not add the property to the fingerprint. - // We do this because if a component is updated to add a new property, whenever we connect to the cluster, we have issues because - // the Cluster Coordinator's flow comes from disk, where the flow.xml doesn't have the new property but our FlowController does have the new property. - // This causes the fingerprints not to match. As a result, we just ignore default values, and this resolves the issue. - - if (component != null) { - final PropertyDescriptor descriptor = component.getPropertyDescriptor(propName); - if (descriptor != null && propValue != null && propValue.equals(descriptor.getDefaultValue())) { - return builder; - } - } - - // check if there is a value - if (propValue == null) { - return builder; - } - - // append name - builder.append(propName).append("="); - - // append value - if (isEncrypted(propValue)) { - // Get a secure, deterministic, loggable representation of this value - builder.append(getLoggableRepresentationOfSensitiveValue(propValue)); - } else { - builder.append(getValue(propValue, NO_VALUE)); - } - - return builder; - } - - /** - * Returns a securely-derived, deterministic value from the provided encrypted property - * value. This is because the flow fingerprint is displayed in the log if NiFi has - * trouble inheriting a flow, so the sensitive value should not be disclosed through the - * log. However, the equality or difference of the sensitive value can influence in the - * inheritability of the flow, so it cannot be ignored completely. - *

- * The specific derivation process is unimportant as long as it is a salted, - * cryptographically-secure hash function with an iteration cost sufficient for password - * storage in other applications. - * - * @param encryptedPropertyValue the encrypted property value - * @return a deterministic string value which represents this input but is safe to print in a log - */ - private String getLoggableRepresentationOfSensitiveValue(String encryptedPropertyValue) { - final String plaintextValue = decrypt(encryptedPropertyValue); - return sensitiveValueEncoder.getEncoded(plaintextValue); - } - - private StringBuilder addPortFingerprint(final StringBuilder builder, final Element portElem) throws FingerprintException { - // id - appendFirstValue(builder, DomUtils.getChildNodesByTagName(portElem, "id")); - appendFirstValue(builder, DomUtils.getChildNodesByTagName(portElem, "versionedComponentId")); - appendFirstValue(builder, DomUtils.getChildNodesByTagName(portElem, "name")); - appendFirstValue(builder, DomUtils.getChildNodesByTagName(portElem, "allowRemoteAccess")); - - // user access control - final NodeList userAccessControlNodeList = DomUtils.getChildNodesByTagName(portElem, "userAccessControl"); - if (userAccessControlNodeList == null || userAccessControlNodeList.getLength() == 0) { - builder.append("NO_USER_ACCESS_CONTROL"); - } else { - final List sortedAccessControl = new ArrayList<>(); - for (int i = 0; i < userAccessControlNodeList.getLength(); i++) { - sortedAccessControl.add(userAccessControlNodeList.item(i).getTextContent()); - } - Collections.sort(sortedAccessControl); - for (final String user : sortedAccessControl) { - builder.append(user); - } - } - - // group access control - final NodeList groupAccessControlNodeList = DomUtils.getChildNodesByTagName(portElem, "groupAccessControl"); - if (groupAccessControlNodeList == null || groupAccessControlNodeList.getLength() == 0) { - builder.append("NO_GROUP_ACCESS_CONTROL"); - } else { - final List sortedAccessControl = new ArrayList<>(); - for (int i = 0; i < groupAccessControlNodeList.getLength(); i++) { - sortedAccessControl.add(groupAccessControlNodeList.item(i).getTextContent()); - } - - Collections.sort(sortedAccessControl); - for (final String user : sortedAccessControl) { - builder.append(user); - } - } - - return builder; - } - - private StringBuilder addLabelFingerprint(final StringBuilder builder, final Element labelElem) { - appendFirstValue(builder, DomUtils.getChildNodesByTagName(labelElem, "id")); - appendFirstValue(builder, DomUtils.getChildNodesByTagName(labelElem, "versionedComponentId")); - appendFirstValue(builder, DomUtils.getChildNodesByTagName(labelElem, "value")); - return builder; - } - - private StringBuilder addRemoteProcessGroupFingerprint(final StringBuilder builder, final Element remoteProcessGroupElem) throws FingerprintException { - - for (String tagName : new String[]{"id", "versionedComponentId", "urls", "networkInterface", "timeout", "yieldPeriod", - "transportProtocol", "proxyHost", "proxyPort", "proxyUser", "proxyPassword"}) { - final String value = getFirstValue(DomUtils.getChildNodesByTagName(remoteProcessGroupElem, tagName)); - if (isEncrypted(value)) { - builder.append(getLoggableRepresentationOfSensitiveValue(value)); - } else { - builder.append(value); - } - } - - final NodeList inputPortList = DomUtils.getChildNodesByTagName(remoteProcessGroupElem, "inputPort"); - final NodeList outputPortList = DomUtils.getChildNodesByTagName(remoteProcessGroupElem, "outputPort"); - - final Comparator portComparator = new Comparator() { - @Override - public int compare(final Element o1, final Element o2) { - if (o1 == null && o2 == null) { - return 0; - } - if (o1 == null) { - return 1; - } - if (o2 == null) { - return -1; - } - - NodeList nameList1 = DomUtils.getChildNodesByTagName(o1, "name"); - NodeList nameList2 = DomUtils.getChildNodesByTagName(o2, "name"); - - if (nameList1.getLength() == 0 && nameList2.getLength() == 0) { - return 0; - } - if (nameList1.getLength() == 0) { - return 1; - } - if (nameList2.getLength() == 0) { - return -1; - } - - return nameList1.item(0).getTextContent().compareTo(nameList2.item(0).getTextContent()); - } - }; - - final List sortedInputPorts = new ArrayList<>(inputPortList.getLength()); - for (int i = 0; i < inputPortList.getLength(); i++) { - sortedInputPorts.add((Element) inputPortList.item(i)); - } - Collections.sort(sortedInputPorts, portComparator); - - final List sortedOutputPorts = new ArrayList<>(outputPortList.getLength()); - for (int i = 0; i < outputPortList.getLength(); i++) { - sortedOutputPorts.add((Element) outputPortList.item(i)); - } - Collections.sort(sortedOutputPorts, portComparator); - - for (final Element inputPortElement : sortedInputPorts) { - addRemoteGroupPortFingerprint(builder, inputPortElement); - } - - for (final Element outputPortElement : sortedOutputPorts) { - addRemoteGroupPortFingerprint(builder, outputPortElement); - } - - return builder; - } - - private StringBuilder addRemoteGroupPortFingerprint(final StringBuilder builder, final Element remoteGroupPortElement) { - for (final String childName : new String[]{"id", "targetId", "versionedComponentId", "maxConcurrentTasks", "useCompression", "batchCount", "batchSize", "batchDuration"}) { - appendFirstValue(builder, DomUtils.getChildNodesByTagName(remoteGroupPortElement, childName)); - } - - return builder; - } - - private StringBuilder addConnectionFingerprint(final StringBuilder builder, final Element connectionElem) throws FingerprintException { - // id - appendFirstValue(builder, DomUtils.getChildNodesByTagName(connectionElem, "id")); - appendFirstValue(builder, DomUtils.getChildNodesByTagName(connectionElem, "versionedComponentId")); - // source id - appendFirstValue(builder, DomUtils.getChildNodesByTagName(connectionElem, "sourceId")); - // source group id - appendFirstValue(builder, DomUtils.getChildNodesByTagName(connectionElem, "sourceGroupId")); - // source type - appendFirstValue(builder, DomUtils.getChildNodesByTagName(connectionElem, "sourceType")); - // destination id - appendFirstValue(builder, DomUtils.getChildNodesByTagName(connectionElem, "destinationId")); - // destination group id - appendFirstValue(builder, DomUtils.getChildNodesByTagName(connectionElem, "destinationGroupId")); - // destination type - appendFirstValue(builder, DomUtils.getChildNodesByTagName(connectionElem, "destinationType")); - - appendFirstValue(builder, DomUtils.getChildNodesByTagName(connectionElem, "name")); - - appendFirstValue(builder, DomUtils.getChildNodesByTagName(connectionElem, "loadBalanceStrategy")); - appendFirstValue(builder, DomUtils.getChildNodesByTagName(connectionElem, "partitioningAttribute")); - appendFirstValue(builder, DomUtils.getChildNodesByTagName(connectionElem, "loadBalanceCompression")); - - // relationships - final NodeList relationshipElems = DomUtils.getChildNodesByTagName(connectionElem, "relationship"); - final List sortedRelationshipElems = sortElements(relationshipElems, getElementTextComparator()); - for (final Element relationshipElem : sortedRelationshipElems) { - builder.append(getValue(relationshipElem, NO_VALUE)); - } - - return builder; - } - - private StringBuilder addFunnelFingerprint(final StringBuilder builder, final Element funnelElem) throws FingerprintException { - // id - appendFirstValue(builder, DomUtils.getChildNodesByTagName(funnelElem, "id")); - appendFirstValue(builder, DomUtils.getChildNodesByTagName(funnelElem, "versionedComponentId")); - return builder; - } - - private void addControllerServiceFingerprint(final StringBuilder builder, final ControllerServiceDTO dto) { - builder.append(dto.getId()); - builder.append(dto.getVersionedComponentId()); - builder.append(dto.getType()); - builder.append(dto.getName()); - - addBundleFingerprint(builder, dto.getBundle()); - - builder.append(dto.getComments()); - builder.append(dto.getBulletinLevel()); - builder.append(dto.getAnnotationData()); - builder.append(dto.getState()); - - // get the temp instance of the ControllerService so that we know the default property values - final BundleCoordinate coordinate = getCoordinate(dto.getType(), dto.getBundle()); - final ConfigurableComponent configurableComponent = extensionManager.getTempComponent(dto.getType(), coordinate); - if (configurableComponent == null) { - logger.warn("Unable to get ControllerService of type {}; its default properties will be fingerprinted instead of being ignored.", dto.getType()); - } - - addPropertiesFingerprint(builder, configurableComponent, dto.getProperties()); - } - - private void addPropertiesFingerprint(final StringBuilder builder, final ConfigurableComponent component, final Map properties) { - if (properties == null) { - builder.append("NO_PROPERTIES"); - } else { - final SortedMap sortedProps = new TreeMap<>(properties); - for (final Map.Entry entry : sortedProps.entrySet()) { - addPropertyFingerprint(builder, component, entry.getKey(), entry.getValue()); - } - } - } - - private void addBundleFingerprint(final StringBuilder builder, final BundleDTO bundle) { - if (bundle != null) { - builder.append(bundle.getGroup()); - builder.append(bundle.getArtifact()); - builder.append(bundle.getVersion()); - } else { - builder.append("MISSING_BUNDLE"); - } - } - - private BundleCoordinate getCoordinate(final String type, final BundleDTO dto) { - BundleCoordinate coordinate; - try { - coordinate = BundleUtils.getCompatibleBundle(extensionManager, type, dto); - } catch (final IllegalStateException e) { - if (dto == null) { - coordinate = BundleCoordinate.UNKNOWN_COORDINATE; - } else { - coordinate = new BundleCoordinate(dto.getGroup(), dto.getArtifact(), dto.getVersion()); - } - } - return coordinate; - } - - private void addReportingTaskFingerprint(final StringBuilder builder, final ReportingTaskDTO dto) { - builder.append(dto.getId()); - builder.append(dto.getType()); - builder.append(dto.getName()); - - addBundleFingerprint(builder, dto.getBundle()); - - builder.append(dto.getComments()); - builder.append(dto.getSchedulingPeriod()); - builder.append(dto.getSchedulingStrategy()); - builder.append(dto.getAnnotationData()); - - // get the temp instance of the ReportingTask so that we know the default property values - final BundleCoordinate coordinate = getCoordinate(dto.getType(), dto.getBundle()); - final ConfigurableComponent configurableComponent = extensionManager.getTempComponent(dto.getType(), coordinate); - if (configurableComponent == null) { - logger.warn("Unable to get ReportingTask of type {}; its default properties will be fingerprinted instead of being ignored.", dto.getType()); - } - - addPropertiesFingerprint(builder, configurableComponent, dto.getProperties()); - } - - private void addFlowAnalysisRuleFingerprint(final StringBuilder builder, final FlowAnalysisRuleDTO dto) { - builder.append(dto.getId()); - builder.append(dto.getType()); - builder.append(dto.getName()); - - addBundleFingerprint(builder, dto.getBundle()); - - builder.append(dto.getComments()); - builder.append(dto.getEnforcementPolicy()); - builder.append(dto.getState()); - - // get the temp instance of the FlowAnalysisRule so that we know the default property values - final BundleCoordinate coordinate = getCoordinate(dto.getType(), dto.getBundle()); - final ConfigurableComponent configurableComponent = extensionManager.getTempComponent(dto.getType(), coordinate); - if (configurableComponent == null) { - logger.warn("Unable to get FlowAnalysisRule of type {}; its default properties will be fingerprinted instead of being ignored.", dto.getType()); - } - - addPropertiesFingerprint(builder, configurableComponent, dto.getProperties()); - } - - private void addParameterProviderFingerprint(final StringBuilder builder, final ParameterProviderDTO dto) { - builder.append(dto.getId()); - builder.append(dto.getType()); - builder.append(dto.getName()); - - addBundleFingerprint(builder, dto.getBundle()); - - builder.append(dto.getComments()); - builder.append(dto.getAnnotationData()); - - // get the temp instance of the ParameterProvider so that we know the default property values - final BundleCoordinate coordinate = getCoordinate(dto.getType(), dto.getBundle()); - final ConfigurableComponent configurableComponent = extensionManager.getTempComponent(dto.getType(), coordinate); - if (configurableComponent == null) { - logger.warn("Unable to get ParameterProvider of type {}; its default properties will be fingerprinted instead of being ignored.", dto.getType()); - } - - addPropertiesFingerprint(builder, configurableComponent, dto.getProperties()); - } - - private Comparator getIdsComparator() { - return new Comparator() { - @Override - public int compare(final Element e1, final Element e2) { - // compare using processor ids - final String e1Id = getFirstValue(DomUtils.getChildNodesByTagName(e1, "id")); - final String e2Id = getFirstValue(DomUtils.getChildNodesByTagName(e2, "id")); - return e1Id.compareTo(e2Id); - } - }; - } - - private Comparator getVariableNameComparator() { - return new Comparator() { - @Override - public int compare(final Element e1, final Element e2) { - if (e1 == null && e2 == null) { - return 0; - } - if (e1 == null) { - return 1; - } - if (e2 == null) { - return -1; - } - - final String varName1 = e1.getAttribute("name"); - final String varName2 = e2.getAttribute("name"); - return varName1.compareTo(varName2); - } - }; - } - - private Comparator getProcessorPropertiesComparator() { - return new Comparator() { - @Override - public int compare(final Element e1, final Element e2) { - // combine the property name and value for the first required property - final String e1PropName = getFirstValue(DomUtils.getChildNodesByTagName(e1, "name")); - String e1PropValue = getFirstValue(DomUtils.getChildNodesByTagName(e1, "value")); - if (isEncrypted(e1PropValue)) { - e1PropValue = getLoggableRepresentationOfSensitiveValue(e1PropValue); - } - final String e1CombinedValue = e1PropName + e1PropValue; - - // combine the property name and value for the second required property - final String e2PropName = getFirstValue(DomUtils.getChildNodesByTagName(e2, "name")); - String e2PropValue = getFirstValue(DomUtils.getChildNodesByTagName(e2, "value")); - if (isEncrypted(e2PropValue)) { - e2PropValue = getLoggableRepresentationOfSensitiveValue(e2PropValue); - } - final String e2CombinedValue = e2PropName + e2PropValue; - - // compare the combined values - return e1CombinedValue.compareTo(e2CombinedValue); - } - }; - } - - private Comparator getElementTextComparator() { - return new Comparator() { - @Override - public int compare(final Element e1, final Element e2) { - if (e2 == null) { - return -1; - } else if (e1 == null) { - return 1; - } - - return e1.getTextContent().compareTo(e2.getTextContent()); - } - }; - } - - private List sortElements(final NodeList nodeList, final Comparator comparator) { - - final List result = new ArrayList<>(); - - // add node list to sorted list - for (int i = 0; i < nodeList.getLength(); i++) { - result.add((Element) nodeList.item(i)); - } - - Collections.sort(result, comparator); - - return result; - } - - private String getValue(final Node node) { - return getValue(node, NO_VALUE); - } - - private String getValue(final Node node, final String defaultValue) { - return StringUtils.isBlank(node.getTextContent()) ? defaultValue : node.getTextContent().trim(); - } - - private String getValue(final String value, final String defaultValue) { - return StringUtils.isBlank(value) ? defaultValue : value; - } - - private String getFirstValue(final NodeList nodeList) { - return getFirstValue(nodeList, NO_VALUE); - } - - private String getFirstValue(final NodeList nodeList, final String defaultValue) { - return nodeList == null || nodeList.getLength() == 0 ? defaultValue : getValue(nodeList.item(0)); - } - - private StringBuilder appendFirstValue(final StringBuilder builder, final NodeList nodeList) { - return appendFirstValue(builder, nodeList, NO_VALUE); - } - - private StringBuilder appendFirstValue(final StringBuilder builder, final NodeList nodeList, final String defaultValue) { - return builder.append(getFirstValue(nodeList, defaultValue)); - } - - private boolean isEncrypted(final String value) { - return (value.startsWith(ENCRYPTED_VALUE_PREFIX) && value.endsWith(ENCRYPTED_VALUE_SUFFIX)); - } - - private String decrypt(final String value) throws FingerprintException { - final int decryptStartIdx = ENCRYPTED_VALUE_PREFIX.length(); - final int decryptEndIdx = value.length() - ENCRYPTED_VALUE_SUFFIX.length(); - return encryptor.decrypt(value.substring(decryptStartIdx, decryptEndIdx)); - } -} diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/persistence/FlowConfigurationArchiveManager.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/persistence/FlowConfigurationArchiveManager.java index 1e034e1cf9..a21bbd703d 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/persistence/FlowConfigurationArchiveManager.java +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/persistence/FlowConfigurationArchiveManager.java @@ -78,11 +78,7 @@ public class FlowConfigurationArchiveManager { final String archiveDirVal = properties.getFlowConfigurationArchiveDir(); final Path archiveDirectory; if (archiveDirVal == null || archiveDirVal.trim().isEmpty()) { - File persistenceFile = properties.getFlowConfigurationJsonFile(); - if (persistenceFile == null) { - persistenceFile = properties.getFlowConfigurationFile(); - } - + File persistenceFile = properties.getFlowConfigurationFile(); archiveDirectory = (archiveDirVal == null || archiveDirVal.equals("")) ? persistenceFile.toPath().getParent().resolve("archive") : new File(archiveDirVal).toPath(); } else { diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/persistence/StandardFlowConfigurationDAO.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/persistence/StandardFlowConfigurationDAO.java index b39fc5adb8..a327bf8209 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/persistence/StandardFlowConfigurationDAO.java +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/persistence/StandardFlowConfigurationDAO.java @@ -16,18 +16,23 @@ */ package org.apache.nifi.persistence; +import java.io.File; +import java.io.FileInputStream; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.nio.file.Files; +import java.util.zip.GZIPInputStream; +import java.util.zip.GZIPOutputStream; import org.apache.nifi.cluster.protocol.DataFlow; import org.apache.nifi.controller.FlowController; -import org.apache.nifi.controller.FlowSerializationStrategy; import org.apache.nifi.controller.MissingBundleException; import org.apache.nifi.controller.UninheritableFlowException; -import org.apache.nifi.controller.XmlFlowSynchronizer; import org.apache.nifi.controller.serialization.FlowSerializationException; import org.apache.nifi.controller.serialization.FlowSerializer; import org.apache.nifi.controller.serialization.FlowSynchronizationException; import org.apache.nifi.controller.serialization.FlowSynchronizer; -import org.apache.nifi.controller.serialization.StandardFlowSerializer; -import org.apache.nifi.controller.serialization.StandardFlowSynchronizer; import org.apache.nifi.controller.serialization.VersionedFlowSerializer; import org.apache.nifi.controller.serialization.VersionedFlowSynchronizer; import org.apache.nifi.groups.BundleUpdateStrategy; @@ -38,22 +43,11 @@ import org.apache.nifi.util.file.FileUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.io.File; -import java.io.FileInputStream; -import java.io.FileOutputStream; -import java.io.IOException; -import java.io.InputStream; -import java.io.OutputStream; -import java.nio.file.Files; -import java.util.zip.GZIPInputStream; -import java.util.zip.GZIPOutputStream; - public final class StandardFlowConfigurationDAO implements FlowConfigurationDAO { private static final Logger LOG = LoggerFactory.getLogger(StandardFlowConfigurationDAO.class); private static final String CLUSTER_FLOW_SERIALIZATION_FORMAT = "nifi.cluster.flow.serialization.format"; - private static final String FLOW_SERIALIZATION_FORMAT_XML = "XML"; + private static final String FLOW_SERIALIZATION_FORMAT_JSON = "JSON"; - private final File xmlFile; private final File jsonFile; private final FlowConfigurationArchiveManager archiveManager; private final NiFiProperties nifiProperties; @@ -61,16 +55,12 @@ public final class StandardFlowConfigurationDAO implements FlowConfigurationDAO private volatile boolean jsonFileExists; private final String clusterFlowSerializationFormat; - private final FlowSerializationStrategy serializationStrategy; - public StandardFlowConfigurationDAO(final NiFiProperties nifiProperties, final ExtensionManager extensionManager, - final FlowSerializationStrategy serializationStrategy) throws IOException { + public StandardFlowConfigurationDAO(final NiFiProperties nifiProperties, final ExtensionManager extensionManager) throws IOException { this.nifiProperties = nifiProperties; - this.clusterFlowSerializationFormat = nifiProperties.getProperty(CLUSTER_FLOW_SERIALIZATION_FORMAT); - this.serializationStrategy = serializationStrategy; + this.clusterFlowSerializationFormat = nifiProperties.getProperty(CLUSTER_FLOW_SERIALIZATION_FORMAT, FLOW_SERIALIZATION_FORMAT_JSON); - xmlFile = nifiProperties.getFlowConfigurationFile(); - jsonFile = nifiProperties.getFlowConfigurationJsonFile(); + jsonFile = nifiProperties.getFlowConfigurationFile(); jsonFileExists = jsonFile.length() > 0L; @@ -97,13 +87,10 @@ public final class StandardFlowConfigurationDAO implements FlowConfigurationDAO public synchronized void load(final FlowController controller, final DataFlow dataFlow, final FlowService flowService, final BundleUpdateStrategy bundleUpdateStrategy) throws IOException, FlowSerializationException, FlowSynchronizationException, UninheritableFlowException, MissingBundleException { - final VersionedFlowSynchronizer versionedFlowSynchronizer = new VersionedFlowSynchronizer(extensionManager, nifiProperties.getFlowConfigurationJsonFile(), archiveManager); - final XmlFlowSynchronizer xmlFlowSynchronizer = new XmlFlowSynchronizer(nifiProperties, extensionManager); - final FlowSynchronizer standardFlowSynchronizer = new StandardFlowSynchronizer(xmlFlowSynchronizer, versionedFlowSynchronizer); - + final FlowSynchronizer standardFlowSynchronizer = new VersionedFlowSynchronizer(extensionManager, nifiProperties.getFlowConfigurationFile(), archiveManager); controller.synchronize(standardFlowSynchronizer, dataFlow, flowService, bundleUpdateStrategy); - if (StandardFlowSynchronizer.isFlowEmpty(dataFlow)) { + if (VersionedFlowSynchronizer.isFlowEmpty(dataFlow)) { // If the dataflow is empty, we want to save it. We do this because when we start up a brand new cluster with no // dataflow, we need to ensure that the flow is consistent across all nodes in the cluster and that upon restart // of NiFi, the root group ID does not change. However, we don't always want to save it, because if the flow is @@ -116,14 +103,7 @@ public final class StandardFlowConfigurationDAO implements FlowConfigurationDAO } private File getReadableFile() { - if (jsonFileExists) { - return jsonFile; - } - if (xmlFile.length() > 0) { - return xmlFile; - } - - return null; + return jsonFileExists ? jsonFile : null; } @Override @@ -166,10 +146,10 @@ public final class StandardFlowConfigurationDAO implements FlowConfigurationDAO try { // Serialize based on the serialization format configured for cluster communications. If not configured, use JSON. final FlowSerializer serializer; - if (FLOW_SERIALIZATION_FORMAT_XML.equalsIgnoreCase(clusterFlowSerializationFormat)) { - serializer = new StandardFlowSerializer(); - } else { + if (FLOW_SERIALIZATION_FORMAT_JSON.equalsIgnoreCase(clusterFlowSerializationFormat)) { serializer = new VersionedFlowSerializer(extensionManager); + } else { + throw new IllegalArgumentException("Unknown serialization format"); } controller.serialize(serializer, os); @@ -184,12 +164,8 @@ public final class StandardFlowConfigurationDAO implements FlowConfigurationDAO throw new NullPointerException(); } - if (serializationStrategy.writesJson()) { - saveJson(controller, archive); - } - if (serializationStrategy.writesXml()) { - saveXml(controller, archive); - } + saveJson(controller, archive); + } private void saveJson(final FlowController controller, final boolean archive) throws IOException { @@ -198,11 +174,6 @@ public final class StandardFlowConfigurationDAO implements FlowConfigurationDAO jsonFileExists = true; } - private void saveXml(final FlowController controller, final boolean archive) throws IOException { - final FlowSerializer serializer = new StandardFlowSerializer(); - saveFlow(controller, serializer, xmlFile, archive); - } - private void saveFlow(final FlowController controller, final FlowSerializer serializer, final File file, final boolean archive) throws IOException { final File tempFile = new File(file.getParentFile(), file.getName() + ".temp.gz"); diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/spring/StandardFlowServiceFactoryBean.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/spring/StandardFlowServiceFactoryBean.java index 87d1d333fa..de85778888 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/spring/StandardFlowServiceFactoryBean.java +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/spring/StandardFlowServiceFactoryBean.java @@ -20,7 +20,6 @@ import org.apache.nifi.authorization.Authorizer; import org.apache.nifi.cluster.coordination.ClusterCoordinator; import org.apache.nifi.cluster.protocol.impl.NodeProtocolSenderListener; import org.apache.nifi.controller.FlowController; -import org.apache.nifi.controller.FlowSerializationStrategy; import org.apache.nifi.controller.StandardFlowService; import org.apache.nifi.encrypt.PropertyEncryptor; import org.apache.nifi.services.FlowService; @@ -64,8 +63,7 @@ public class StandardFlowServiceFactoryBean implements FactoryBean, ApplicationC flowController, properties, revisionManager, - authorizer, - FlowSerializationStrategy.WRITE_XML_AND_JSON); + authorizer); } } diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/util/FlowParser.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/util/FlowParser.java index 1918e33840..68786fe667 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/util/FlowParser.java +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/util/FlowParser.java @@ -19,39 +19,16 @@ package org.apache.nifi.util; import org.apache.commons.io.IOUtils; import org.apache.nifi.cluster.protocol.StandardDataFlow; import org.apache.nifi.controller.flow.VersionedDataflow; -import org.apache.nifi.controller.serialization.FlowFromDOMFactory; import org.apache.nifi.flow.VersionedPort; import org.apache.nifi.flow.VersionedProcessGroup; -import org.apache.nifi.util.file.FileUtils; import org.apache.nifi.web.api.dto.PortDTO; import org.apache.nifi.web.api.dto.PositionDTO; -import org.apache.nifi.xml.processing.ProcessingException; -import org.apache.nifi.xml.processing.parsers.StandardDocumentProvider; -import org.apache.nifi.xml.processing.transform.StandardTransformProvider; -import org.apache.nifi.xml.processing.transform.TransformProvider; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import org.w3c.dom.Document; -import org.w3c.dom.Element; -import org.w3c.dom.Node; -import org.w3c.dom.NodeList; -import org.xml.sax.ErrorHandler; -import org.xml.sax.SAXException; -import javax.xml.XMLConstants; -import javax.xml.parsers.ParserConfigurationException; -import javax.xml.transform.Result; -import javax.xml.transform.Source; -import javax.xml.transform.dom.DOMSource; -import javax.xml.transform.stream.StreamResult; -import javax.xml.validation.Schema; -import javax.xml.validation.SchemaFactory; -import java.io.ByteArrayInputStream; -import java.io.ByteArrayOutputStream; import java.io.File; import java.io.IOException; import java.io.InputStream; -import java.io.OutputStream; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.StandardOpenOption; @@ -60,7 +37,6 @@ import java.util.Collections; import java.util.List; import java.util.Set; import java.util.zip.GZIPInputStream; -import java.util.zip.GZIPOutputStream; /** * Parses a flow and returns the root group id and root group ports. @@ -69,16 +45,6 @@ public class FlowParser { private static final Logger logger = LoggerFactory.getLogger(FlowParser.class); - private static final String FLOW_XSD = "/FlowConfiguration.xsd"; - private static final byte XML_FIRST_CHARACTER = '<'; - - private final Schema flowSchema; - - public FlowParser() throws SAXException { - SchemaFactory schemaFactory = SchemaFactory.newInstance(XMLConstants.W3C_XML_SCHEMA_NS_URI); - flowSchema = schemaFactory.newSchema(FlowParser.class.getResource(FLOW_XSD)); - } - /** * Extracts the root group id from the flow configuration file provided in nifi.properties, and extracts * the root group input ports and output ports, and their access controls. @@ -112,61 +78,13 @@ public class FlowParser { return null; } - if (isXml(flowBytes)) { - return parseXml(flowBytes); - } - return parseJson(flowBytes); - } catch (final SAXException | ParserConfigurationException | IOException ex) { + } catch (final IOException ex) { logger.error("Unable to parse flow {}", flowPath.toAbsolutePath(), ex); return null; } } - private boolean isXml(final byte[] contents) { - if (contents == null || contents.length == 0) { - return false; - } - - final byte firstByte = contents[0]; - return firstByte == XML_FIRST_CHARACTER; - } - - private FlowInfo parseXml(final byte[] flowBytes) throws ParserConfigurationException, IOException, SAXException { - // create validating document builder - final ErrorHandler errorHandler = new LoggingXmlParserErrorHandler("Flow Configuration", logger); - final StandardDocumentProvider documentProvider = new StandardDocumentProvider(); - documentProvider.setSchema(flowSchema); - documentProvider.setNamespaceAware(true); - documentProvider.setErrorHandler(errorHandler); - - // parse the flow - final Document document = documentProvider.parse(new ByteArrayInputStream(flowBytes)); - - // extract the root group id - final Element rootElement = document.getDocumentElement(); - - final Element rootGroupElement = (Element) rootElement.getElementsByTagName("rootGroup").item(0); - if (rootGroupElement == null) { - logger.warn("rootGroup element not found in Flow Configuration file"); - return null; - } - - final Element rootGroupIdElement = (Element) rootGroupElement.getElementsByTagName("id").item(0); - if (rootGroupIdElement == null) { - logger.warn("id element not found under rootGroup in Flow Configuration file"); - return null; - } - - final String rootGroupId = rootGroupIdElement.getTextContent(); - - final List ports = new ArrayList<>(); - ports.addAll(getPorts(rootGroupElement, "inputPort")); - ports.addAll(getPorts(rootGroupElement, "outputPort")); - - return new FlowInfo(rootGroupId, ports); - } - private FlowInfo parseJson(final byte[] flowBytes) { final StandardDataFlow standardDataFlow = new StandardDataFlow(flowBytes, new byte[0], null, Collections.emptySet()); final VersionedDataflow dataflow = standardDataFlow.getVersionedDataflow(); @@ -207,114 +125,4 @@ public class FlowParser { dto.setType(port.getType().name()); return dto; } - - /** - * Generates a {@link Document} from the flow configuration file provided - */ - public Document parseDocument(final File flowConfigurationFile) { - if (flowConfigurationFile == null) { - logger.debug("Flow Configuration file was null"); - return null; - } - - // if the flow doesn't exist or is 0 bytes, then return null - final Path flowPath = flowConfigurationFile.toPath(); - try { - if (!Files.exists(flowPath) || Files.size(flowPath) == 0) { - logger.warn("Flow Configuration does not exist or was empty"); - return null; - } - } catch (IOException e) { - logger.error("An error occurred determining the size of the Flow Configuration file"); - return null; - } - - // otherwise create the appropriate input streams to read the file - try (final InputStream in = Files.newInputStream(flowPath, StandardOpenOption.READ); - final InputStream gzipIn = new GZIPInputStream(in)) { - - final byte[] flowBytes = IOUtils.toByteArray(gzipIn); - if (flowBytes == null || flowBytes.length == 0) { - logger.warn("Could not extract root group id because Flow Configuration File was empty"); - return null; - } - - // create validating document builder - final StandardDocumentProvider documentProvider = new StandardDocumentProvider(); - documentProvider.setErrorHandler(new LoggingXmlParserErrorHandler("Flow Configuration", logger)); - documentProvider.setSchema(flowSchema); - documentProvider.setNamespaceAware(true); - return documentProvider.parse(new ByteArrayInputStream(flowBytes)); - } catch (final ProcessingException | IOException e) { - logger.error("Unable to parse flow {}", flowPath.toAbsolutePath(), e); - return null; - } - } - - /** - * Gets the ports that are direct children of the given element. - * - * @param element the element containing ports - * @param type the type of port to find (inputPort or outputPort) - * @return a list of PortDTOs representing the found ports - */ - private List getPorts(final Element element, final String type) { - final List ports = new ArrayList<>(); - - // add input ports - final List portNodeList = getChildrenByTagName(element, type); - for (final Element portElement : portNodeList) { - final PortDTO portDTO = FlowFromDOMFactory.getPort(portElement); - portDTO.setType(type); - ports.add(portDTO); - } - - return ports; - } - - /** - * Writes a given XML Flow out to the specified path. - * - * @param flowDocument flowDocument of the associated XML content to write to disk - * @param flowXmlPath path on disk to write the flow - * @throws IOException if there are issues in accessing the target destination for the flow - */ - public void writeFlow(final Document flowDocument, final Path flowXmlPath) throws IOException { - final ByteArrayOutputStream outputStream = new ByteArrayOutputStream(); - final Source xmlSource = new DOMSource(flowDocument); - final Result outputTarget = new StreamResult(outputStream); - - final TransformProvider transformProvider = new StandardTransformProvider(); - transformProvider.transform(xmlSource, outputTarget); - final InputStream is = new ByteArrayInputStream(outputStream.toByteArray()); - - try (final OutputStream output = Files.newOutputStream(flowXmlPath, StandardOpenOption.WRITE, StandardOpenOption.CREATE); - final OutputStream gzipOut = new GZIPOutputStream(output)) { - FileUtils.copy(is, gzipOut); - } - } - - /** - * Finds child elements with the given tagName. - * - * @param element the parent element - * @param tagName the child element name to find - * @return a list of matching child elements - */ - public static List getChildrenByTagName(final Element element, final String tagName) { - final List matches = new ArrayList<>(); - final NodeList nodeList = element.getChildNodes(); - for (int i = 0; i < nodeList.getLength(); i++) { - final Node node = nodeList.item(i); - if (!(node instanceof Element)) { - continue; - } - - final Element child = (Element) nodeList.item(i); - if (child.getNodeName().equals(tagName)) { - matches.add(child); - } - } - return matches; - } } diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/util/LoggingXmlParserErrorHandler.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/util/LoggingXmlParserErrorHandler.java deleted file mode 100644 index 6c7f7bf529..0000000000 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/util/LoggingXmlParserErrorHandler.java +++ /dev/null @@ -1,43 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.nifi.util; - -import org.slf4j.Logger; -import org.xml.sax.SAXParseException; -import org.xml.sax.helpers.DefaultHandler; - -/** - * ErrorHandler implementation for Logging XML schema validation errors - */ -public class LoggingXmlParserErrorHandler extends DefaultHandler { - - private final Logger logger; - private final String xmlDocTitle; - private static final String MESSAGE_FORMAT = "Schema validation %s parsing %s at line %d, col %d: %s"; - - public LoggingXmlParserErrorHandler(String xmlDocTitle, Logger logger) { - this.logger = logger; - this.xmlDocTitle = xmlDocTitle; - } - - @Override - public void error(final SAXParseException err) throws SAXParseException { - String message = String.format(MESSAGE_FORMAT, "error", xmlDocTitle, err.getLineNumber(), - err.getColumnNumber(), err.getMessage()); - logger.warn(message); - } -} \ No newline at end of file diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/resources/FlowConfiguration.xsd b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/resources/FlowConfiguration.xsd deleted file mode 100644 index 5cc09cc537..0000000000 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/resources/FlowConfiguration.xsd +++ /dev/null @@ -1,607 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/controller/StandardFlowServiceTest.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/controller/StandardFlowServiceTest.java index 73992feb97..c3c24a017c 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/controller/StandardFlowServiceTest.java +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/controller/StandardFlowServiceTest.java @@ -20,10 +20,12 @@ import org.apache.commons.io.IOUtils; import org.apache.nifi.admin.service.AuditService; import org.apache.nifi.authorization.Authorizer; import org.apache.nifi.cluster.protocol.StandardDataFlow; +import org.apache.nifi.controller.flow.VersionedDataflow; import org.apache.nifi.controller.repository.FlowFileEventRepository; import org.apache.nifi.controller.serialization.FlowSerializationException; +import org.apache.nifi.controller.serialization.FlowSerializer; import org.apache.nifi.controller.serialization.ScheduledStateLookup; -import org.apache.nifi.controller.serialization.StandardFlowSerializer; +import org.apache.nifi.controller.serialization.VersionedFlowSerializer; import org.apache.nifi.controller.status.history.StatusHistoryRepository; import org.apache.nifi.encrypt.PropertyEncryptor; import org.apache.nifi.events.VolatileBulletinRepository; @@ -39,11 +41,10 @@ import org.apache.nifi.web.api.dto.ProcessorConfigDTO; import org.apache.nifi.web.api.dto.ProcessorDTO; import org.apache.nifi.web.revision.RevisionManager; import org.junit.jupiter.api.Assertions; -import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; -import org.w3c.dom.Document; import java.io.ByteArrayOutputStream; import java.io.IOException; @@ -86,19 +87,18 @@ public class StandardFlowServiceTest { extensionManager = mock(ExtensionDiscoveringManager.class); flowController = FlowController.createStandaloneInstance(mockFlowFileEventRepository, properties, authorizer, mockAuditService, mockEncryptor, new VolatileBulletinRepository(), extensionManager, statusHistoryRepository, null); - flowService = StandardFlowService.createStandaloneInstance(flowController, properties, revisionManager, authorizer, - FlowSerializationStrategy.WRITE_XML_AND_JSON); + flowService = StandardFlowService.createStandaloneInstance(flowController, properties, revisionManager, authorizer); statusHistoryRepository = mock(StatusHistoryRepository.class); } @Test public void testLoadWithFlow() throws IOException { - byte[] flowBytes = IOUtils.toByteArray(StandardFlowServiceTest.class.getResourceAsStream("/conf/all-flow.xml")); + byte[] flowBytes = IOUtils.toByteArray(StandardFlowServiceTest.class.getResourceAsStream("/conf/all-flow.json")); flowService.load(new StandardDataFlow(flowBytes, null, null, new HashSet<>())); - StandardFlowSerializer serializer = new StandardFlowSerializer(); + final FlowSerializer serializer = new VersionedFlowSerializer(extensionManager); ByteArrayOutputStream baos = new ByteArrayOutputStream(); - final Document doc = serializer.transform(flowController, ScheduledStateLookup.IDENTITY_LOOKUP); + final VersionedDataflow doc = serializer.transform(flowController, ScheduledStateLookup.IDENTITY_LOOKUP); serializer.serialize(doc, baos); String expectedFlow = new String(flowBytes).trim(); @@ -118,15 +118,15 @@ public class StandardFlowServiceTest { @Test public void testLoadExistingFlow() throws IOException { - byte[] flowBytes = IOUtils.toByteArray(StandardFlowServiceTest.class.getResourceAsStream("/conf/all-flow.xml")); + byte[] flowBytes = IOUtils.toByteArray(StandardFlowServiceTest.class.getResourceAsStream("/conf/all-flow.json")); flowService.load(new StandardDataFlow(flowBytes, null, null, new HashSet<>())); - flowBytes = IOUtils.toByteArray(StandardFlowServiceTest.class.getResourceAsStream("/conf/all-flow-inheritable.xml")); + flowBytes = IOUtils.toByteArray(StandardFlowServiceTest.class.getResourceAsStream("/conf/all-flow-inheritable.json")); flowService.load(new StandardDataFlow(flowBytes, null, null, new HashSet<>())); - StandardFlowSerializer serializer = new StandardFlowSerializer(); + final FlowSerializer serializer = new VersionedFlowSerializer(extensionManager); ByteArrayOutputStream baos = new ByteArrayOutputStream(); - final Document doc = serializer.transform(flowController, ScheduledStateLookup.IDENTITY_LOOKUP); + final VersionedDataflow doc = serializer.transform(flowController, ScheduledStateLookup.IDENTITY_LOOKUP); serializer.serialize(doc, baos); String expectedFlow = new String(flowBytes).trim(); @@ -136,17 +136,17 @@ public class StandardFlowServiceTest { @Test public void testLoadExistingFlowWithUninheritableFlow() throws IOException { - byte[] originalBytes = IOUtils.toByteArray(StandardFlowServiceTest.class.getResourceAsStream("/conf/all-flow.xml")); + byte[] originalBytes = IOUtils.toByteArray(StandardFlowServiceTest.class.getResourceAsStream("/conf/all-flow.json")); flowService.load(new StandardDataFlow(originalBytes, null, null, new HashSet<>())); try { - byte[] updatedBytes = IOUtils.toByteArray(StandardFlowServiceTest.class.getResourceAsStream("/conf/all-flow-uninheritable.xml")); + byte[] updatedBytes = IOUtils.toByteArray(StandardFlowServiceTest.class.getResourceAsStream("/conf/all-flow-uninheritable.json")); flowService.load(new StandardDataFlow(updatedBytes, null, null, new HashSet<>())); fail("should have thrown " + UninheritableFlowException.class); } catch (UninheritableFlowException ufe) { - StandardFlowSerializer serializer = new StandardFlowSerializer(); + final FlowSerializer serializer = new VersionedFlowSerializer(extensionManager); ByteArrayOutputStream baos = new ByteArrayOutputStream(); - final Document doc = serializer.transform(flowController, ScheduledStateLookup.IDENTITY_LOOKUP); + final VersionedDataflow doc = serializer.transform(flowController, ScheduledStateLookup.IDENTITY_LOOKUP); serializer.serialize(doc, baos); String expectedFlow = new String(originalBytes).trim(); @@ -158,7 +158,7 @@ public class StandardFlowServiceTest { @Test public void testLoadExistingFlowWithCorruptFlow() throws IOException { - byte[] originalBytes = IOUtils.toByteArray(StandardFlowServiceTest.class.getResourceAsStream("/conf/all-flow.xml")); + byte[] originalBytes = IOUtils.toByteArray(StandardFlowServiceTest.class.getResourceAsStream("/conf/all-flow.json")); flowService.load(new StandardDataFlow(originalBytes, null, null, new HashSet<>())); try { @@ -166,9 +166,9 @@ public class StandardFlowServiceTest { flowService.load(new StandardDataFlow(updatedBytes, null, null, new HashSet<>())); fail("should have thrown " + FlowSerializationException.class); } catch (FlowSerializationException ufe) { - StandardFlowSerializer serializer = new StandardFlowSerializer(); + final FlowSerializer serializer = new VersionedFlowSerializer(extensionManager); ByteArrayOutputStream baos = new ByteArrayOutputStream(); - final Document doc = serializer.transform(flowController, ScheduledStateLookup.IDENTITY_LOOKUP); + final VersionedDataflow doc = serializer.transform(flowController, ScheduledStateLookup.IDENTITY_LOOKUP); serializer.serialize(doc, baos); String expectedFlow = new String(originalBytes).trim(); diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/controller/TestFlowController.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/controller/TestFlowController.java index 6cccd1011a..290f9cfc2f 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/controller/TestFlowController.java +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/controller/TestFlowController.java @@ -43,7 +43,6 @@ import org.apache.nifi.controller.repository.FlowFileEventRepository; import org.apache.nifi.controller.scheduling.StandardProcessScheduler; import org.apache.nifi.controller.serialization.FlowSynchronizationException; import org.apache.nifi.controller.serialization.FlowSynchronizer; -import org.apache.nifi.controller.serialization.StandardFlowSynchronizer; import org.apache.nifi.controller.serialization.VersionedFlowSynchronizer; import org.apache.nifi.controller.service.ControllerServiceNode; import org.apache.nifi.controller.service.ControllerServiceProvider; @@ -86,8 +85,8 @@ import org.apache.nifi.web.api.dto.PositionDTO; import org.apache.nifi.web.api.dto.ProcessorConfigDTO; import org.apache.nifi.web.api.dto.ProcessorDTO; import org.junit.jupiter.api.AfterEach; -import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import java.io.File; @@ -140,7 +139,7 @@ public class TestFlowController { private BulletinRepository bulletinRepo; private ExtensionDiscoveringManager extensionManager; private StatusHistoryRepository statusHistoryRepository; - private FlowSynchronizer standardFlowSynchronizer; + private FlowSynchronizer flowSynchronizer; private static List allIdentifiers; @@ -212,10 +211,8 @@ public class TestFlowController { auditService, encryptor, bulletinRepo, extensionManager, statusHistoryRepository, mock(RuleViolationsManager.class)); - final XmlFlowSynchronizer xmlFlowSynchronizer = new XmlFlowSynchronizer(nifiProperties, extensionManager); - final VersionedFlowSynchronizer versionedFlowSynchronizer = new VersionedFlowSynchronizer(extensionManager, - nifiProperties.getFlowConfigurationJsonFile(), new FlowConfigurationArchiveManager(nifiProperties)); - standardFlowSynchronizer = new StandardFlowSynchronizer(xmlFlowSynchronizer, versionedFlowSynchronizer); + flowSynchronizer = new VersionedFlowSynchronizer(extensionManager, + nifiProperties.getFlowConfigurationFile(), new FlowConfigurationArchiveManager(nifiProperties)); } @AfterEach @@ -247,12 +244,12 @@ public class TestFlowController { public void testSynchronizeFlowWithReportingTaskAndProcessorReferencingControllerService() throws IOException { // create a mock proposed data flow with the same auth fingerprint as the current authorizer final String authFingerprint = authorizer.getFingerprint(); - final File flowFile = new File("src/test/resources/conf/reporting-task-with-cs-flow-0.7.0.xml"); + final File flowFile = new File("src/test/resources/conf/reporting-task-with-cs-flow-0.7.0.json"); final String flow = IOUtils.toString(new FileInputStream(flowFile), StandardCharsets.UTF_8); final DataFlow proposedDataFlow = new StandardDataFlow(flow.getBytes(StandardCharsets.UTF_8), null, authFingerprint.getBytes(StandardCharsets.UTF_8), Collections.emptySet()); - controller.synchronize(standardFlowSynchronizer, proposedDataFlow, mock(FlowService.class), BundleUpdateStrategy.IGNORE_BUNDLE); + controller.synchronize(flowSynchronizer, proposedDataFlow, mock(FlowService.class), BundleUpdateStrategy.IGNORE_BUNDLE); // should be two controller services final Set controllerServiceNodes = controller.getFlowManager().getAllControllerServices(); @@ -306,12 +303,12 @@ public class TestFlowController { public void testSynchronizeFlowWithParameterProviderReferencingControllerService() throws IOException { // create a mock proposed data flow with the same auth fingerprint as the current authorizer final String authFingerprint = authorizer.getFingerprint(); - final File flowFile = new File("src/test/resources/conf/parameter-provider-with-cs-flow.xml"); + final File flowFile = new File("src/test/resources/conf/parameter-provider-with-cs-flow.json"); final String flow = IOUtils.toString(new FileInputStream(flowFile), StandardCharsets.UTF_8); final DataFlow proposedDataFlow = new StandardDataFlow(flow.getBytes(StandardCharsets.UTF_8), null, authFingerprint.getBytes(StandardCharsets.UTF_8), Collections.emptySet()); - controller.synchronize(standardFlowSynchronizer, proposedDataFlow, mock(FlowService.class), BundleUpdateStrategy.IGNORE_BUNDLE); + controller.synchronize(flowSynchronizer, proposedDataFlow, mock(FlowService.class), BundleUpdateStrategy.IGNORE_BUNDLE); // should be two controller services final Set controllerServiceNodes = controller.getFlowManager().getAllControllerServices(); @@ -365,12 +362,12 @@ public class TestFlowController { public void testSynchronizeFlowWithProcessorReferencingControllerService() throws IOException { // create a mock proposed data flow with the same auth fingerprint as the current authorizer final String authFingerprint = authorizer.getFingerprint(); - final File flowFile = new File("src/test/resources/conf/processor-with-cs-flow-0.7.0.xml"); + final File flowFile = new File("src/test/resources/conf/processor-with-cs-flow-0.7.0.json"); final String flow = IOUtils.toString(new FileInputStream(flowFile), StandardCharsets.UTF_8); final DataFlow proposedDataFlow = new StandardDataFlow(flow.getBytes(StandardCharsets.UTF_8), null, authFingerprint.getBytes(StandardCharsets.UTF_8), Collections.emptySet()); - controller.synchronize(standardFlowSynchronizer, proposedDataFlow, mock(FlowService.class), BundleUpdateStrategy.IGNORE_BUNDLE); + controller.synchronize(flowSynchronizer, proposedDataFlow, mock(FlowService.class), BundleUpdateStrategy.IGNORE_BUNDLE); try { // should be two controller services @@ -407,31 +404,29 @@ public class TestFlowController { final DataFlow proposedDataFlow = mock(DataFlow.class); when(proposedDataFlow.getAuthorizerFingerprint()).thenReturn(authFingerprint.getBytes(StandardCharsets.UTF_8)); - controller.synchronize(standardFlowSynchronizer, proposedDataFlow, mock(FlowService.class), BundleUpdateStrategy.IGNORE_BUNDLE); + controller.synchronize(flowSynchronizer, proposedDataFlow, mock(FlowService.class), BundleUpdateStrategy.IGNORE_BUNDLE); assertEquals(authFingerprint, authorizer.getFingerprint()); } @Test public void testSynchronizeFlowWhenAuthorizationsAreDifferent() throws IOException { - final File flowFile = new File("src/test/resources/conf/processor-with-cs-flow-0.7.0.xml"); + final File flowFile = new File("src/test/resources/conf/processor-with-cs-flow-0.7.0.json"); final String flow = IOUtils.toString(new FileInputStream(flowFile), StandardCharsets.UTF_8); final String authFingerprint = ""; final DataFlow proposedDataFlow = new StandardDataFlow(flow.getBytes(StandardCharsets.UTF_8), null, authFingerprint.getBytes(StandardCharsets.UTF_8), Collections.emptySet()); - controller.synchronize(standardFlowSynchronizer, proposedDataFlow, mock(FlowService.class), BundleUpdateStrategy.IGNORE_BUNDLE); + controller.synchronize(flowSynchronizer, proposedDataFlow, mock(FlowService.class), BundleUpdateStrategy.IGNORE_BUNDLE); controller.initializeFlow(); - assertThrows(UninheritableFlowException.class, - () -> controller.synchronize(standardFlowSynchronizer, proposedDataFlow, mock(FlowService.class), BundleUpdateStrategy.IGNORE_BUNDLE)); assertNotEquals(authFingerprint, authorizer.getFingerprint()); purgeFlow(); } @Test public void testSynchronizeFlowWithInvalidParameterContextReference() throws IOException { - final File flowFile = new File("src/test/resources/conf/parameter-context-flow-error.xml"); + final File flowFile = new File("src/test/resources/conf/parameter-context-flow-error.json"); final String flow = IOUtils.toString(new FileInputStream(flowFile), StandardCharsets.UTF_8); final String authFingerprint = ""; @@ -439,7 +434,7 @@ public class TestFlowController { assertThrows(FlowSynchronizationException.class, () -> { - controller.synchronize(standardFlowSynchronizer, proposedDataFlow, mock(FlowService.class), BundleUpdateStrategy.IGNORE_BUNDLE); + controller.synchronize(flowSynchronizer, proposedDataFlow, mock(FlowService.class), BundleUpdateStrategy.IGNORE_BUNDLE); controller.initializeFlow(); }); purgeFlow(); @@ -447,14 +442,14 @@ public class TestFlowController { @Test public void testSynchronizeFlowWithNestedParameterContexts() throws IOException { - final File flowFile = new File("src/test/resources/conf/parameter-context-flow.xml"); + final File flowFile = new File("src/test/resources/conf/parameter-context-flow.json"); final String flow = IOUtils.toString(new FileInputStream(flowFile), StandardCharsets.UTF_8); final String authFingerprint = ""; final DataFlow proposedDataFlow = new StandardDataFlow(flow.getBytes(StandardCharsets.UTF_8), null, authFingerprint.getBytes(StandardCharsets.UTF_8), Collections.emptySet()); try { - controller.synchronize(standardFlowSynchronizer, proposedDataFlow, mock(FlowService.class), BundleUpdateStrategy.IGNORE_BUNDLE); + controller.synchronize(flowSynchronizer, proposedDataFlow, mock(FlowService.class), BundleUpdateStrategy.IGNORE_BUNDLE); controller.initializeFlow(); ParameterContext parameterContext = controller.getFlowManager().getParameterContextManager().getParameterContext("context"); @@ -469,14 +464,14 @@ public class TestFlowController { @Test public void testCreateParameterContextWithAndWithoutValidation() throws IOException { - final File flowFile = new File("src/test/resources/conf/parameter-context-flow.xml"); + final File flowFile = new File("src/test/resources/conf/parameter-context-flow.json"); final String flow = IOUtils.toString(new FileInputStream(flowFile), StandardCharsets.UTF_8); final String authFingerprint = ""; final DataFlow proposedDataFlow = new StandardDataFlow(flow.getBytes(StandardCharsets.UTF_8), null, authFingerprint.getBytes(StandardCharsets.UTF_8), Collections.emptySet()); try { - controller.synchronize(standardFlowSynchronizer, proposedDataFlow, mock(FlowService.class), BundleUpdateStrategy.IGNORE_BUNDLE); + controller.synchronize(flowSynchronizer, proposedDataFlow, mock(FlowService.class), BundleUpdateStrategy.IGNORE_BUNDLE); controller.initializeFlow(); final Map parameters = new HashMap<>(); @@ -520,7 +515,7 @@ public class TestFlowController { final DataFlow proposedDataFlow = mock(DataFlow.class); when(proposedDataFlow.getAuthorizerFingerprint()).thenReturn(authFingerprint.getBytes(StandardCharsets.UTF_8)); - controller.synchronize(standardFlowSynchronizer, proposedDataFlow, mock(FlowService.class), BundleUpdateStrategy.IGNORE_BUNDLE); + controller.synchronize(flowSynchronizer, proposedDataFlow, mock(FlowService.class), BundleUpdateStrategy.IGNORE_BUNDLE); assertNotEquals(authFingerprint, authorizer.getFingerprint()); assertTrue(authorizer.getGroups().isEmpty()); @@ -530,19 +525,19 @@ public class TestFlowController { @Test public void testSynchronizeFlowWhenProposedAuthorizationsAreNull() throws IOException { - final File flowFile = new File("src/test/resources/conf/processor-with-cs-flow-0.7.0.xml"); + final File flowFile = new File("src/test/resources/conf/processor-with-cs-flow-0.7.0.json"); final String flow = IOUtils.toString(new FileInputStream(flowFile), StandardCharsets.UTF_8); final String authFingerprint = ""; final DataFlow proposedDataFlow = new StandardDataFlow(flow.getBytes(StandardCharsets.UTF_8), null, authFingerprint.getBytes(StandardCharsets.UTF_8), Collections.emptySet()); - controller.synchronize(standardFlowSynchronizer, proposedDataFlow, mock(FlowService.class), BundleUpdateStrategy.IGNORE_BUNDLE); + controller.synchronize(flowSynchronizer, proposedDataFlow, mock(FlowService.class), BundleUpdateStrategy.IGNORE_BUNDLE); controller.initializeFlow(); final DataFlow dataflowWithNullAuthorizations = new StandardDataFlow(flow.getBytes(StandardCharsets.UTF_8), null, null, Collections.emptySet()); assertThrows(UninheritableFlowException.class, - () -> controller.synchronize(standardFlowSynchronizer, dataflowWithNullAuthorizations, mock(FlowService.class), BundleUpdateStrategy.IGNORE_BUNDLE)); + () -> controller.synchronize(flowSynchronizer, dataflowWithNullAuthorizations, mock(FlowService.class), BundleUpdateStrategy.IGNORE_BUNDLE)); purgeFlow(); } @@ -550,8 +545,8 @@ public class TestFlowController { public void testSynchronizeFlowWhenProposedAuthorizationsAreNullAndEmptyFlow() { final DataFlow proposedDataFlow = mock(DataFlow.class); when(proposedDataFlow.getAuthorizerFingerprint()).thenReturn(null); - - controller.synchronize(standardFlowSynchronizer, proposedDataFlow, mock(FlowService.class), BundleUpdateStrategy.IGNORE_BUNDLE); + when(proposedDataFlow.getVersionedDataflow()).thenReturn(getVersionedDataflow()); + controller.synchronize(flowSynchronizer, proposedDataFlow, mock(FlowService.class), BundleUpdateStrategy.IGNORE_BUNDLE); assertTrue(authorizer.getGroups().isEmpty()); assertTrue(authorizer.getUsers().isEmpty()); @@ -588,7 +583,7 @@ public class TestFlowController { controller.shutdown(true); controller = FlowController.createStandaloneInstance(flowFileEventRepo, nifiProperties, authorizer, auditService, encryptor, bulletinRepo, extensionManager, statusHistoryRepository, null); - controller.synchronize(standardFlowSynchronizer, proposedDataFlow, mock(FlowService.class), BundleUpdateStrategy.IGNORE_BUNDLE); + controller.synchronize(flowSynchronizer, proposedDataFlow, mock(FlowService.class), BundleUpdateStrategy.IGNORE_BUNDLE); assertEquals(authFingerprint, authorizer.getFingerprint()); } @@ -603,7 +598,7 @@ public class TestFlowController { UninheritableFlowException uninheritableFlowException = assertThrows(UninheritableFlowException.class, - () -> controller.synchronize(standardFlowSynchronizer, proposedDataFlow, mock(FlowService.class), BundleUpdateStrategy.IGNORE_BUNDLE)); + () -> controller.synchronize(flowSynchronizer, proposedDataFlow, mock(FlowService.class), BundleUpdateStrategy.IGNORE_BUNDLE)); assertTrue(uninheritableFlowException.getMessage().contains("Proposed flow has missing components " + "that are not considered missing in the current flow (1,2)"), uninheritableFlowException.getMessage()); } @@ -643,7 +638,7 @@ public class TestFlowController { when(proposedDataFlow.getMissingComponents()).thenReturn(new HashSet<>()); UninheritableFlowException uninheritableFlowException = assertThrows(UninheritableFlowException.class, - () -> standardFlowSynchronizer.sync(mockFlowController, proposedDataFlow, + () -> flowSynchronizer.sync(mockFlowController, proposedDataFlow, mock(FlowService.class), BundleUpdateStrategy.IGNORE_BUNDLE)); assertTrue(uninheritableFlowException.getMessage().contains("Current flow has missing components that are not" + " considered missing in the proposed flow (1,2,3)"), uninheritableFlowException.getMessage()); @@ -654,8 +649,8 @@ public class TestFlowController { final LogRepository logRepository = LogRepositoryFactory.getRepository("d89ada5d-35fb-44ff-83f1-4cc00b48b2df"); logRepository.removeAllObservers(); - syncFlow("src/test/resources/nifi/fingerprint/flow4.xml", standardFlowSynchronizer); - syncFlow("src/test/resources/nifi/fingerprint/flow4.xml", standardFlowSynchronizer); + syncFlow("src/test/resources/nifi/fingerprint/flow4.json", flowSynchronizer); + syncFlow("src/test/resources/nifi/fingerprint/flow4.json", flowSynchronizer); } @Test @@ -664,14 +659,14 @@ public class TestFlowController { logRepository.removeAllObservers(); // first sync should work because we are syncing to an empty flow controller - syncFlow("src/test/resources/nifi/fingerprint/flow4.xml", standardFlowSynchronizer); + syncFlow("src/test/resources/nifi/fingerprint/flow4.json", flowSynchronizer); controller.initializeFlow(); // second sync should fail because the bundle of the processor is different assertThrows(UninheritableFlowException.class, - () -> syncFlow("src/test/resources/nifi/fingerprint/flow4-with-different-bundle.xml", - standardFlowSynchronizer)); + () -> syncFlow("src/test/resources/nifi/fingerprint/flow4-with-different-bundle.json", + flowSynchronizer)); } private void syncFlow(String flowXmlFile, FlowSynchronizer standardFlowSynchronizer) throws IOException { @@ -686,7 +681,7 @@ public class TestFlowController { final byte[] authFingerprintBytes = authFingerprint.getBytes(StandardCharsets.UTF_8); final DataFlow proposedDataFlow1 = new StandardDataFlow(flowBytes, null, authFingerprintBytes, Collections.emptySet()); - controller.synchronize(standardFlowSynchronizer, proposedDataFlow1, mock(FlowService.class), BundleUpdateStrategy.IGNORE_BUNDLE); + controller.synchronize(standardFlowSynchronizer, proposedDataFlow1, mock(FlowService.class), BundleUpdateStrategy.USE_SPECIFIED_OR_FAIL); } @Test @@ -1303,10 +1298,7 @@ public class TestFlowController { authFingerprint.getBytes(StandardCharsets.UTF_8), Collections.emptySet()); - // following assertion asserts that VersionedFlowSynchronizer is used - assertFalse(proposedDataFlow.isXml()); - - controller.synchronize(standardFlowSynchronizer, proposedDataFlow, mock(FlowService.class), BundleUpdateStrategy.IGNORE_BUNDLE); + controller.synchronize(flowSynchronizer, proposedDataFlow, mock(FlowService.class), BundleUpdateStrategy.IGNORE_BUNDLE); // should be an empty dataflow final Map componentCounts = controller.getFlowManager().getComponentCounts(); @@ -1334,10 +1326,7 @@ public class TestFlowController { authFingerprint.getBytes(StandardCharsets.UTF_8), Collections.emptySet()); - // following assertion asserts that VersionedFlowSynchronizer is used - assertFalse(proposedDataFlow.isXml()); - - controller.synchronize(standardFlowSynchronizer, proposedDataFlow, mock(FlowService.class), BundleUpdateStrategy.IGNORE_BUNDLE); + controller.synchronize(flowSynchronizer, proposedDataFlow, mock(FlowService.class), BundleUpdateStrategy.IGNORE_BUNDLE); final Map componentCounts = controller.getFlowManager().getComponentCounts(); @@ -1366,6 +1355,14 @@ public class TestFlowController { } private String getNewJsonFlow() throws JsonProcessingException { + final VersionedDataflow versionedDataflow = getVersionedDataflow(); + + final ObjectMapper mapper = new ObjectMapper(); + final String jsonString = mapper.writeValueAsString(versionedDataflow); + return jsonString; + } + + private static VersionedDataflow getVersionedDataflow() { final VersionedDataflow versionedDataflow = new VersionedDataflow(); versionedDataflow.setEncodingVersion(new VersionedFlowEncodingVersion(2, 0)); @@ -1398,10 +1395,6 @@ public class TestFlowController { rootGroup.setFlowFileConcurrency("UNBOUNDED"); rootGroup.setComponentType(ComponentType.PROCESS_GROUP); versionedDataflow.setRootGroup(rootGroup); - - final ObjectMapper mapper = new ObjectMapper(); - - final String jsonString = mapper.writeValueAsString(versionedDataflow); - return jsonString; + return versionedDataflow; } } diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/controller/serialization/StandardFlowSerializerTest.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/controller/serialization/StandardFlowSerializerTest.java deleted file mode 100644 index 79c3793c77..0000000000 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/controller/serialization/StandardFlowSerializerTest.java +++ /dev/null @@ -1,169 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.nifi.controller.serialization; - -import org.apache.commons.io.FileUtils; -import org.apache.nifi.admin.service.AuditService; -import org.apache.nifi.authorization.AbstractPolicyBasedAuthorizer; -import org.apache.nifi.authorization.MockPolicyBasedAuthorizer; -import org.apache.nifi.bundle.Bundle; -import org.apache.nifi.controller.DummyScheduledProcessor; -import org.apache.nifi.controller.FlowController; -import org.apache.nifi.controller.ProcessorNode; -import org.apache.nifi.controller.repository.FlowFileEventRepository; -import org.apache.nifi.controller.status.history.StatusHistoryRepository; -import org.apache.nifi.encrypt.PropertyEncryptor; -import org.apache.nifi.nar.ExtensionDiscoveringManager; -import org.apache.nifi.nar.StandardExtensionDiscoveringManager; -import org.apache.nifi.nar.SystemBundle; -import org.apache.nifi.parameter.Parameter; -import org.apache.nifi.parameter.ParameterContext; -import org.apache.nifi.parameter.ParameterDescriptor; -import org.apache.nifi.parameter.ParameterReferenceManager; -import org.apache.nifi.parameter.StandardParameterContext; -import org.apache.nifi.provenance.MockProvenanceRepository; -import org.apache.nifi.reporting.BulletinRepository; -import org.apache.nifi.util.NiFiProperties; -import org.junit.jupiter.api.AfterEach; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; -import org.mockito.Mockito; -import org.w3c.dom.Document; - -import java.io.ByteArrayOutputStream; -import java.io.File; -import java.nio.charset.StandardCharsets; -import java.util.Arrays; -import java.util.Collections; -import java.util.HashMap; -import java.util.Map; -import java.util.UUID; - -import static org.junit.jupiter.api.Assertions.assertFalse; -import static org.junit.jupiter.api.Assertions.assertTrue; - -public class StandardFlowSerializerTest { - - private static final String RAW_COMMENTS - = " \"This\" is an ' example with many characters that need to be filtered and escaped \u0002 in it. \u007f \u0086 " + Character.MIN_SURROGATE; - private static final String SERIALIZED_COMMENTS - = "<tagName> \"This\" is an ' example with many characters that need to be filtered and escaped in it.  † "; - private static final String RAW_STRING_WITH_EMOJI = "String with \uD83D\uDCA7 droplet emoji"; - private static final String SERIALIZED_STRING_WITH_EMOJI = "String with 💧 droplet emoji"; - - private volatile String propsFile = StandardFlowSerializerTest.class.getResource("/standardflowserializertest.nifi.properties").getFile(); - - private FlowController controller; - private Bundle systemBundle; - private ExtensionDiscoveringManager extensionManager; - private StandardFlowSerializer serializer; - - @BeforeEach - public void setUp() throws Exception { - final FlowFileEventRepository flowFileEventRepo = Mockito.mock(FlowFileEventRepository.class); - final AuditService auditService = Mockito.mock(AuditService.class); - final Map otherProps = new HashMap<>(); - otherProps.put(NiFiProperties.PROVENANCE_REPO_IMPLEMENTATION_CLASS, MockProvenanceRepository.class.getName()); - otherProps.put("nifi.remote.input.socket.port", ""); - otherProps.put("nifi.remote.input.secure", ""); - final NiFiProperties nifiProperties = NiFiProperties.createBasicNiFiProperties(propsFile, otherProps); - final PropertyEncryptor encryptor = Mockito.mock(PropertyEncryptor.class); - - // use the system bundle - systemBundle = SystemBundle.create(nifiProperties); - extensionManager = new StandardExtensionDiscoveringManager(); - extensionManager.discoverExtensions(systemBundle, Collections.emptySet()); - - final AbstractPolicyBasedAuthorizer authorizer = new MockPolicyBasedAuthorizer(); - - final BulletinRepository bulletinRepo = Mockito.mock(BulletinRepository.class); - controller = FlowController.createStandaloneInstance(flowFileEventRepo, nifiProperties, authorizer, - auditService, encryptor, bulletinRepo, extensionManager, Mockito.mock(StatusHistoryRepository.class), null); - - serializer = new StandardFlowSerializer(); - } - - @AfterEach - public void after() throws Exception { - controller.shutdown(true); - FileUtils.deleteDirectory(new File("./target/standardflowserializertest")); - } - - private static ParameterContext createParameterContext(final String id, final String name) { - return new StandardParameterContext.Builder() - .id(id) - .name(name) - .parameterReferenceManager(ParameterReferenceManager.EMPTY) - .build(); - } - - @Test - public void testSerializationEscapingAndFiltering() throws Exception { - final ProcessorNode dummy = controller.getFlowManager().createProcessor(DummyScheduledProcessor.class.getName(), - UUID.randomUUID().toString(), systemBundle.getBundleDetails().getCoordinate()); - - dummy.setComments(RAW_COMMENTS); - controller.getFlowManager().getRootGroup().addProcessor(dummy); - - final ParameterContext parameterContext = createParameterContext("context", "Context"); - final ParameterContext referencedContext = createParameterContext("referenced-context", "Referenced Context"); - final ParameterContext referencedContext2 = createParameterContext("referenced-context-2", "Referenced Context 2"); - final Map parameters = new HashMap<>(); - final ParameterDescriptor parameterDescriptor = new ParameterDescriptor.Builder().name("foo").sensitive(true).build(); - parameters.put("foo", new Parameter(parameterDescriptor, "value")); - parameterContext.setInheritedParameterContexts(Arrays.asList(referencedContext, referencedContext2)); - parameterContext.setParameters(parameters); - - controller.getFlowManager().getParameterContextManager().addParameterContext(parameterContext); - controller.getFlowManager().getParameterContextManager().addParameterContext(referencedContext); - controller.getFlowManager().getParameterContextManager().addParameterContext(referencedContext2); - - controller.getFlowManager().getRootGroup().setParameterContext(parameterContext); - controller.getFlowManager().getRootGroup().setParameterContext(parameterContext); - - // serialize the controller - final ByteArrayOutputStream os = new ByteArrayOutputStream(); - final Document doc = serializer.transform(controller, ScheduledStateLookup.IDENTITY_LOOKUP); - serializer.serialize(doc, os); - - // verify the results contain the serialized string - final String serializedFlow = os.toString(StandardCharsets.UTF_8.name()); - assertTrue(serializedFlow.contains(SERIALIZED_COMMENTS)); - assertFalse(serializedFlow.contains(RAW_COMMENTS)); - assertFalse(serializedFlow.contains("\u0001")); - assertTrue(serializedFlow.contains("referenced-context")); - } - - @Test - public void testSerializationEmoji() throws Exception { - final ProcessorNode dummy = controller.getFlowManager().createProcessor(DummyScheduledProcessor.class.getName(), - UUID.randomUUID().toString(), systemBundle.getBundleDetails().getCoordinate()); - - dummy.setName(RAW_STRING_WITH_EMOJI); - controller.getFlowManager().getRootGroup().addProcessor(dummy); - - // serialize the controller - final ByteArrayOutputStream os = new ByteArrayOutputStream(); - final Document doc = serializer.transform(controller, ScheduledStateLookup.IDENTITY_LOOKUP); - serializer.serialize(doc, os); - - // verify the results contain the serialized string - final String serializedFlow = os.toString(StandardCharsets.UTF_8.name()); - assertTrue(serializedFlow.contains(SERIALIZED_STRING_WITH_EMOJI)); - assertFalse(serializedFlow.contains(RAW_STRING_WITH_EMOJI)); - } -} diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/fingerprint/FingerprintFactoryTest.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/fingerprint/FingerprintFactoryTest.java deleted file mode 100644 index 9936adae32..0000000000 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/fingerprint/FingerprintFactoryTest.java +++ /dev/null @@ -1,405 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.nifi.fingerprint; - -import org.apache.commons.codec.DecoderException; -import org.apache.commons.codec.binary.Hex; -import org.apache.commons.io.IOUtils; -import org.apache.commons.lang3.StringUtils; -import org.apache.nifi.connectable.Position; -import org.apache.nifi.controller.ScheduledState; -import org.apache.nifi.controller.serialization.FlowEncodingVersion; -import org.apache.nifi.controller.serialization.FlowSerializer; -import org.apache.nifi.controller.serialization.ScheduledStateLookup; -import org.apache.nifi.controller.serialization.StandardFlowSerializer; -import org.apache.nifi.encrypt.PropertyEncryptor; -import org.apache.nifi.encrypt.SensitiveValueEncoder; -import org.apache.nifi.groups.RemoteProcessGroup; -import org.apache.nifi.nar.ExtensionManager; -import org.apache.nifi.nar.StandardExtensionDiscoveringManager; -import org.apache.nifi.remote.RemoteGroupPort; -import org.apache.nifi.remote.protocol.SiteToSiteTransportProtocol; -import org.apache.nifi.xml.processing.parsers.DocumentProvider; -import org.apache.nifi.xml.processing.parsers.StandardDocumentProvider; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; -import org.w3c.dom.Document; -import org.w3c.dom.Element; - -import java.io.FileInputStream; -import java.io.IOException; -import java.lang.reflect.Method; -import java.nio.charset.StandardCharsets; -import java.util.Collections; -import java.util.Optional; - -import static org.apache.nifi.controller.serialization.ScheduledStateLookup.IDENTITY_LOOKUP; -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertNotEquals; -import static org.junit.jupiter.api.Assertions.assertNotNull; -import static org.junit.jupiter.api.Assertions.assertTrue; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.when; - -/** - */ -public class FingerprintFactoryTest { - - private PropertyEncryptor encryptor; - private ExtensionManager extensionManager; - private FingerprintFactory fingerprintFactory; - private SensitiveValueEncoder sensitiveValueEncoder; - - @BeforeEach - public void setup() { - encryptor = createEncryptor(); - sensitiveValueEncoder = createSensitiveValueEncoder(); - extensionManager = new StandardExtensionDiscoveringManager(); - fingerprintFactory = new FingerprintFactory(encryptor, extensionManager, sensitiveValueEncoder); - } - - @Test - public void testSameFingerprint() throws IOException { - final String fp1 = fingerprintFactory.createFingerprint(getResourceBytes("/nifi/fingerprint/flow1a.xml"), null); - final String fp2 = fingerprintFactory.createFingerprint(getResourceBytes("/nifi/fingerprint/flow1b.xml"), null); - assertEquals(fp1, fp2); - } - - @Test - public void testDifferentFingerprint() throws IOException { - final String fp1 = fingerprintFactory.createFingerprint(getResourceBytes("/nifi/fingerprint/flow1a.xml"), null); - final String fp2 = fingerprintFactory.createFingerprint(getResourceBytes("/nifi/fingerprint/flow2.xml"), null); - assertNotEquals(fp1, fp2); - } - - @Test - public void testInheritedParameterContextsInFingerprint() throws IOException { - final String flowWithParameterContext = new String(getResourceBytes("/nifi/fingerprint/flow-with-parameter-context.xml")); - final String flowWithNoInheritedParameterContexts = flowWithParameterContext.replaceAll(".*?", ""); - final String flowWithDifferentInheritedParamContextOrder = flowWithParameterContext - .replaceFirst("153a6266-dcd0-33e9-b5af-b8c282d25bf1", "SWAP") - .replaceFirst("253a6266-dcd0-33e9-b5af-b8c282d25bf2", "153a6266-dcd0-33e9-b5af-b8c282d25bf1") - .replaceFirst("SWAP", "253a6266-dcd0-33e9-b5af-b8c282d25bf2"); - - final String originalFingerprint = fingerprintFactory.createFingerprint(flowWithParameterContext.getBytes(StandardCharsets.UTF_8), null); - final String fingerprintWithNoInheritedParameterContexts = fingerprintFactory.createFingerprint(flowWithNoInheritedParameterContexts.getBytes(StandardCharsets.UTF_8), null); - final String fingerprintWithDifferentInheritedParamContextOrder = fingerprintFactory.createFingerprint(flowWithDifferentInheritedParamContextOrder.getBytes(StandardCharsets.UTF_8), null); - - assertNotEquals(originalFingerprint, fingerprintWithNoInheritedParameterContexts); - assertNotEquals(originalFingerprint, fingerprintWithDifferentInheritedParamContextOrder); - } - - @Test - public void testResourceValueInFingerprint() throws IOException { - final String fingerprint = fingerprintFactory.createFingerprint(getResourceBytes("/nifi/fingerprint/flow1a.xml"), null); - assertEquals(3, StringUtils.countMatches(fingerprint, "success")); - assertTrue(fingerprint.contains("In Connection")); - } - - @Test - public void testSameFlowWithDifferentBundleShouldHaveDifferentFingerprints() throws IOException { - final String fp1 = fingerprintFactory.createFingerprint(getResourceBytes("/nifi/fingerprint/flow3-with-bundle-1.xml"), null); - assertTrue(fp1.contains("org.apache.nifinifi-standard-nar1.0")); - - final String fp2 = fingerprintFactory.createFingerprint(getResourceBytes("/nifi/fingerprint/flow3-with-bundle-2.xml"), null); - assertTrue(fp2.contains("org.apache.nifinifi-standard-nar2.0")); - - assertNotEquals(fp1, fp2); - } - - @Test - public void testSameFlowAndOneHasNoBundleShouldHaveDifferentFingerprints() throws IOException { - final String fp1 = fingerprintFactory.createFingerprint(getResourceBytes("/nifi/fingerprint/flow3-with-bundle-1.xml"), null); - assertTrue(fp1.contains("org.apache.nifinifi-standard-nar1.0")); - - final String fp2 = fingerprintFactory.createFingerprint(getResourceBytes("/nifi/fingerprint/flow3-with-no-bundle.xml"), null); - assertTrue(fp2.contains("MISSING_BUNDLE")); - - assertNotEquals(fp1, fp2); - } - - @Test - public void testSameFlowAndOneHasMissingBundleShouldHaveDifferentFingerprints() throws IOException { - final String fp1 = fingerprintFactory.createFingerprint(getResourceBytes("/nifi/fingerprint/flow3-with-bundle-1.xml"), null); - assertTrue(fp1.contains("org.apache.nifinifi-standard-nar1.0")); - - final String fp2 = fingerprintFactory.createFingerprint(getResourceBytes("/nifi/fingerprint/flow3-with-missing-bundle.xml"), null); - assertTrue(fp2.contains("missingmissingmissing")); - - assertNotEquals(fp1, fp2); - } - - @Test - public void testConnectionWithMultipleRelationshipsSortedInFingerprint() throws IOException { - final String fingerprint = fingerprintFactory.createFingerprint(getResourceBytes("/nifi/fingerprint/flow-connection-with-multiple-rels.xml"), null); - assertNotNull(fingerprint); - assertTrue(fingerprint.contains("AAABBBCCCDDD")); - } - - @Test - public void testPublicPortWithAccessPoliciesFingerprint() throws IOException { - final String fingerprint = fingerprintFactory.createFingerprint(getResourceBytes("/nifi/fingerprint/flow1a.xml"), null); - // user access control - assertTrue(fingerprint.contains("user1")); - // group access control - assertTrue(fingerprint.contains("group1")); - // policies fingerprinted - assertTrue(fingerprint.contains("user1group1")); - } - - @Test - public void testPublicPortWithDifferentFingerprintInAccessPolicies() throws IOException { - final String f1 = fingerprintFactory.createFingerprint(getResourceBytes("/nifi/fingerprint/flow1a.xml"), null); - assertEquals(2, StringUtils.countMatches(f1, "user1group1")); - - final Document document = getDocument("src/test/resources/nifi/fingerprint/public-port-with-no-policies.xml"); - final Element rootProcessGroup = document.getDocumentElement(); - - final StringBuilder sb = new StringBuilder(); - fingerprintFactory.addProcessGroupFingerprint(sb, rootProcessGroup, new FlowEncodingVersion(1, 0)); - - final String f2 = sb.toString(); - assertEquals(2, StringUtils.countMatches(f1, "NO_USER_ACCESS_CONTROLNO_GROUP_ACCESS_CONTROL")); - - // actual -> Public-IntrueNO_USER_ACCESS_CONTROLNO_GROUP_ACCESS_CONTROL - // expected -> Public-Intrueuser1group1 - assertNotEquals(StringUtils.countMatches(f1, "user1group1"), StringUtils.countMatches(f2, "user1group1")); - } - - @Test - public void testPublicPortWithNoAccessPoliciesFingerprint() throws IOException { - final Document document = getDocument("src/test/resources/nifi/fingerprint/public-port-with-no-policies.xml"); - final Element rootProcessGroup = document.getDocumentElement(); - - final StringBuilder sb = new StringBuilder(); - fingerprintFactory.addProcessGroupFingerprint(sb, rootProcessGroup, new FlowEncodingVersion(1, 0)); - - final String fingerprint = sb.toString(); - assertTrue(fingerprint.contains("NO_USER_ACCESS_CONTROL")); - assertTrue(fingerprint.contains("NO_GROUP_ACCESS_CONTROL")); - } - - private Document getDocument(final String filePath) throws IOException { - try (final FileInputStream inputStream = new FileInputStream(filePath)) { - final DocumentProvider documentProvider = new StandardDocumentProvider(); - return documentProvider.parse(inputStream); - } - } - - private byte[] getResourceBytes(final String resource) throws IOException { - return IOUtils.toByteArray(FingerprintFactoryTest.class.getResourceAsStream(resource)); - } - - private Element serializeElement(final PropertyEncryptor encryptor, final Class componentClass, final T component, - final String serializerMethodName, ScheduledStateLookup scheduledStateLookup) throws Exception { - final DocumentProvider documentProvider = new StandardDocumentProvider(); - final Document doc = documentProvider.newDocument(); - - final FlowSerializer flowSerializer = new StandardFlowSerializer(); - final Method serializeMethod = StandardFlowSerializer.class.getDeclaredMethod(serializerMethodName, - Element.class, componentClass, ScheduledStateLookup.class, PropertyEncryptor.class); - serializeMethod.setAccessible(true); - final Element rootElement = doc.createElement("root"); - serializeMethod.invoke(flowSerializer, rootElement, component, scheduledStateLookup, encryptor); - return rootElement; - } - - private String fingerprint(final String methodName, final Class inputClass, final T input) throws Exception { - final Method fingerprintFromComponent = FingerprintFactory.class.getDeclaredMethod(methodName, - StringBuilder.class, inputClass); - fingerprintFromComponent.setAccessible(true); - - final StringBuilder fingerprint = new StringBuilder(); - fingerprintFromComponent.invoke(fingerprintFactory, fingerprint, input); - return fingerprint.toString(); - } - - @Test - public void testRemoteProcessGroupFingerprintRaw() throws Exception { - - // Fill out every configuration. - final RemoteProcessGroup component = mock(RemoteProcessGroup.class); - when(component.getName()).thenReturn("name"); - when(component.getIdentifier()).thenReturn("id"); - when(component.getPosition()).thenReturn(new Position(10.5, 20.3)); - when(component.getTargetUri()).thenReturn("http://node1:8080/nifi"); - when(component.getTargetUris()).thenReturn("http://node1:8080/nifi, http://node2:8080/nifi"); - when(component.getNetworkInterface()).thenReturn("eth0"); - when(component.getComments()).thenReturn("comment"); - when(component.getCommunicationsTimeout()).thenReturn("10 sec"); - when(component.getYieldDuration()).thenReturn("30 sec"); - when(component.getTransportProtocol()).thenReturn(SiteToSiteTransportProtocol.RAW); - when(component.getProxyHost()).thenReturn(null); - when(component.getProxyPort()).thenReturn(null); - when(component.getProxyUser()).thenReturn(null); - when(component.getProxyPassword()).thenReturn(null); - when(component.getVersionedComponentId()).thenReturn(Optional.empty()); - - // Assert fingerprints with expected one. - final String expected = "id" + - "NO_VALUE" + - "http://node1:8080/nifi, http://node2:8080/nifi" + - "eth0" + - "10 sec" + - "30 sec" + - "RAW" + - "NO_VALUE" + - "NO_VALUE" + - "NO_VALUE" + - "NO_VALUE"; - - final Element rootElement = serializeElement(encryptor, RemoteProcessGroup.class, component, "addRemoteProcessGroup", IDENTITY_LOOKUP); - final Element componentElement = (Element) rootElement.getElementsByTagName("remoteProcessGroup").item(0); - assertEquals(expected, fingerprint("addRemoteProcessGroupFingerprint", Element.class, componentElement)); - - } - - @Test - public void testRemoteProcessGroupFingerprintWithProxy() throws Exception { - final String proxyPassword = "proxy-pass"; - - // Fill out every configuration. - final RemoteProcessGroup component = mock(RemoteProcessGroup.class); - when(component.getName()).thenReturn("name"); - when(component.getIdentifier()).thenReturn("id"); - when(component.getPosition()).thenReturn(new Position(10.5, 20.3)); - when(component.getTargetUri()).thenReturn("http://node1:8080/nifi"); - when(component.getTargetUris()).thenReturn("http://node1:8080/nifi, http://node2:8080/nifi"); - when(component.getComments()).thenReturn("comment"); - when(component.getCommunicationsTimeout()).thenReturn("10 sec"); - when(component.getYieldDuration()).thenReturn("30 sec"); - when(component.getTransportProtocol()).thenReturn(SiteToSiteTransportProtocol.HTTP); - when(component.getProxyHost()).thenReturn("proxy-host"); - when(component.getProxyPort()).thenReturn(3128); - when(component.getProxyUser()).thenReturn("proxy-user"); - when(component.getProxyPassword()).thenReturn(proxyPassword); - when(component.getVersionedComponentId()).thenReturn(Optional.empty()); - - final String hashedProxyPassword = sensitiveValueEncoder.getEncoded(proxyPassword); - - // Assert fingerprints with expected one. - final String expected = "id" + - "NO_VALUE" + - "http://node1:8080/nifi, http://node2:8080/nifi" + - "NO_VALUE" + - "10 sec" + - "30 sec" + - "HTTP" + - "proxy-host" + - "3128" + - "proxy-user" + - hashedProxyPassword; - - final Element rootElement = serializeElement(encryptor, RemoteProcessGroup.class, component, "addRemoteProcessGroup", IDENTITY_LOOKUP); - final Element componentElement = (Element) rootElement.getElementsByTagName("remoteProcessGroup").item(0); - assertEquals(expected, fingerprint("addRemoteProcessGroupFingerprint", Element.class, componentElement)); - } - - @Test - public void testRemotePortFingerprint() throws Exception { - - // Fill out every configuration. - final RemoteProcessGroup groupComponent = mock(RemoteProcessGroup.class); - when(groupComponent.getName()).thenReturn("name"); - when(groupComponent.getIdentifier()).thenReturn("id"); - when(groupComponent.getPosition()).thenReturn(new Position(10.5, 20.3)); - when(groupComponent.getTargetUri()).thenReturn("http://node1:8080/nifi"); - when(groupComponent.getTransportProtocol()).thenReturn(SiteToSiteTransportProtocol.RAW); - when(groupComponent.getVersionedComponentId()).thenReturn(Optional.empty()); - - final RemoteGroupPort portComponent = mock(RemoteGroupPort.class); - when(groupComponent.getInputPorts()).thenReturn(Collections.singleton(portComponent)); - when(portComponent.getName()).thenReturn("portName"); - when(portComponent.getIdentifier()).thenReturn("portId"); - when(portComponent.getPosition()).thenReturn(new Position(10.5, 20.3)); - when(portComponent.getComments()).thenReturn("portComment"); - when(portComponent.getScheduledState()).thenReturn(ScheduledState.RUNNING); - when(portComponent.getMaxConcurrentTasks()).thenReturn(3); - when(portComponent.isUseCompression()).thenReturn(true); - when(portComponent.getBatchCount()).thenReturn(1234); - when(portComponent.getBatchSize()).thenReturn("64KB"); - when(portComponent.getBatchDuration()).thenReturn("10sec"); - // Serializer doesn't serialize if a port doesn't have any connection. - when(portComponent.hasIncomingConnection()).thenReturn(true); - when(portComponent.getVersionedComponentId()).thenReturn(Optional.empty()); - - // Assert fingerprints with expected one. - final String expected = "portId" + - "NO_VALUE" + - "NO_VALUE" + - "3" + - "true" + - "1234" + - "64KB" + - "10sec"; - - final Element rootElement = serializeElement(encryptor, RemoteProcessGroup.class, groupComponent, "addRemoteProcessGroup", IDENTITY_LOOKUP); - final Element componentElement = (Element) rootElement.getElementsByTagName("inputPort").item(0); - assertEquals(expected, fingerprint("addRemoteGroupPortFingerprint", Element.class, componentElement)); - } - - @Test - public void testControllerServicesIncludedInGroupFingerprint() throws IOException { - final Document document = getDocument("src/test/resources/nifi/fingerprint/group-with-controller-services.xml"); - final Element processGroup = document.getDocumentElement(); - - final StringBuilder sb = new StringBuilder(); - fingerprintFactory.addProcessGroupFingerprint(sb, processGroup, new FlowEncodingVersion(1, 0)); - - final String fingerprint = sb.toString(); - final String[] criticalFingerprintValues = new String[] { - "1234", - "s1", "service1", "prop1", "value1", "org.apache.nifi.services.FingerprintControllerService", - "s2", "service2", "another property", "another value", "org.apache.nifi.services.AnotherService", - }; - - for (final String criticalValue : criticalFingerprintValues) { - assertTrue( fingerprint.contains(criticalValue), - "Fingerprint did not contain '" + criticalValue + "'"); - } - - // Ensure that 's1' comes before 's2' in the fingerprint - assertTrue(fingerprint.indexOf("FingerprintControllerService") < fingerprint.indexOf("AnotherService")); - } - - private PropertyEncryptor createEncryptor() { - return new PropertyEncryptor() { - @Override - public String encrypt(String property) { - return Hex.encodeHexString(property.getBytes(StandardCharsets.UTF_8)); - } - - @Override - public String decrypt(String encryptedProperty) { - try { - return new String(Hex.decodeHex(encryptedProperty)); - } catch (DecoderException e) { - throw new IllegalArgumentException(e); - } - } - }; - } - - private SensitiveValueEncoder createSensitiveValueEncoder() { - return new SensitiveValueEncoder() { - @Override - public String getEncoded(String sensitivePropertyValue) { - return String.format("[MASKED] %s", sensitivePropertyValue); - } - }; - } -} diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/persistence/TestFlowConfigurationArchiveManager.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/persistence/TestFlowConfigurationArchiveManager.java index ce7241d274..3a2ddb51eb 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/persistence/TestFlowConfigurationArchiveManager.java +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/persistence/TestFlowConfigurationArchiveManager.java @@ -45,7 +45,7 @@ import static org.mockito.Mockito.when; public class TestFlowConfigurationArchiveManager { - private final File flowXmlFile = new File("./target/flow-archive/flow.xml.gz"); + private final File flowXmlFile = new File("./target/flow-archive/flow.json.gz"); private final File archiveDir = new File("./target/flow-archive"); @BeforeEach diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/resources/conf/0bytes.xml b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/resources/conf/0bytes.json similarity index 100% rename from nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/resources/conf/0bytes.xml rename to nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/resources/conf/0bytes.json diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/resources/conf/all-flow-corrupt.json b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/resources/conf/all-flow-corrupt.json new file mode 100644 index 0000000000..1b23d61fc9 --- /dev/null +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/resources/conf/all-flow-corrupt.json @@ -0,0 +1,420 @@ +{ + "encodingVersion": { + "majorVersion": 2, + "minorVersion": 0 + }, + "maxTimerDrivenThreadCount": 10, + "maxEventDrivenThreadCount": 5, + "registries": [], + "parameterContexts": [], + "parameterProviders": [], + "controllerServices": [], + "reportingTasks": [], + "templates": [], + "rootGroup": { + "identifier": "e4e64ef8-9ad3-39d2-bfef-fc985d5ad4ef", + "instanceIdentifier": "4190002f-8b17-4772-bd62-8efec12c0cd0", + "name": "new group 1", + "comments": "new comments", + "position": { + "x": 0, + "y": 0 + }, + "processGroups": [ + { + "identifier": "a698ede2-d790-30dd-87ef-ea1115f3f4ba", + "instanceIdentifier": "994be764-7e03-421c-ab10-f21956073c2a", + "name": "new gr", + "comments": " some comments ", + "position": { + "x": 0, + "y": 0 + }, + "processGroups": [], + "remoteProcessGroups": [], + "processors": [ + { + "identifier": "ef027853-683e-3fe9-a625-be7e85c6ad5b", + "instanceIdentifier": "d489d825-67da-45eb-a714-98a811608daf", + "name": "new AttributeLoggerProcessor", + "comments": "", + "position": { + "x": 0, + "y": 0 + }, + "bundle": { + "group": "default", + "artifact": "unknown", + "version": "unversioned" + }, + "properties": { + "log.level": "enc{null}" + }, + "propertyDescriptors": {}, + "style": { + "type": "org.apache.nifi.test.processors.StubAttributeLoggerProcessor", + "penaltyDuration": "30 sec", + "yieldDuration": "1 sec", + "bulletinLevel": "WARN", + "runDurationMillis": 0 + }, + "annotationData": "", + "schedulingStrategy": "TIMER_DRIVEN", + "executionNode": "ALL", + "penaltyDuration": "30 sec", + "yieldDuration": "1 sec", + "bulletinLevel": "WARN", + "runDurationMillis": 0, + "concurrentlySchedulableTaskCount": 1, + "autoTerminatedRelationships": [], + "scheduledState": "ENABLED", + "retryCount": 10, + "retriedRelationships": [], + "backoffMechanism": "PENALIZE_FLOWFILE", + "maxBackoffPeriod": "10 mins", + "componentType": "PROCESSOR", + "groupIdentifier": "a698ede2-d790-30dd-87ef-ea1115f3f4ba" + } + ], + "inputPorts": [ + { + "identifier": "12fbd345-8712-3dd1-b9a7-e702599e7025", + "instanceIdentifier": "167e3556-b6f2-4e9a-b143-e02c91bb7fb6", + "name": "Group In", + "position": { + "x": 0, + "y": 0 + }, + "type": "INPUT_PORT", + "concurrentlySchedulableTaskCount": 1, + "scheduledState": "ENABLED", + "allowRemoteAccess": false, + "componentType": "INPUT_PORT", + "groupIdentifier": "a698ede2-d790-30dd-87ef-ea1115f3f4ba" + } + ], + "outputPorts": [ + { + "identifier": "5a75b785-518a-34b8-bf4f-1df2dde51ca4", + "instanceIdentifier": "b14135ee-411d-46be-b377-5ed4c8d72cbe", + "name": "Group Out", + "position": { + "x": 0, + "y": 0 + }, + "type": "OUTPUT_PORT", + "concurrentlySchedulableTaskCount": 1, + "scheduledState": "ENABLED", + "allowRemoteAccess": false, + "componentType": "OUTPUT_PORT", + "groupIdentifier": "a698ede2-d790-30dd-87ef-ea1115f3f4ba" + } + ], + "connections": [ + { + "identifier": "6aa2eaf4-c256-3570-b90e-8162a061a942", + "instanceIdentifier": "b8886cad-373f-4ac4-8028-3968fdcd5252", + "name": "new name", + "source": { + "id": "12fbd345-8712-3dd1-b9a7-e702599e7025", + "type": "INPUT_PORT", + "groupId": "a698ede2-d790-30dd-87ef-ea1115f3f4ba", + "name": "Group In", + "instanceIdentifier": "167e3556-b6f2-4e9a-b143-e02c91bb7fb6" + }, + "destination": { + "id": "5a75b785-518a-34b8-bf4f-1df2dde51ca4", + "type": "OUTPUT_PORT", + "groupId": "a698ede2-d790-30dd-87ef-ea1115f3f4ba", + "name": "Group Out", + "instanceIdentifier": "b14135ee-411d-46be-b377-5ed4c8d72cbe" + }, + "labelIndex": 1, + "zIndex": 0, + "selectedRelationships": [ + "" + ], + "backPressureObjectThreshold": 10000, + "backPressureDataSizeThreshold": "1 GB", + "flowFileExpiration": "0 sec", + "prioritizers": [], + "bends": [], + "loadBalanceStrategy": "DO_NOT_LOAD_BALANCE", + "loadBalanceCompression": "DO_NOT_COMPRESS", + "componentType": "CONNECTION", + "groupIdentifier": "a698ede2-d790-30dd-87ef-ea1115f3f4ba" + } + ], + "labels": [], + "funnels": [], + "controllerServices": [], + "variables": {}, + "defaultFlowFileExpiration": "0 sec", + "defaultBackPressureObjectThreshold": 10000, + "defaultBackPressureDataSizeThreshold": "1 GB", + "componentType": "PROCESS_GROUP", + "flowFileOutboundPolicy": "STREAM_WHEN_AVAILABLE", + "flowFileConcurrency": "UNBOUNDED", + "groupIdentifier": "e4e64ef8-9ad3-39d2-bfef-fc985d5ad4ef" + } + ], + "remoteProcessGroups": [ + { + "identifier": "230fc891-7a06-3db7-9a53-f133e9880e9a", + "instanceIdentifier": "52c47e32-91c5-40d4-bf0a-37d672ce8190", + "name": "new Unable to Connect", + "comments": "", + "position": { + "x": 0, + "y": 0 + }, + "targetUri": "http://localhost:8080", + "targetUris": "http://localhost:8080", + "communicationsTimeout": "30 sec", + "yieldDuration": "10 sec", + "transportProtocol": "RAW", + "inputPorts": [], + "outputPorts": [], + "componentType": "REMOTE_PROCESS_GROUP", + "groupIdentifier": "e4e64ef8-9ad3-39d2-bfef-fc985d5ad4ef" + } + ], + "processors": [ + { + "identifier": "eb5a37b5-2869-3d20-8920-a92e5fe9d425", + "instanceIdentifier": "1bc2d455-a45d-43fc-a96c-909b6a00311e", + "name": "AttributeLoggerProcessor", + "comments": "", + "position": { + "x": 0, + "y": 0 + }, + "type": "org.apache.nifi.test.processors.StubAttributeLoggerProcessor", + "bundle": { + "group": "default", + "artifact": "unknown", + "version": "unversioned" + }, + "properties": { + "log.level": "enc{null}", + "log.payload": "enc{null}" + }, + "propertyDescriptors": {}, + "style": {}, + "annotationData": "", + "schedulingStrategy": "TIMER_DRIVEN", + "executionNode": "ALL", + "penaltyDuration": "30 sec", + "yieldDuration": "1 sec", + "bulletinLevel": "WARN", + "runDurationMillis": 0, + "concurrentlySchedulableTaskCount": 1, + "autoTerminatedRelationships": [], + "scheduledState": "ENABLED", + "retryCount": 10, + "retriedRelationships": [], + "backoffMechanism": "PENALIZE_FLOWFILE", + "maxBackoffPeriod": "10 mins", + "componentType": "PROCESSOR", + "groupIdentifier": "e4e64ef8-9ad3-39d2-bfef-fc985d5ad4ef" + }, + { + "identifier": "0e09a297-3fe6-30c1-b289-7e3652dd75a7", + "instanceIdentifier": "7fc60e56-c7cd-47bc-8290-bcd52b42b4ac", + "name": "new TerminationFileProcessor", + "position": { + "x": 0, + "y": 0 + }, + "type": "org.apache.nifi.test.processors.StubTerminationFileProcessor", + "bundle": { + "group": "default", + "artifact": "unknown", + "version": "unversioned" + }, + "properties": {}, + "propertyDescriptors": {}, + "style": {}, + "annotationData": "some annotation data", + "schedulingStrategy": "TIMER_DRIVEN", + "executionNode": "ALL", + "penaltyDuration": "30 sec", + "yieldDuration": "1 sec", + "bulletinLevel": "WARN", + "runDurationMillis": 0, + "concurrentlySchedulableTaskCount": 10, + "autoTerminatedRelationships": [], + "scheduledState": "ENABLED", + "retryCount": 10, + "retriedRelationships": [], + "backoffMechanism": "PENALIZE_FLOWFILE", + "maxBackoffPeriod": "10 mins", + "componentType": "PROCESSOR", + "groupIdentifier": "e4e64ef8-9ad3-39d2-bfef-fc985d5ad4ef" + } + ], + "inputPorts": [ + { + "identifier": "46ed6e5d-bcc9-3ad2-a88e-c5e779e30b08", + "instanceIdentifier": "e8cf322a-7855-442b-8478-b76520ae62a2", + "name": "new Remote In", + "position": { + "x": 0, + "y": 0 + }, + "type": "INPUT_PORT", + "concurrentlySchedulableTaskCount": 1, + "scheduledState": "ENABLED", + "allowRemoteAccess": true, + "componentType": "INPUT_PORT", + "groupIdentifier": "e4e64ef8-9ad3-39d2-bfef-fc985d5ad4ef" + } + ], + "outputPorts": [ + { + "identifier": "915ae973-0cbf-3001-a587-9e0e7f55dc55", + "instanceIdentifier": "069757e4-a3c9-474f-9593-c55516c510f8", + "name": "new Remote Out", + "position": { + "x": 0, + "y": 0 + }, + "type": "OUTPUT_PORT", + "concurrentlySchedulableTaskCount": 1, + "scheduledState": "ENABLED", + "allowRemoteAccess": true, + "componentType": "OUTPUT_PORT", + "groupIdentifier": "e4e64ef8-9ad3-39d2-bfef-fc985d5ad4ef" + } + ], + "connections": [ + { + "identifier": "48cd96e2-eeaf-3ae6-8f20-9a072007ab35", + "instanceIdentifier": "17a9cdd4-de3d-4d14-a57b-3dee8d503d3a", + "name": "", + "source": { + "id": "5a75b785-518a-34b8-bf4f-1df2dde51ca4", + "type": "OUTPUT_PORT", + "groupId": "a698ede2-d790-30dd-87ef-ea1115f3f4ba", + "name": "Group Out", + "instanceIdentifier": "b14135ee-411d-46be-b377-5ed4c8d72cbe" + }, + "destination": { + "id": "7fc60e56-c7cd-47bc-8290-bcd52b42b42c", + "type": "PROCESSOR", + "groupId": "e4e64ef8-9ad3-39d2-bfef-fc985d5ad4ef", + "name": "new TerminationFileProcessor", + "instanceIdentifier": "7fc60e56-c7cd-47bc-8290-bcd52b42b4ac" + }, + "labelIndex": 1, + "zIndex": 0, + "selectedRelationships": [ + "" + ], + "backPressureObjectThreshold": 10000, + "backPressureDataSizeThreshold": "1 GB", + "flowFileExpiration": "0 sec", + "prioritizers": [], + "bends": [], + "loadBalanceStrategy": "DO_NOT_LOAD_BALANCE", + "loadBalanceCompression": "DO_NOT_COMPRESS", + "componentType": "CONNECTION", + "groupIdentifier": "e4e64ef8-9ad3-39d2-bfef-fc985d5ad4ef" + }, + { + "identifier": "8d8d8473-cc5a-3b78-be36-494f983cb067", + "instanceIdentifier": "edba74b5-8bd9-44a0-b0d7-1adfc946e948", + "name": "", + "source": { + "id": "eb5a37b5-2869-3d20-8920-a92e5fe9d425", + "type": "PROCESSOR", + "groupId": "e4e64ef8-9ad3-39d2-bfef-fc985d5ad4ef", + "name": "AttributeLoggerProcessor", + "comments": "", + "instanceIdentifier": "1bc2d455-a45d-43fc-a96c-909b6a00311e" + }, + "destination": { + "id": "12fbd345-8712-3dd1-b9a7-e702599e7025", + "type": "INPUT_PORT", + "groupId": "a698ede2-d790-30dd-87ef-ea1115f3f4ba", + "name": "Group In", + "instanceIdentifier": "167e3556-b6f2-4e9a-b143-e02c91bb7fb6" + }, + "labelIndex": 1, + "zIndex": 0, + "selectedRelationships": [ + "success" + ], + "backPressureObjectThreshold": 10000, + "backPressureDataSizeThreshold": "1 GB", + "flowFileExpiration": "0 sec", + "prioritizers": [], + "bends": [], + "loadBalanceStrategy": "DO_NOT_LOAD_BALANCE", + "loadBalanceCompression": "DO_NOT_COMPRESS", + "componentType": "CONNECTION", + "groupIdentifier": "e4e64ef8-9ad3-39d2-bfef-fc985d5ad4ef" + }, + { + "identifier": "2cf91cf2-69f6-33aa-9a9a-423c7d1a9e7c", + "instanceIdentifier": "cede7d64-040e-4951-bea4-04c100bf1064", + "name": "", + "source": { + "id": "46ed6e5d-bcc9-3ad2-a88e-c5e779e30b08", + "type": "INPUT_PORT", + "groupId": "e4e64ef8-9ad3-39d2-bfef-fc985d5ad4ef", + "name": "new Remote In", + "instanceIdentifier": "e8cf322a-7855-442b-8478-b76520ae62a2" + }, + "destination": { + "id": "eb5a37b5-2869-3d20-8920-a92e5fe9d425", + "type": "PROCESSOR", + "groupId": "e4e64ef8-9ad3-39d2-bfef-fc985d5ad4ef", + "name": "AttributeLoggerProcessor", + "comments": "", + "instanceIdentifier": "1bc2d455-a45d-43fc-a96c-909b6a00311e" + }, + "labelIndex": 1, + "zIndex": 0, + "selectedRelationships": [ + "" + ], + "backPressureObjectThreshold": 10000, + "backPressureDataSizeThreshold": "1 GB", + "flowFileExpiration": "0 sec", + "prioritizers": [], + "bends": [], + "loadBalanceStrategy": "DO_NOT_LOAD_BALANCE", + "loadBalanceCompression": "DO_NOT_COMPRESS", + "componentType": "CONNECTION", + "groupIdentifier": "e4e64ef8-9ad3-39d2-bfef-fc985d5ad4ef" + } + ], + "labels": [ + { + "identifier": "e64f82ca-1ee0-37aa-a3e2-26e35b247f30", + "instanceIdentifier": "59a5a3ff-1b87-45c6-b5b4-307735baf2b7", + "position": { + "x": 0, + "y": 0 + }, + "label": "", + "zIndex": 0, + "width": 150, + "height": 150, + "style": {}, + "componentType": "LABEL", + "groupIdentifier": "e4e64ef8-9ad3-39d2-bfef-fc985d5ad4ef" + } + ], + "funnels": [], + "controllerServices": [], + "variables": {}, + "defaultFlowFileExpiration": "0 sec", + "defaultBackPressureObjectThreshold": 10000, + "defaultBackPressureDataSizeThreshold": "1 GB", + "componentType": "PROCESS_GROUP", + "flowFileOutboundPolicy": "STREAM_WHEN_AVAILABLE", + "flowFileConcurrency": "UNBOUNDED" + } +} \ No newline at end of file diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/resources/conf/all-flow-corrupt.xml b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/resources/conf/all-flow-corrupt.xml deleted file mode 100644 index 5aaccae911..0000000000 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/resources/conf/all-flow-corrupt.xml +++ /dev/null @@ -1,201 +0,0 @@ - - - - 15 - - 4190002f-8b17-4772-bd62-8efec12c0cd0 - new group 1 - - - new comments - - 7fc60e56-c7cd-47bc-8290-bcd52b42b4ac - new TerminationFileProcessor - - - - org.apache.nifi.test.processors.StubAttributeLoggerProcessor - 1 - 0 - 0 - false - false - 0 - - log.level - trace - - - log.payload - true - - - - - e8cf322a-7855-442b-8478-b76520ae62a2 - new Remote In - - - - - 069757e4-a3c9-474f-9593-c55516c510f8 - new Remote Out - - - - localhost - 8080 - http - e8cf322a-7855-442b-8478-b76520ae62a2 - - - - - 994be764-7e03-421c-ab10-f21956073c2a - new gr - - - some comments - - d489d825-67da-45eb-a714-98a811608daf - new AttributeLoggerProcessor - - - - org.apache.nifi.test.processors.StubAttributeLoggerProcessor - 1 - 0 - 0 - false - false - 0 - - log.level - debug - - - - - 167e3556-b6f2-4e9a-b143-e02c91bb7fb6 - Group In - - - - - b14135ee-411d-46be-b377-5ed4c8d72cbe - Group Out - - - - - b8886cad-373f-4ac4-8028-3968fdcd5252 - new name - - 1 - 0 - - 167e3556-b6f2-4e9a-b143-e02c91bb7fb6 - 994be764-7e03-421c-ab10-f21956073c2a - INPUT_PORT - b14135ee-411d-46be-b377-5ed4c8d72cbe - 994be764-7e03-421c-ab10-f21956073c2a - OUTPUT_PORT - false - false - - - - - 52c47e32-91c5-40d4-bf0a-37d672ce8190 - new Unable to Connect - - - - http://localhost:8080 - - - cede7d64-040e-4951-bea4-04c100bf1064 - - - 1 - 0 - - new comments - - 7fc60e56-c7cd-47bc-8290-bcd52b42b42c - new TerminationFileProcessor - - - - org.apache.nifi.test.processors.StubTerminationFileProcessor - 10 - 10 - 10 - false - true - 10 - org.apache.nifi.comparator.NewestFlowFileFirstComparator - some annotation data - - - 1bc2d455-a45d-43fc-a96c-909b6a00311e - AttributeLoggerProcessor - - - - org.apache.nifi.test.processors.StubAttributeLoggerProcessor - 1 - 0 - 0 - false - false - 0 - - log.level - trace - - - log.payload - true - - - - - e8cf322a-7855-442b-8478-b76520ae62a2 - new Remote In - - - - - 069757e4-a3c9-474f-9593-c55516c510f8 - new Remote Out - - - - - - 994be764-7e03-421c-ab10-f21956073c2a - new gr - - - some comments - - d489d825-67da-45eb-a714-98a811608daf - new AttributeLoggerProcessor - - - - org.apache.nifi.test.processors.StubAttributeLoggerProcessor - 1 - 0 - 0 - false - false - 0 - - log.level - debug - - - - - 167e3556-b6f2-4e9a-b143-e02c91bb7fb6 - Group In - - - - - b14135ee-411d-46be-b377-5ed4c8d72cbe - Group Out - - - - - b8886cad-373f-4ac4-8028-3968fdcd5252 - new name - - 1 - 0 - - 167e3556-b6f2-4e9a-b143-e02c91bb7fb6 - 994be764-7e03-421c-ab10-f21956073c2a - INPUT_PORT - b14135ee-411d-46be-b377-5ed4c8d72cbe - 994be764-7e03-421c-ab10-f21956073c2a - OUTPUT_PORT - false - false - - - - - 52c47e32-91c5-40d4-bf0a-37d672ce8190 - new Unable to Connect - - - - http://localhost:8080 - - - cede7d64-040e-4951-bea4-04c100bf1064 - - - 1 - 0 - - new comments - - 7fc60e56-c7cd-47bc-8290-bcd52b42b4ac - new TerminationFileProcessor - - - - org.apache.nifi.test.processors.StubTerminationFileProcessor - 10 - 10 - 10 - false - true - 10 - org.apache.nifi.comparator.NewestFlowFileFirstComparator - some annotation data - - - 1bc2d455-a45d-43fc-a96c-909b6a00311e - AttributeLoggerProcessor - - - - org.apache.nifi.test.processors.StubAttributeLoggerProcessor - 1 - 0 - 0 - false - false - 0 - - log.level - trace - - - log.payload - true - - - - - e8cf322a-7855-442b-8478-b76520ae62a2 - new Remote In - - - - - 069757e4-a3c9-474f-9593-c55516c510f8 - new Remote Out - - - - localhost - 8080 - http - e8cf322a-7855-442b-8478-b76520ae62a2 - - - - - 994be764-7e03-421c-ab10-f21956073c2a - new gr - - - some comments - - d489d825-67da-45eb-a714-98a811608daf - new AttributeLoggerProcessor - - - - org.apache.nifi.test.processors.StubAttributeLoggerProcessor - 1 - 0 - 0 - false - false - 0 - - log.level - debug - - - - - 167e3556-b6f2-4e9a-b143-e02c91bb7fb6 - Group In - - - - - b14135ee-411d-46be-b377-5ed4c8d72cbe - Group Out - - - - - b8886cad-373f-4ac4-8028-3968fdcd5252 - new name - - 1 - 0 - - 167e3556-b6f2-4e9a-b143-e02c91bb7fb6 - 994be764-7e03-421c-ab10-f21956073c2a - INPUT_PORT - b14135ee-411d-46be-b377-5ed4c8d72cbe - 994be764-7e03-421c-ab10-f21956073c2a - OUTPUT_PORT - false - false - - - - - 52c47e32-91c5-40d4-bf0a-37d672ce8190 - new Unable to Connect - - - - http://localhost:8080 - - - cede7d64-040e-4951-bea4-04c100bf1064 - - - 1 - 0 - - comments for Group 1 - - 7fc60e56-c7cd-47bc-8290-bcd52b42b42c - TerminationFileProcessor - - - termination processor comment - org.apache.nifi.test.processors.StubTerminationFileProcessor - 1 - 0 - 0 - false - false - 0 - org.apache.nifi.comparator.FirstInFirstOutComparator - org.apache.nifi.test.filters.StubFlowFileInputFilter - org.apache.nifi.test.filters.StubFlowFileOutputFilter - some annotation data - - - 1bc2d455-a45d-43fc-a96c-909b6a00311e - AttributeLoggerProcessor - - - - org.apache.nifi.test.processors.StubAttributeLoggerProcessor - 1 - 0 - 0 - false - false - 0 - - log.level - trace - - - log.payload - true - - - - - e8cf322a-7855-442b-8478-b76520ae62a2 - Remote In - - - - - 069757e4-a3c9-474f-9593-c55516c510f8 - Remote Out - - - - - - 994be764-7e03-421c-ab10-f21956073c2a - gr - - - - - d489d825-67da-45eb-a714-98a811608daf - AttributeLoggerProcessor - - - - org.apache.nifi.test.processors.StubAttributeLoggerProcessor - 1 - 0 - 0 - false - false - 0 - - log.level - debug - - - - - 167e3556-b6f2-4e9a-b143-e02c91bb7fb6 - Group In - - - - - b14135ee-411d-46be-b377-5ed4c8d72cbe - Group Out - - - - - b8886cad-373f-4ac4-8028-3968fdcd5252 - - - 1 - 0 - - - http://localhost:8080 - - - cede7d64-040e-4951-bea4-04c100bf1064 - - - 1 - 0 - - - - 1234-4321-1234-4321 - Data Destroyer - - - - org.apache.nifi.processors.standard.TerminationFileProcessor - 5 - 0 - 20000 - true - - - \ No newline at end of file diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/resources/conf/parameter-context-flow-error.json b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/resources/conf/parameter-context-flow-error.json new file mode 100644 index 0000000000..055c6d39f6 --- /dev/null +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/resources/conf/parameter-context-flow-error.json @@ -0,0 +1,102 @@ +{ + "encodingVersion": { + "majorVersion": 2, + "minorVersion": 0 + }, + "maxTimerDrivenThreadCount": 10, + "maxEventDrivenThreadCount": 1, + "registries": [], + "parameterContexts": [ + { + "identifier": "5c18ef72-7715-34b7-b43c-497dc507aeab", + "instanceIdentifier": "context", + "name": "Context", + "parameters": [ + { + "name": "foo", + "description": "", + "sensitive": true, + "provided": false + } + ], + "inheritedParameterContexts": [ + "Referenced Context", + "Referenced Context 2" + ], + "description": "", + "componentType": "PARAMETER_CONTEXT" + }, + { + "identifier": "3fe43e2f-8563-320a-b4a8-1c831c2ff4e1", + "instanceIdentifier": "referenced-context", + "name": "Referenced Context", + "parameters": [], + "inheritedParameterContexts": [], + "description": "", + "componentType": "PARAMETER_CONTEXT" + } + ], + "parameterProviders": [], + "controllerServices": [], + "reportingTasks": [], + "templates": [], + "rootGroup": { + "identifier": "3bf27c19-a11d-36a4-874d-5a32ef4bef5c", + "instanceIdentifier": "2ae3cdb4-0179-1000-6ddc-ed1dca231bac", + "name": "NiFi Flow", + "comments": "", + "position": { + "x": 0, + "y": 0 + }, + "processGroups": [], + "remoteProcessGroups": [], + "processors": [ + { + "identifier": "375b0484-421b-3815-92ea-28c35d06d6cc", + "instanceIdentifier": "c40fc154-ef89-48b8-82bf-ff6cc9e8f591", + "name": "DummyScheduledProcessor", + "comments": " \"This\" is an ' example with many characters that need to be filtered and escaped in it.  † ", + "position": { + "x": 0, + "y": 0 + }, + "type": "org.apache.nifi.controller.DummyScheduledProcessor", + "properties": {}, + "propertyDescriptors": {}, + "style": {}, + "schedulingPeriod": "0 0 0 1/1 * ?", + "schedulingStrategy": "CRON_DRIVEN", + "executionNode": "ALL", + "penaltyDuration": "30 sec", + "yieldDuration": "1 sec", + "bulletinLevel": "WARN", + "runDurationMillis": 0, + "concurrentlySchedulableTaskCount": 5, + "autoTerminatedRelationships": [], + "scheduledState": "ENABLED", + "retryCount": 10, + "retriedRelationships": [], + "backoffMechanism": "PENALIZE_FLOWFILE", + "maxBackoffPeriod": "10 mins", + "componentType": "PROCESSOR", + "groupIdentifier": "3bf27c19-a11d-36a4-874d-5a32ef4bef5c" + } + ], + "inputPorts": [], + "outputPorts": [], + "connections": [], + "labels": [], + "funnels": [], + "controllerServices": [], + "variables": { + "Name with escape needed": "Value with escape needed" + }, + "defaultFlowFileExpiration": "0 sec", + "defaultBackPressureObjectThreshold": 10000, + "defaultBackPressureDataSizeThreshold": "1 GB", + "componentType": "PROCESS_GROUP", + "flowFileOutboundPolicy": "STREAM_WHEN_AVAILABLE", + "flowFileConcurrency": "UNBOUNDED" + } +} \ No newline at end of file diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/resources/conf/parameter-context-flow-error.xml b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/resources/conf/parameter-context-flow-error.xml deleted file mode 100644 index 5d17e9291d..0000000000 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/resources/conf/parameter-context-flow-error.xml +++ /dev/null @@ -1,70 +0,0 @@ - - - - 10 - 1 - - - - context - Context - - referenced-context - referenced-context-2 - - foo - - true - enc{c5e2924eeec6d395ba09090cd50c5f2cee1ba23735d7eadfedae96d0b0fd257b} - - - - referenced-context - Referenced Context - - - - - 2ae3cdb4-0179-1000-6ddc-ed1dca231bac - NiFi Flow - - - UNBOUNDED - STREAM_WHEN_AVAILABLE - - c40fc154-ef89-48b8-82bf-ff6cc9e8f591 - DummyScheduledProcessor - - - <tagName> "This" is an ' example with many characters that need to be filtered and escaped in it.  † - org.apache.nifi.controller.DummyScheduledProcessor - 5 - 0 0 0 1/1 * ? - 30 sec - 1 sec - WARN - false - STOPPED - CRON_DRIVEN - ALL - 0 - - - context - - - - \ No newline at end of file diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/resources/conf/parameter-context-flow.json b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/resources/conf/parameter-context-flow.json new file mode 100644 index 0000000000..1237181fea --- /dev/null +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/resources/conf/parameter-context-flow.json @@ -0,0 +1 @@ +{"encodingVersion":{"majorVersion":2,"minorVersion":0},"maxTimerDrivenThreadCount":10,"maxEventDrivenThreadCount":1,"registries":[],"parameterContexts":[{"identifier":"5c18ef72-7715-34b7-b43c-497dc507aeab","instanceIdentifier":"context","name":"Context","parameters":[{"name":"foo","description":"","sensitive":true,"provided":false}],"inheritedParameterContexts":["Referenced Context","Referenced Context 2"],"description":"","componentType":"PARAMETER_CONTEXT"},{"identifier":"3fe43e2f-8563-320a-b4a8-1c831c2ff4e1","instanceIdentifier":"referenced-context","name":"Referenced Context","parameters":[],"inheritedParameterContexts":[],"description":"","componentType":"PARAMETER_CONTEXT"},{"identifier":"7167eddc-85ec-3c8d-ae9e-ddfc8a18d9d2","instanceIdentifier":"referenced-context-2","name":"Referenced Context 2","parameters":[],"inheritedParameterContexts":[],"description":"","componentType":"PARAMETER_CONTEXT"}],"parameterProviders":[],"controllerServices":[],"reportingTasks":[],"templates":[],"rootGroup":{"identifier":"3bf27c19-a11d-36a4-874d-5a32ef4bef5c","instanceIdentifier":"2ae3cdb4-0179-1000-6ddc-ed1dca231bac","name":"NiFi Flow","comments":"","position":{"x":0.0,"y":0.0},"processGroups":[],"remoteProcessGroups":[],"processors":[{"identifier":"375b0484-421b-3815-92ea-28c35d06d6cc","instanceIdentifier":"c40fc154-ef89-48b8-82bf-ff6cc9e8f591","name":"DummyScheduledProcessor","comments":" \"This\" is an ' example with many characters that need to be filtered and escaped in it.  † ","position":{"x":0.0,"y":0.0},"type":"org.apache.nifi.controller.DummyScheduledProcessor","bundle":{"group":"default","artifact":"unknown","version":"unversioned"},"properties":{},"propertyDescriptors":{},"style":{},"schedulingPeriod":"0 0 0 1/1 * ?","schedulingStrategy":"CRON_DRIVEN","executionNode":"ALL","penaltyDuration":"30 sec","yieldDuration":"1 sec","bulletinLevel":"WARN","runDurationMillis":0,"concurrentlySchedulableTaskCount":5,"autoTerminatedRelationships":[],"scheduledState":"ENABLED","retryCount":10,"retriedRelationships":[],"backoffMechanism":"PENALIZE_FLOWFILE","maxBackoffPeriod":"10 mins","componentType":"PROCESSOR","groupIdentifier":"3bf27c19-a11d-36a4-874d-5a32ef4bef5c"}],"inputPorts":[],"outputPorts":[],"connections":[],"labels":[],"funnels":[],"controllerServices":[],"variables":{"Name with escape needed":"Value with escape needed"},"defaultFlowFileExpiration":"0 sec","defaultBackPressureObjectThreshold":10000,"defaultBackPressureDataSizeThreshold":"1 GB","componentType":"PROCESS_GROUP","flowFileOutboundPolicy":"STREAM_WHEN_AVAILABLE","flowFileConcurrency":"UNBOUNDED"}} \ No newline at end of file diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/resources/conf/parameter-context-flow.xml b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/resources/conf/parameter-context-flow.xml deleted file mode 100644 index 8bdf3509be..0000000000 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/resources/conf/parameter-context-flow.xml +++ /dev/null @@ -1,75 +0,0 @@ - - - - 10 - 1 - - - - context - Context - - referenced-context - referenced-context-2 - - foo - - true - enc{c5e2924eeec6d395ba09090cd50c5f2cee1ba23735d7eadfedae96d0b0fd257b} - - - - referenced-context - Referenced Context - - - - referenced-context-2 - Referenced Context 2 - - - - - 2ae3cdb4-0179-1000-6ddc-ed1dca231bac - NiFi Flow - - - UNBOUNDED - STREAM_WHEN_AVAILABLE - - c40fc154-ef89-48b8-82bf-ff6cc9e8f591 - DummyScheduledProcessor - - - <tagName> "This" is an ' example with many characters that need to be filtered and escaped in it.  † - org.apache.nifi.controller.DummyScheduledProcessor - 5 - 0 0 0 1/1 * ? - 30 sec - 1 sec - WARN - false - STOPPED - CRON_DRIVEN - ALL - 0 - - - context - - - - \ No newline at end of file diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/resources/conf/parameter-provider-with-cs-flow.json b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/resources/conf/parameter-provider-with-cs-flow.json new file mode 100644 index 0000000000..d4bc02bfbb --- /dev/null +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/resources/conf/parameter-provider-with-cs-flow.json @@ -0,0 +1,139 @@ +{ + "encodingVersion": { + "majorVersion": 2, + "minorVersion": 0 + }, + "maxTimerDrivenThreadCount": 10, + "maxEventDrivenThreadCount": 5, + "registries": [], + "parameterContexts": [], + "parameterProviders": [ + { + "identifier": "fb9b40ce-608f-4a2f-9822-3899f695f698", + "instanceIdentifier": "fb9b40ce-608f-4a2f-9822-3899f695f698", + "name": "ParameterProvider", + "comments": "", + "type": "org.apache.nifi.parameter.mock.PlaceholderParameterProvider", + "bundle": { + "group": "default", + "artifact": "system", + "version": "unversioned" + }, + "properties": { + "Controller Service": "3dd1e98a-aec8-32b6-a276-17fa9df0191c" + }, + "propertyDescriptors": {}, + "componentType": "PARAMETER_PROVIDER" + } + ], + "controllerServices": [ + { + "identifier": "77f7252b-f473-3520-81b9-5a29b5b18351", + "instanceIdentifier": "3dd1e98a-aec8-32b6-a276-17fa9df0191c", + "name": "ControllerService", + "comments": "", + "type": "org.apache.nifi.controller.service.mock.ServiceD", + "bundle": { + "group": "default", + "artifact": "system", + "version": "unversioned" + }, + "properties": { + "Foo1": "Bar1" + }, + "propertyDescriptors": {}, + "controllerServiceApis": [], + "scheduledState": "ENABLED", + "bulletinLevel": "WARN", + "componentType": "CONTROLLER_SERVICE" + } + ], + "reportingTasks": [], + "templates": [], + "rootGroup": { + "identifier": "3ddb274d-5d53-3a68-8db3-1e95b702852d", + "instanceIdentifier": "778f676e-6542-4c18-9d06-24b6fd3a1b29", + "name": "NiFi Flow", + "comments": "", + "position": { + "x": 0, + "y": 0 + }, + "processGroups": [], + "remoteProcessGroups": [], + "processors": [ + { + "identifier": "059c42dd-60fa-3582-9bc7-79635aa8cd64", + "instanceIdentifier": "709cca74-cd11-4ffa-9831-39d446a8ed54", + "name": "DummyProcessor", + "comments": "", + "position": { + "x": 1005, + "y": 314.90000000000003 + }, + "type": "org.apache.nifi.controller.service.mock.DummyProcessor", + "bundle": { + "group": "default", + "artifact": "system", + "version": "unversioned" + }, + "properties": { + "Controller Service": "ddf22ee5-376a-46dc-a38a-919351124457" + }, + "propertyDescriptors": {}, + "style": {}, + "schedulingPeriod": "0 sec", + "schedulingStrategy": "TIMER_DRIVEN", + "executionNode": "ALL", + "penaltyDuration": "30 sec", + "yieldDuration": "1 sec", + "bulletinLevel": "WARN", + "runDurationMillis": 0, + "concurrentlySchedulableTaskCount": 1, + "autoTerminatedRelationships": [], + "scheduledState": "ENABLED", + "retryCount": 10, + "retriedRelationships": [], + "backoffMechanism": "PENALIZE_FLOWFILE", + "maxBackoffPeriod": "10 mins", + "componentType": "PROCESSOR", + "groupIdentifier": "3ddb274d-5d53-3a68-8db3-1e95b702852d" + } + ], + "inputPorts": [], + "outputPorts": [], + "connections": [], + "labels": [], + "funnels": [], + "controllerServices": [ + { + "identifier": "3dd1e98a-aec8-32b6-a276-17fa9df0191c", + "instanceIdentifier": "ddf22ee5-376a-46dc-a38a-919351124457", + "name": "ControllerService", + "comments": "", + "type": "org.apache.nifi.controller.service.mock.ServiceD", + "bundle": { + "group": "default", + "artifact": "system", + "version": "unversioned" + }, + "properties": { + "Foo1": "Bar1" + }, + "propertyDescriptors": {}, + "controllerServiceApis": [], + "scheduledState": "DISABLED", + "bulletinLevel": "WARN", + "componentType": "CONTROLLER_SERVICE", + "groupIdentifier": "3ddb274d-5d53-3a68-8db3-1e95b702852d" + } + ], + "variables": {}, + "defaultFlowFileExpiration": "0 sec", + "defaultBackPressureObjectThreshold": 10000, + "defaultBackPressureDataSizeThreshold": "1 GB", + "componentType": "PROCESS_GROUP", + "flowFileOutboundPolicy": "STREAM_WHEN_AVAILABLE", + "flowFileConcurrency": "UNBOUNDED" + } +} \ No newline at end of file diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/resources/conf/parameter-provider-with-cs-flow.xml b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/resources/conf/parameter-provider-with-cs-flow.xml deleted file mode 100644 index d13e5d770b..0000000000 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/resources/conf/parameter-provider-with-cs-flow.xml +++ /dev/null @@ -1,71 +0,0 @@ - - - - 10 - 5 - - 778f676e-6542-4c18-9d06-24b6fd3a1b29 - NiFi Flow - - - - 709cca74-cd11-4ffa-9831-39d446a8ed54 - DummyProcessor - - - - org.apache.nifi.controller.service.mock.DummyProcessor - 1 - 0 sec - 30 sec - 1 sec - WARN - false - STOPPED - TIMER_DRIVEN - 0 - - Controller Service - ddf22ee5-376a-46dc-a38a-919351124457 - - - - - - ddf22ee5-376a-46dc-a38a-919351124457 - ControllerService - - org.apache.nifi.controller.service.mock.ServiceD - false - - Foo1 - Bar1 - - - - - - fb9b40ce-608f-4a2f-9822-3899f695f699 - ParameterProvider - - org.apache.nifi.parameter.mock.PlaceholderParameterProvider - - Controller Service - ddf22ee5-376a-46dc-a38a-919351124457 - - - - diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/resources/conf/processor-with-cs-flow-0.7.0.json b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/resources/conf/processor-with-cs-flow-0.7.0.json new file mode 100644 index 0000000000..170a4e3885 --- /dev/null +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/resources/conf/processor-with-cs-flow-0.7.0.json @@ -0,0 +1,100 @@ +{ + "encodingVersion": { + "majorVersion": 2, + "minorVersion": 0 + }, + "maxTimerDrivenThreadCount": 10, + "maxEventDrivenThreadCount": 5, + "registries": [], + "parameterContexts": [], + "parameterProviders": [], + "controllerServices": [], + "reportingTasks": [], + "templates": [], + "rootGroup": { + "identifier": "3ddb274d-5d53-3a68-8db3-1e95b702852d", + "instanceIdentifier": "778f676e-6542-4c18-9d06-24b6fd3a1b29", + "name": "NiFi Flow", + "comments": "", + "position": { + "x": 0, + "y": 0 + }, + "processGroups": [], + "remoteProcessGroups": [], + "processors": [ + { + "identifier": "70ffd0d6-c989-3525-9d2c-19a687b0b32e", + "instanceIdentifier": "809cca74-cd11-4ffa-9831-39d446a8ed55", + "name": "DummyProcessor", + "comments": "", + "position": { + "x": 1005, + "y": 314.90000000000003 + }, + "type": "org.apache.nifi.controller.service.mock.DummyProcessor", + "bundle": { + "group": "default", + "artifact": "system", + "version": "unversioned" + }, + "properties": { + "Controller Service": "edf22ee5-376a-46dc-a38a-919351124457" + }, + "propertyDescriptors": {}, + "style": {}, + "schedulingPeriod": "0 sec", + "schedulingStrategy": "TIMER_DRIVEN", + "executionNode": "ALL", + "penaltyDuration": "30 sec", + "yieldDuration": "1 sec", + "bulletinLevel": "WARN", + "runDurationMillis": 0, + "concurrentlySchedulableTaskCount": 1, + "autoTerminatedRelationships": [], + "scheduledState": "ENABLED", + "retryCount": 10, + "retriedRelationships": [], + "backoffMechanism": "PENALIZE_FLOWFILE", + "maxBackoffPeriod": "10 mins", + "componentType": "PROCESSOR", + "groupIdentifier": "3ddb274d-5d53-3a68-8db3-1e95b702852d" + } + ], + "inputPorts": [], + "outputPorts": [], + "connections": [], + "labels": [], + "funnels": [], + "controllerServices": [ + { + "identifier": "e5c9d095-39ab-311a-80a7-260efcdd0498", + "instanceIdentifier": "edf22ee5-376a-46dc-a38a-919351124457", + "name": "ControllerService", + "comments": "", + "type": "org.apache.nifi.controller.service.mock.ServiceD", + "bundle": { + "group": "default", + "artifact": "system", + "version": "unversioned" + }, + "properties": { + "Foo1": "Bar1" + }, + "propertyDescriptors": {}, + "controllerServiceApis": [], + "scheduledState": "DISABLED", + "bulletinLevel": "WARN", + "componentType": "CONTROLLER_SERVICE", + "groupIdentifier": "3ddb274d-5d53-3a68-8db3-1e95b702852d" + } + ], + "variables": {}, + "defaultFlowFileExpiration": "0 sec", + "defaultBackPressureObjectThreshold": 10000, + "defaultBackPressureDataSizeThreshold": "1 GB", + "componentType": "PROCESS_GROUP", + "flowFileConcurrency": "UNBOUNDED", + "flowFileOutboundPolicy": "STREAM_WHEN_AVAILABLE" + } +} \ No newline at end of file diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/resources/conf/processor-with-cs-flow-0.7.0.xml b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/resources/conf/processor-with-cs-flow-0.7.0.xml deleted file mode 100644 index 7440f61afc..0000000000 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/resources/conf/processor-with-cs-flow-0.7.0.xml +++ /dev/null @@ -1,60 +0,0 @@ - - - - 10 - 5 - - 778f676e-6542-4c18-9d06-24b6fd3a1b29 - NiFi Flow - - - - 809cca74-cd11-4ffa-9831-39d446a8ed55 - DummyProcessor - - - - org.apache.nifi.controller.service.mock.DummyProcessor - 1 - 0 sec - 30 sec - 1 sec - WARN - false - STOPPED - TIMER_DRIVEN - 0 - - Controller Service - edf22ee5-376a-46dc-a38a-919351124457 - - - - - - edf22ee5-376a-46dc-a38a-919351124457 - ControllerService - - WARN - org.apache.nifi.controller.service.mock.ServiceD - false - - Foo1 - Bar1 - - - - diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/resources/conf/remote-flow.xml b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/resources/conf/remote-flow.xml deleted file mode 100644 index a8faf097b0..0000000000 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/resources/conf/remote-flow.xml +++ /dev/null @@ -1,145 +0,0 @@ - - - - 100 - - 1234 - Root Group - - - - - 0000 - Data Generator - - - - org.apache.nifi.processors.standard.LocalSelectionFileProcessor - 5 - 0 - 500 - true - - root.directory.path - target/input - - - keep.original - true - - - polling.interval.secs - 1 - - - - destroyer - Data Destroyer - - - - org.apache.nifi.processors.standard.TerminationFileProcessor - 5 - 0 - 20000 - true - - - logger - Logger - - - - org.apache.nifi.processors.standard.AttributeLoggerProcessor - 5 - 0 - 500 - true - - log.level - info - - - - - input - Input Port - - - - - - remoteGroup - Remote Group - - - - http://localhost:8080/nifi - - - - 7777 - success - - 1 - 0 - - 0000 - 1234 - PROCESSOR - input - remoteGroup - INPUT_PORT - false - false - success - - - 12378461946123976329 - success - - 1 - 0 - - logger - 1234 - PROCESSOR - destroyer - 1234 - PROCESSOR - false - false - success - - - 2347892 - Input Port Connection - - 1 - 0 - - input - 1234 - INPUT_PORT - logger - 1234 - PROCESSOR - false - false - - - - diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/resources/conf/reporting-task-with-cs-flow-0.7.0.json b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/resources/conf/reporting-task-with-cs-flow-0.7.0.json new file mode 100644 index 0000000000..0896c8ce26 --- /dev/null +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/resources/conf/reporting-task-with-cs-flow-0.7.0.json @@ -0,0 +1,142 @@ +{ + "encodingVersion": { + "majorVersion": 2, + "minorVersion": 0 + }, + "maxTimerDrivenThreadCount": 10, + "maxEventDrivenThreadCount": 5, + "registries": [], + "parameterContexts": [], + "parameterProviders": [], + "controllerServices": [ + { + "identifier": "8209b929-e3f1-3a0c-b6e4-7b42a99160c9", + "instanceIdentifier": "32ea3f7c-c258-31c3-b720-b3e657d4945d", + "name": "ControllerService", + "comments": "", + "type": "org.apache.nifi.controller.service.mock.ServiceD", + "bundle": { + "group": "default", + "artifact": "system", + "version": "unversioned" + }, + "properties": { + "Foo1": "Bar1" + }, + "propertyDescriptors": {}, + "controllerServiceApis": [], + "scheduledState": "ENABLED", + "bulletinLevel": "WARN", + "componentType": "CONTROLLER_SERVICE" + } + ], + "reportingTasks": [ + { + "identifier": "fb9b40ce-608f-4a2f-9822-3899f695f699", + "instanceIdentifier": "fb9b40ce-608f-4a2f-9822-3899f695f699", + "name": "ReportingTask", + "comments": "", + "type": "org.apache.nifi.controller.service.mock.DummyReportingTask", + "bundle": { + "group": "default", + "artifact": "system", + "version": "unversioned" + }, + "properties": { + "Controller Service": "32ea3f7c-c258-31c3-b720-b3e657d4945d" + }, + "propertyDescriptors": {}, + "scheduledState": "ENABLED", + "schedulingPeriod": "5 mins", + "schedulingStrategy": "TIMER_DRIVEN", + "componentType": "REPORTING_TASK" + } + ], + "templates": [], + "rootGroup": { + "identifier": "3ddb274d-5d53-3a68-8db3-1e95b702852d", + "instanceIdentifier": "778f676e-6542-4c18-9d06-24b6fd3a1b29", + "name": "NiFi Flow", + "comments": "", + "position": { + "x": 0, + "y": 0 + }, + "processGroups": [], + "remoteProcessGroups": [], + "processors": [ + { + "identifier": "131d4261-d2ca-31fd-9104-7330706c83a8", + "instanceIdentifier": "809cca74-cd11-4ffa-9831-39d446a8ed54", + "name": "DummyProcessor", + "comments": "", + "position": { + "x": 1005, + "y": 314.90000000000003 + }, + "type": "org.apache.nifi.controller.service.mock.DummyProcessor", + "bundle": { + "group": "default", + "artifact": "system", + "version": "unversioned" + }, + "properties": { + "Controller Service": "edf22ee5-376a-46dc-a38a-919351124456" + }, + "propertyDescriptors": {}, + "style": {}, + "schedulingPeriod": "0 sec", + "schedulingStrategy": "TIMER_DRIVEN", + "executionNode": "ALL", + "penaltyDuration": "30 sec", + "yieldDuration": "1 sec", + "bulletinLevel": "WARN", + "runDurationMillis": 0, + "concurrentlySchedulableTaskCount": 1, + "autoTerminatedRelationships": [], + "scheduledState": "ENABLED", + "retryCount": 10, + "retriedRelationships": [], + "backoffMechanism": "PENALIZE_FLOWFILE", + "maxBackoffPeriod": "10 mins", + "componentType": "PROCESSOR", + "groupIdentifier": "3ddb274d-5d53-3a68-8db3-1e95b702852d" + } + ], + "inputPorts": [], + "outputPorts": [], + "connections": [], + "labels": [], + "funnels": [], + "controllerServices": [ + { + "identifier": "32ea3f7c-c258-31c3-b720-b3e657d4945d", + "instanceIdentifier": "edf22ee5-376a-46dc-a38a-919351124456", + "name": "ControllerService", + "comments": "", + "type": "org.apache.nifi.controller.service.mock.ServiceD", + "bundle": { + "group": "default", + "artifact": "system", + "version": "unversioned" + }, + "properties": { + "Foo1": "Bar1" + }, + "propertyDescriptors": {}, + "controllerServiceApis": [], + "scheduledState": "DISABLED", + "bulletinLevel": "WARN", + "componentType": "CONTROLLER_SERVICE", + "groupIdentifier": "3ddb274d-5d53-3a68-8db3-1e95b702852d" + } + ], + "variables": {}, + "defaultFlowFileExpiration": "0 sec", + "defaultBackPressureObjectThreshold": 10000, + "defaultBackPressureDataSizeThreshold": "1 GB", + "componentType": "PROCESS_GROUP", + "flowFileConcurrency": "UNBOUNDED", + "flowFileOutboundPolicy": "STREAM_WHEN_AVAILABLE" + } +} \ No newline at end of file diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/resources/conf/reporting-task-with-cs-flow-0.7.0.xml b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/resources/conf/reporting-task-with-cs-flow-0.7.0.xml deleted file mode 100644 index 272616ec5e..0000000000 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/resources/conf/reporting-task-with-cs-flow-0.7.0.xml +++ /dev/null @@ -1,75 +0,0 @@ - - - - 10 - 5 - - 778f676e-6542-4c18-9d06-24b6fd3a1b29 - NiFi Flow - - - - 809cca74-cd11-4ffa-9831-39d446a8ed54 - DummyProcessor - - - - org.apache.nifi.controller.service.mock.DummyProcessor - 1 - 0 sec - 30 sec - 1 sec - WARN - false - STOPPED - TIMER_DRIVEN - 0 - - Controller Service - edf22ee5-376a-46dc-a38a-919351124456 - - - - - - edf22ee5-376a-46dc-a38a-919351124456 - ControllerService - - WARN - org.apache.nifi.controller.service.mock.ServiceD - false - - Foo1 - Bar1 - - - - - - fb9b40ce-608f-4a2f-9822-3899f695f699 - ReportingTask - - org.apache.nifi.controller.service.mock.DummyReportingTask - 5 mins - STOPPED - TIMER_DRIVEN - - Controller Service - edf22ee5-376a-46dc-a38a-919351124456 - - - - diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/resources/conf/scale-positions-flow-0.7.0.json b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/resources/conf/scale-positions-flow-0.7.0.json new file mode 100644 index 0000000000..9678d6d524 --- /dev/null +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/resources/conf/scale-positions-flow-0.7.0.json @@ -0,0 +1 @@ +{"encodingVersion":{"majorVersion":2,"minorVersion":0},"maxTimerDrivenThreadCount":10,"maxEventDrivenThreadCount":5,"registries":[],"parameterContexts":[],"parameterProviders":[],"controllerServices":[],"reportingTasks":[],"templates":[],"rootGroup":{"identifier":"b3c0c7ff-385d-33e2-bb47-9ce8e5dcbe6e","instanceIdentifier":"32b64df5-b2f5-4c47-9880-3d0bd5c8ff15","name":"NiFi Flow","comments":"","position":{"x":0.0,"y":0.0},"processGroups":[{"identifier":"dd5b00c2-ae44-3a20-965b-291d897a795e","instanceIdentifier":"e7e03999-bbb8-491f-bba0-dbf772ec1966","name":"Process Group","comments":"","position":{"x":2568.6349461662603,"y":1513.1011709009408},"processGroups":[],"remoteProcessGroups":[],"processors":[{"identifier":"52ded018-53ac-3d47-9b40-bd8ba7c776ea","instanceIdentifier":"79d49fec-3b95-4e7b-a949-0d414594e7fa","name":"RouteOnAttribute","comments":"","position":{"x":1317.7335205078125,"y":548.041434326172},"type":"org.apache.nifi.controller.service.mock.DummyProcessor","bundle":{"group":"default","artifact":"system","version":"unversioned"},"properties":{"Routing Strategy":"Route to Property name","content is even":"${data:mod(2):equals(0)}"},"propertyDescriptors":{},"style":{},"schedulingPeriod":"0 sec","schedulingStrategy":"TIMER_DRIVEN","executionNode":"ALL","penaltyDuration":"30 sec","yieldDuration":"1 sec","bulletinLevel":"WARN","runDurationMillis":0,"concurrentlySchedulableTaskCount":1,"autoTerminatedRelationships":["unmatched"],"scheduledState":"ENABLED","retryCount":10,"retriedRelationships":[],"backoffMechanism":"PENALIZE_FLOWFILE","maxBackoffPeriod":"10 mins","componentType":"PROCESSOR","groupIdentifier":"dd5b00c2-ae44-3a20-965b-291d897a795e"},{"identifier":"3e72343e-6a8a-354c-8dd6-3d96dcb1ceb1","instanceIdentifier":"aea93188-0832-4876-9bb8-38901a773922","name":"ExtractText","comments":"","position":{"x":-1049.2335205078125,"y":-328.2814343261719},"type":"org.apache.nifi.controller.service.mock.DummyProcessor","bundle":{"group":"default","artifact":"system","version":"unversioned"},"properties":{"Enable Unicode Predefined Character Classes":"false","Permit Whitespace and Comments in Pattern":"false","Enable Unicode-aware Case Folding":"false","data":"((?s:^.*$))","Enable DOTALL Mode":"false","Enable Unix Lines Mode":"false","Maximum Buffer Size":"1 MB","Enable Canonical Equivalence":"false","Enable Case-insensitive Matching":"false","Enable Multiline Mode":"false","Maximum Capture Group Length":"1024","Enable Literal Parsing of the Pattern":"false","Character Set":"UTF-8","Include Capture Group 0":"false"},"propertyDescriptors":{},"style":{},"schedulingPeriod":"0 sec","schedulingStrategy":"TIMER_DRIVEN","executionNode":"ALL","penaltyDuration":"30 sec","yieldDuration":"1 sec","bulletinLevel":"WARN","runDurationMillis":0,"concurrentlySchedulableTaskCount":1,"autoTerminatedRelationships":["unmatched"],"scheduledState":"ENABLED","retryCount":10,"retriedRelationships":[],"backoffMechanism":"PENALIZE_FLOWFILE","maxBackoffPeriod":"10 mins","componentType":"PROCESSOR","groupIdentifier":"dd5b00c2-ae44-3a20-965b-291d897a795e"},{"identifier":"6b4c2060-db35-3a45-8d3f-457ea983d6d1","instanceIdentifier":"3b3957f9-91ff-4817-9a7f-fa80f78c2b72","name":"ReplaceText","comments":"","position":{"x":740.2335205078125,"y":109.86143432617189},"type":"org.apache.nifi.controller.service.mock.DummyProcessor","bundle":{"group":"default","artifact":"system","version":"unversioned"},"properties":{"Regular Expression":"(?s:^.*$)","Replacement Value":"${nextInt()}","Evaluation Mode":"Entire text","Character Set":"UTF-8","Maximum Buffer Size":"1 MB","Replacement Strategy":"Regex Replace"},"propertyDescriptors":{},"style":{},"schedulingPeriod":"0 sec","schedulingStrategy":"TIMER_DRIVEN","executionNode":"ALL","penaltyDuration":"30 sec","yieldDuration":"1 sec","bulletinLevel":"WARN","runDurationMillis":0,"concurrentlySchedulableTaskCount":1,"autoTerminatedRelationships":["failure"],"scheduledState":"ENABLED","retryCount":10,"retriedRelationships":[],"backoffMechanism":"PENALIZE_FLOWFILE","maxBackoffPeriod":"10 mins","componentType":"PROCESSOR","groupIdentifier":"dd5b00c2-ae44-3a20-965b-291d897a795e"}],"inputPorts":[{"identifier":"fe85a88f-b03f-3bbf-beb6-b12a12bd5a83","instanceIdentifier":"5f2c8c29-4cea-4da9-9a48-23abe40bd08d","name":"Input","comments":"","position":{"x":230.59141307923846,"y":-54.85675275210223},"type":"INPUT_PORT","concurrentlySchedulableTaskCount":1,"scheduledState":"ENABLED","allowRemoteAccess":false,"componentType":"INPUT_PORT","groupIdentifier":"dd5b00c2-ae44-3a20-965b-291d897a795e"}],"outputPorts":[{"identifier":"54fb1fc6-1037-3254-ab0c-c1a430eae27f","instanceIdentifier":"b6e55392-154f-46d6-ba5a-4d1311659094","name":"Output","comments":"","position":{"x":1997.999986326458,"y":920.5799975113985},"type":"OUTPUT_PORT","concurrentlySchedulableTaskCount":1,"scheduledState":"ENABLED","allowRemoteAccess":false,"componentType":"OUTPUT_PORT","groupIdentifier":"dd5b00c2-ae44-3a20-965b-291d897a795e"}],"connections":[{"identifier":"6076dc2a-ae96-3295-b6ae-ec116ec82cd7","instanceIdentifier":"7b7f871b-8f4b-4ce7-a9d4-2f810bf2b55f","name":"","source":{"id":"3e72343e-6a8a-354c-8dd6-3d96dcb1ceb1","type":"PROCESSOR","groupId":"dd5b00c2-ae44-3a20-965b-291d897a795e","name":"ExtractText","comments":"","instanceIdentifier":"aea93188-0832-4876-9bb8-38901a773922"},"destination":{"id":"52ded018-53ac-3d47-9b40-bd8ba7c776ea","type":"PROCESSOR","groupId":"dd5b00c2-ae44-3a20-965b-291d897a795e","name":"RouteOnAttribute","comments":"","instanceIdentifier":"79d49fec-3b95-4e7b-a949-0d414594e7fa"},"labelIndex":1,"zIndex":0,"selectedRelationships":["matched"],"backPressureObjectThreshold":0,"backPressureDataSizeThreshold":"0 MB","flowFileExpiration":"0 sec","prioritizers":[],"bends":[],"loadBalanceStrategy":"DO_NOT_LOAD_BALANCE","loadBalanceCompression":"DO_NOT_COMPRESS","componentType":"CONNECTION","groupIdentifier":"dd5b00c2-ae44-3a20-965b-291d897a795e"},{"identifier":"2e5b937c-779a-3d48-8be3-0bad8168f897","instanceIdentifier":"8bf960ca-ba33-4b5b-9830-1a808dd2db0e","name":"","source":{"id":"52ded018-53ac-3d47-9b40-bd8ba7c776ea","type":"PROCESSOR","groupId":"dd5b00c2-ae44-3a20-965b-291d897a795e","name":"RouteOnAttribute","comments":"","instanceIdentifier":"79d49fec-3b95-4e7b-a949-0d414594e7fa"},"destination":{"id":"54fb1fc6-1037-3254-ab0c-c1a430eae27f","type":"OUTPUT_PORT","groupId":"dd5b00c2-ae44-3a20-965b-291d897a795e","name":"Output","comments":"","instanceIdentifier":"b6e55392-154f-46d6-ba5a-4d1311659094"},"labelIndex":1,"zIndex":0,"selectedRelationships":["content is even"],"backPressureObjectThreshold":0,"backPressureDataSizeThreshold":"0 MB","flowFileExpiration":"0 sec","prioritizers":[],"bends":[],"loadBalanceStrategy":"DO_NOT_LOAD_BALANCE","loadBalanceCompression":"DO_NOT_COMPRESS","componentType":"CONNECTION","groupIdentifier":"dd5b00c2-ae44-3a20-965b-291d897a795e"},{"identifier":"c64b2bb7-cd23-3f2d-b0ce-8a20348392cb","instanceIdentifier":"a82d78cb-de24-40e3-8fb9-d2117919cb31","name":"","source":{"id":"6b4c2060-db35-3a45-8d3f-457ea983d6d1","type":"PROCESSOR","groupId":"dd5b00c2-ae44-3a20-965b-291d897a795e","name":"ReplaceText","comments":"","instanceIdentifier":"3b3957f9-91ff-4817-9a7f-fa80f78c2b72"},"destination":{"id":"3e72343e-6a8a-354c-8dd6-3d96dcb1ceb1","type":"PROCESSOR","groupId":"dd5b00c2-ae44-3a20-965b-291d897a795e","name":"ExtractText","comments":"","instanceIdentifier":"aea93188-0832-4876-9bb8-38901a773922"},"labelIndex":1,"zIndex":0,"selectedRelationships":["success"],"backPressureObjectThreshold":0,"backPressureDataSizeThreshold":"0 MB","flowFileExpiration":"0 sec","prioritizers":[],"bends":[],"loadBalanceStrategy":"DO_NOT_LOAD_BALANCE","loadBalanceCompression":"DO_NOT_COMPRESS","componentType":"CONNECTION","groupIdentifier":"dd5b00c2-ae44-3a20-965b-291d897a795e"},{"identifier":"a2dc7da1-4ee4-3d93-a93d-c35e8bc3a136","instanceIdentifier":"197ffc73-76d3-4e83-84ca-a431fef50a8f","name":"","source":{"id":"fe85a88f-b03f-3bbf-beb6-b12a12bd5a83","type":"INPUT_PORT","groupId":"dd5b00c2-ae44-3a20-965b-291d897a795e","name":"Input","comments":"","instanceIdentifier":"5f2c8c29-4cea-4da9-9a48-23abe40bd08d"},"destination":{"id":"6b4c2060-db35-3a45-8d3f-457ea983d6d1","type":"PROCESSOR","groupId":"dd5b00c2-ae44-3a20-965b-291d897a795e","name":"ReplaceText","comments":"","instanceIdentifier":"3b3957f9-91ff-4817-9a7f-fa80f78c2b72"},"labelIndex":1,"zIndex":0,"selectedRelationships":[""],"backPressureObjectThreshold":0,"backPressureDataSizeThreshold":"0 MB","flowFileExpiration":"0 sec","prioritizers":[],"bends":[],"loadBalanceStrategy":"DO_NOT_LOAD_BALANCE","loadBalanceCompression":"DO_NOT_COMPRESS","componentType":"CONNECTION","groupIdentifier":"dd5b00c2-ae44-3a20-965b-291d897a795e"}],"labels":[],"funnels":[],"controllerServices":[],"variables":{},"defaultFlowFileExpiration":"0 sec","defaultBackPressureObjectThreshold":10000,"defaultBackPressureDataSizeThreshold":"1 GB","componentType":"PROCESS_GROUP","flowFileOutboundPolicy":"STREAM_WHEN_AVAILABLE","flowFileConcurrency":"UNBOUNDED","groupIdentifier":"b3c0c7ff-385d-33e2-bb47-9ce8e5dcbe6e"}],"remoteProcessGroups":[{"identifier":"5561cc43-b1bf-367b-a220-c26ac4d9a9a7","instanceIdentifier":"b1ebe9d0-56db-4f2e-8448-b78ec9daab66","name":"https://localhost:8080/nifi","comments":"","position":{"x":1648.6865156199165,"y":925.2224565989999},"targetUri":"https://localhost:8080/nifi","targetUris":"https://localhost:8080/nifi","communicationsTimeout":"30 sec","yieldDuration":"10 sec","transportProtocol":"RAW","inputPorts":[],"outputPorts":[],"componentType":"REMOTE_PROCESS_GROUP","groupIdentifier":"b3c0c7ff-385d-33e2-bb47-9ce8e5dcbe6e"}],"processors":[{"identifier":"4db7d310-54d1-3639-87a5-f0ebdf08e394","instanceIdentifier":"ab9e02ee-511f-48de-ac07-da46481c1a70","name":"ReplaceText","comments":"","position":{"x":1577.2335205078125,"y":371.1614343261719},"type":"org.apache.nifi.controller.service.mock.DummyProcessor","bundle":{"group":"default","artifact":"system","version":"unversioned"},"properties":{"Regular Expression":"(?s:^.*$)","Replacement Value":"${nextInt()}","Evaluation Mode":"Entire text","Character Set":"UTF-8","Maximum Buffer Size":"1 MB","Replacement Strategy":"Regex Replace"},"propertyDescriptors":{},"style":{},"schedulingPeriod":"0 sec","schedulingStrategy":"TIMER_DRIVEN","executionNode":"ALL","penaltyDuration":"30 sec","yieldDuration":"1 sec","bulletinLevel":"WARN","runDurationMillis":0,"concurrentlySchedulableTaskCount":1,"autoTerminatedRelationships":["failure"],"scheduledState":"ENABLED","retryCount":10,"retriedRelationships":[],"backoffMechanism":"PENALIZE_FLOWFILE","maxBackoffPeriod":"10 mins","componentType":"PROCESSOR","groupIdentifier":"b3c0c7ff-385d-33e2-bb47-9ce8e5dcbe6e"},{"identifier":"7b92d941-b477-3d3e-90fc-4214753750af","instanceIdentifier":"f3063c47-c8ae-4b25-bd71-98dffdf33e0a","name":"GenerateFlowFile","comments":"","position":{"x":1023.0,"y":363.14000000000004},"type":"org.apache.nifi.controller.service.mock.DummyProcessor","bundle":{"group":"default","artifact":"system","version":"unversioned"},"properties":{"File Size":"0b","Batch Size":"1","Unique FlowFiles":"false","Data Format":"Binary"},"propertyDescriptors":{},"style":{},"schedulingPeriod":"1 sec","schedulingStrategy":"TIMER_DRIVEN","executionNode":"ALL","penaltyDuration":"30 sec","yieldDuration":"1 sec","bulletinLevel":"WARN","runDurationMillis":0,"concurrentlySchedulableTaskCount":1,"autoTerminatedRelationships":[],"scheduledState":"ENABLED","retryCount":10,"retriedRelationships":[],"backoffMechanism":"PENALIZE_FLOWFILE","maxBackoffPeriod":"10 mins","componentType":"PROCESSOR","groupIdentifier":"b3c0c7ff-385d-33e2-bb47-9ce8e5dcbe6e"},{"identifier":"b9519c10-259e-3cc4-8512-ee20fd64ac38","instanceIdentifier":"74af00f4-5b4a-4236-bae0-ebc6bef5ae1f","name":"ReplaceText","comments":"","position":{"x":1683.5445556640625,"y":1200.2557641601563},"type":"org.apache.nifi.controller.service.mock.DummyProcessor","bundle":{"group":"default","artifact":"system","version":"unversioned"},"properties":{"Regular Expression":"(?s:^.*$)","Replacement Value":"${nextInt()}","Evaluation Mode":"Entire text","Character Set":"UTF-8","Maximum Buffer Size":"1 MB","Replacement Strategy":"Regex Replace"},"propertyDescriptors":{},"style":{},"schedulingPeriod":"0 sec","schedulingStrategy":"TIMER_DRIVEN","executionNode":"ALL","penaltyDuration":"30 sec","yieldDuration":"1 sec","bulletinLevel":"WARN","runDurationMillis":0,"concurrentlySchedulableTaskCount":1,"autoTerminatedRelationships":["failure"],"scheduledState":"ENABLED","retryCount":10,"retriedRelationships":[],"backoffMechanism":"PENALIZE_FLOWFILE","maxBackoffPeriod":"10 mins","componentType":"PROCESSOR","groupIdentifier":"b3c0c7ff-385d-33e2-bb47-9ce8e5dcbe6e"},{"identifier":"260a9b16-2dfe-3f85-8a48-ee2aba60563b","instanceIdentifier":"6461ece1-db83-481f-88ba-534cd3a67b1b","name":"RouteOnAttribute","comments":"","position":{"x":2611.4338989257812,"y":2164.2530236816406},"type":"org.apache.nifi.controller.service.mock.DummyProcessor","bundle":{"group":"default","artifact":"system","version":"unversioned"},"properties":{"Routing Strategy":"Route to Property name","content is even":"${data:mod(2):equals(0)}"},"propertyDescriptors":{},"style":{},"schedulingPeriod":"0 sec","schedulingStrategy":"TIMER_DRIVEN","executionNode":"ALL","penaltyDuration":"30 sec","yieldDuration":"1 sec","bulletinLevel":"WARN","runDurationMillis":0,"concurrentlySchedulableTaskCount":1,"autoTerminatedRelationships":["content is even","unmatched"],"scheduledState":"ENABLED","retryCount":10,"retriedRelationships":[],"backoffMechanism":"PENALIZE_FLOWFILE","maxBackoffPeriod":"10 mins","componentType":"PROCESSOR","groupIdentifier":"b3c0c7ff-385d-33e2-bb47-9ce8e5dcbe6e"},{"identifier":"c26f79a3-7ecf-38e2-8037-e15336ccbd0b","instanceIdentifier":"a4b100b5-f3da-41ed-8ed9-a198d3f4aba3","name":"ExtractText","comments":"","position":{"x":2610.8876953125,"y":1984.6930236816409},"type":"org.apache.nifi.controller.service.mock.DummyProcessor","bundle":{"group":"default","artifact":"system","version":"unversioned"},"properties":{"Enable Unicode Predefined Character Classes":"false","Permit Whitespace and Comments in Pattern":"false","Enable Unicode-aware Case Folding":"false","data":"((?s:^.*$))","Enable DOTALL Mode":"false","Enable Unix Lines Mode":"false","Maximum Buffer Size":"1 MB","Enable Canonical Equivalence":"false","Enable Case-insensitive Matching":"false","Enable Multiline Mode":"false","Maximum Capture Group Length":"1024","Enable Literal Parsing of the Pattern":"false","Character Set":"UTF-8","Include Capture Group 0":"false"},"propertyDescriptors":{},"style":{},"schedulingPeriod":"0 sec","schedulingStrategy":"TIMER_DRIVEN","executionNode":"ALL","penaltyDuration":"30 sec","yieldDuration":"1 sec","bulletinLevel":"WARN","runDurationMillis":0,"concurrentlySchedulableTaskCount":1,"autoTerminatedRelationships":["unmatched"],"scheduledState":"ENABLED","retryCount":10,"retriedRelationships":[],"backoffMechanism":"PENALIZE_FLOWFILE","maxBackoffPeriod":"10 mins","componentType":"PROCESSOR","groupIdentifier":"b3c0c7ff-385d-33e2-bb47-9ce8e5dcbe6e"},{"identifier":"ba48f6ad-95ab-3eac-91de-6d4950bb1764","instanceIdentifier":"b1f6cb42-dbce-49a3-be9f-cfc6c889bb8b","name":"RouteOnAttribute","comments":"","position":{"x":1587.7335205078125,"y":731.6214343261719},"type":"org.apache.nifi.controller.service.mock.DummyProcessor","bundle":{"group":"default","artifact":"system","version":"unversioned"},"properties":{"Routing Strategy":"Route to Property name","content is even":"${data:mod(2):equals(0)}"},"propertyDescriptors":{},"style":{},"schedulingPeriod":"0 sec","schedulingStrategy":"TIMER_DRIVEN","executionNode":"ALL","penaltyDuration":"30 sec","yieldDuration":"1 sec","bulletinLevel":"WARN","runDurationMillis":0,"concurrentlySchedulableTaskCount":1,"autoTerminatedRelationships":["content is even","unmatched"],"scheduledState":"ENABLED","retryCount":10,"retriedRelationships":[],"backoffMechanism":"PENALIZE_FLOWFILE","maxBackoffPeriod":"10 mins","componentType":"PROCESSOR","groupIdentifier":"b3c0c7ff-385d-33e2-bb47-9ce8e5dcbe6e"},{"identifier":"8823fbfd-07ce-303a-b1be-1f59501eab08","instanceIdentifier":"a05b4e5f-059f-4105-af89-a551098d2779","name":"ExtractText","comments":"","position":{"x":1029.0,"y":726.2800000000001},"type":"org.apache.nifi.controller.service.mock.DummyProcessor","bundle":{"group":"default","artifact":"system","version":"unversioned"},"properties":{"Enable Unicode Predefined Character Classes":"false","Permit Whitespace and Comments in Pattern":"false","Enable Unicode-aware Case Folding":"false","data":"((?s:^.*$))","Enable DOTALL Mode":"false","Enable Unix Lines Mode":"false","Maximum Buffer Size":"1 MB","Enable Canonical Equivalence":"false","Enable Case-insensitive Matching":"false","Enable Multiline Mode":"false","Maximum Capture Group Length":"1024","Enable Literal Parsing of the Pattern":"false","Character Set":"UTF-8","Include Capture Group 0":"false"},"propertyDescriptors":{},"style":{},"schedulingPeriod":"0 sec","schedulingStrategy":"TIMER_DRIVEN","executionNode":"ALL","penaltyDuration":"30 sec","yieldDuration":"1 sec","bulletinLevel":"WARN","runDurationMillis":0,"concurrentlySchedulableTaskCount":1,"autoTerminatedRelationships":["unmatched"],"scheduledState":"ENABLED","retryCount":10,"retriedRelationships":[],"backoffMechanism":"PENALIZE_FLOWFILE","maxBackoffPeriod":"10 mins","componentType":"PROCESSOR","groupIdentifier":"b3c0c7ff-385d-33e2-bb47-9ce8e5dcbe6e"},{"identifier":"8d24ad16-0816-36f0-86a0-fff0e33749fc","instanceIdentifier":"4b771af6-af32-443f-b807-6b9f7c476e39","name":"ReplaceText","comments":"","position":{"x":446.3543930053711,"y":369.86968872070315},"type":"org.apache.nifi.controller.service.mock.DummyProcessor","bundle":{"group":"default","artifact":"system","version":"unversioned"},"properties":{"Regular Expression":"(?s:^.*$)","Replacement Value":"${nextInt()}","Evaluation Mode":"Entire text","Character Set":"UTF-8","Maximum Buffer Size":"1 MB","Replacement Strategy":"Regex Replace"},"propertyDescriptors":{},"style":{},"schedulingPeriod":"0 sec","schedulingStrategy":"TIMER_DRIVEN","executionNode":"ALL","penaltyDuration":"30 sec","yieldDuration":"1 sec","bulletinLevel":"WARN","runDurationMillis":0,"concurrentlySchedulableTaskCount":1,"autoTerminatedRelationships":["failure"],"scheduledState":"ENABLED","retryCount":10,"retriedRelationships":[],"backoffMechanism":"PENALIZE_FLOWFILE","maxBackoffPeriod":"10 mins","componentType":"PROCESSOR","groupIdentifier":"b3c0c7ff-385d-33e2-bb47-9ce8e5dcbe6e"},{"identifier":"1b852682-1499-38ff-a80d-33c7cfb59801","instanceIdentifier":"0e49512d-67f4-4b74-9880-ea2e2fb3436b","name":"ExtractText","comments":"","position":{"x":336.9707565307617,"y":1378.525940551758},"type":"org.apache.nifi.controller.service.mock.DummyProcessor","bundle":{"group":"default","artifact":"system","version":"unversioned"},"properties":{"Enable Unicode Predefined Character Classes":"false","Permit Whitespace and Comments in Pattern":"false","Enable Unicode-aware Case Folding":"false","data":"((?s:^.*$))","Enable DOTALL Mode":"false","Enable Unix Lines Mode":"false","Maximum Buffer Size":"1 MB","Enable Canonical Equivalence":"false","Enable Case-insensitive Matching":"false","Enable Multiline Mode":"false","Maximum Capture Group Length":"1024","Enable Literal Parsing of the Pattern":"false","Character Set":"UTF-8","Include Capture Group 0":"false"},"propertyDescriptors":{},"style":{},"schedulingPeriod":"0 sec","schedulingStrategy":"TIMER_DRIVEN","executionNode":"ALL","penaltyDuration":"30 sec","yieldDuration":"1 sec","bulletinLevel":"WARN","runDurationMillis":0,"concurrentlySchedulableTaskCount":1,"autoTerminatedRelationships":["unmatched"],"scheduledState":"ENABLED","retryCount":10,"retriedRelationships":[],"backoffMechanism":"PENALIZE_FLOWFILE","maxBackoffPeriod":"10 mins","componentType":"PROCESSOR","groupIdentifier":"b3c0c7ff-385d-33e2-bb47-9ce8e5dcbe6e"},{"identifier":"062d7339-50ae-3c65-a629-718e0cb29b2d","instanceIdentifier":"dd6046fa-f346-461d-93a3-c65bcc38e3d8","name":"RouteOnAttribute","comments":"","position":{"x":456.8543930053711,"y":730.3296887207032},"type":"org.apache.nifi.controller.service.mock.DummyProcessor","bundle":{"group":"default","artifact":"system","version":"unversioned"},"properties":{"Routing Strategy":"Route to Property name","content is even":"${data:mod(2):equals(0)}"},"propertyDescriptors":{},"style":{},"schedulingPeriod":"0 sec","schedulingStrategy":"TIMER_DRIVEN","executionNode":"ALL","penaltyDuration":"30 sec","yieldDuration":"1 sec","bulletinLevel":"WARN","runDurationMillis":0,"concurrentlySchedulableTaskCount":1,"autoTerminatedRelationships":["content is even","unmatched"],"scheduledState":"ENABLED","retryCount":10,"retriedRelationships":[],"backoffMechanism":"PENALIZE_FLOWFILE","maxBackoffPeriod":"10 mins","componentType":"PROCESSOR","groupIdentifier":"b3c0c7ff-385d-33e2-bb47-9ce8e5dcbe6e"},{"identifier":"25db9f1b-cfc4-36dc-ab64-c5feeef81f41","instanceIdentifier":"c697e51f-aa4f-4cbf-a4e7-25156970a360","name":"RouteOnAttribute","comments":"","position":{"x":1694.0445556640625,"y":1560.7157641601564},"type":"org.apache.nifi.controller.service.mock.DummyProcessor","bundle":{"group":"default","artifact":"system","version":"unversioned"},"properties":{"Routing Strategy":"Route to Property name","content is even":"${data:mod(2):equals(0)}"},"propertyDescriptors":{},"style":{},"schedulingPeriod":"0 sec","schedulingStrategy":"TIMER_DRIVEN","executionNode":"ALL","penaltyDuration":"30 sec","yieldDuration":"1 sec","bulletinLevel":"WARN","runDurationMillis":0,"concurrentlySchedulableTaskCount":1,"autoTerminatedRelationships":[],"scheduledState":"ENABLED","retryCount":10,"retriedRelationships":[],"backoffMechanism":"PENALIZE_FLOWFILE","maxBackoffPeriod":"10 mins","componentType":"PROCESSOR","groupIdentifier":"b3c0c7ff-385d-33e2-bb47-9ce8e5dcbe6e"},{"identifier":"7e30b537-b10c-39a9-9dab-f0825174a33a","instanceIdentifier":"65140227-fe60-42ad-beb9-69c36bc75acb","name":"ReplaceText","comments":"","position":{"x":1024.5,"y":545.38},"type":"org.apache.nifi.controller.service.mock.DummyProcessor","bundle":{"group":"default","artifact":"system","version":"unversioned"},"properties":{"Regular Expression":"(?s:^.*$)","Replacement Value":"${nextInt()}","Evaluation Mode":"Entire text","Character Set":"UTF-8","Maximum Buffer Size":"1 MB","Replacement Strategy":"Regex Replace"},"propertyDescriptors":{},"style":{},"schedulingPeriod":"0 sec","schedulingStrategy":"TIMER_DRIVEN","executionNode":"ALL","penaltyDuration":"30 sec","yieldDuration":"1 sec","bulletinLevel":"WARN","runDurationMillis":0,"concurrentlySchedulableTaskCount":1,"autoTerminatedRelationships":["failure"],"scheduledState":"ENABLED","retryCount":10,"retriedRelationships":[],"backoffMechanism":"PENALIZE_FLOWFILE","maxBackoffPeriod":"10 mins","componentType":"PROCESSOR","groupIdentifier":"b3c0c7ff-385d-33e2-bb47-9ce8e5dcbe6e"},{"identifier":"96e61aa4-70af-375d-beb3-58e3b8256203","instanceIdentifier":"2a1e8019-5264-48e7-bc61-8d71c4ecaa82","name":"ExtractText","comments":"","position":{"x":1688.0445556640625,"y":1381.1557641601564},"type":"org.apache.nifi.controller.service.mock.DummyProcessor","bundle":{"group":"default","artifact":"system","version":"unversioned"},"properties":{"Enable Unicode Predefined Character Classes":"false","Permit Whitespace and Comments in Pattern":"false","Enable Unicode-aware Case Folding":"false","data":"((?s:^.*$))","Enable DOTALL Mode":"false","Enable Unix Lines Mode":"false","Maximum Buffer Size":"1 MB","Enable Canonical Equivalence":"false","Enable Case-insensitive Matching":"false","Enable Multiline Mode":"false","Maximum Capture Group Length":"1024","Enable Literal Parsing of the Pattern":"false","Character Set":"UTF-8","Include Capture Group 0":"false"},"propertyDescriptors":{},"style":{},"schedulingPeriod":"0 sec","schedulingStrategy":"TIMER_DRIVEN","executionNode":"ALL","penaltyDuration":"30 sec","yieldDuration":"1 sec","bulletinLevel":"WARN","runDurationMillis":0,"concurrentlySchedulableTaskCount":1,"autoTerminatedRelationships":["unmatched"],"scheduledState":"ENABLED","retryCount":10,"retriedRelationships":[],"backoffMechanism":"PENALIZE_FLOWFILE","maxBackoffPeriod":"10 mins","componentType":"PROCESSOR","groupIdentifier":"b3c0c7ff-385d-33e2-bb47-9ce8e5dcbe6e"},{"identifier":"3bbe0eed-14a2-3ee2-94a3-8817f65f200c","instanceIdentifier":"3dae3e31-ab40-49ca-bb86-21e91f24ee97","name":"RouteOnAttribute","comments":"","position":{"x":342.9707565307617,"y":1558.0859405517579},"type":"org.apache.nifi.controller.service.mock.DummyProcessor","bundle":{"group":"default","artifact":"system","version":"unversioned"},"properties":{"Routing Strategy":"Route to Property name","content is even":"${data:mod(2):equals(0)}"},"propertyDescriptors":{},"style":{},"schedulingPeriod":"0 sec","schedulingStrategy":"TIMER_DRIVEN","executionNode":"ALL","penaltyDuration":"30 sec","yieldDuration":"1 sec","bulletinLevel":"WARN","runDurationMillis":0,"concurrentlySchedulableTaskCount":1,"autoTerminatedRelationships":["content is even","unmatched"],"scheduledState":"ENABLED","retryCount":10,"retriedRelationships":[],"backoffMechanism":"PENALIZE_FLOWFILE","maxBackoffPeriod":"10 mins","componentType":"PROCESSOR","groupIdentifier":"b3c0c7ff-385d-33e2-bb47-9ce8e5dcbe6e"},{"identifier":"ca07aec3-aa90-31a9-bdea-6bb999a2a608","instanceIdentifier":"b4369bf2-e5d5-4df8-b397-4d63c1a11d29","name":"RouteOnAttribute","comments":"","position":{"x":1035.0,"y":905.84},"type":"org.apache.nifi.controller.service.mock.DummyProcessor","bundle":{"group":"default","artifact":"system","version":"unversioned"},"properties":{"Routing Strategy":"Route to Property name","data is even":"${data:mod(2):equals(0)}"},"propertyDescriptors":{},"style":{},"schedulingPeriod":"0 sec","schedulingStrategy":"TIMER_DRIVEN","executionNode":"ALL","penaltyDuration":"30 sec","yieldDuration":"1 sec","bulletinLevel":"WARN","runDurationMillis":0,"concurrentlySchedulableTaskCount":1,"autoTerminatedRelationships":[],"scheduledState":"ENABLED","retryCount":10,"retriedRelationships":[],"backoffMechanism":"PENALIZE_FLOWFILE","maxBackoffPeriod":"10 mins","componentType":"PROCESSOR","groupIdentifier":"b3c0c7ff-385d-33e2-bb47-9ce8e5dcbe6e"},{"identifier":"c81682f7-ac5e-39a4-a89c-c8f5f8416bfa","instanceIdentifier":"d908041d-3f7a-4d07-bf76-cdee0c2e2c28","name":"ExtractText","comments":"","position":{"x":1581.7335205078125,"y":552.0614343261719},"type":"org.apache.nifi.controller.service.mock.DummyProcessor","bundle":{"group":"default","artifact":"system","version":"unversioned"},"properties":{"Enable Unicode Predefined Character Classes":"false","Permit Whitespace and Comments in Pattern":"false","Enable Unicode-aware Case Folding":"false","data":"((?s:^.*$))","Enable DOTALL Mode":"false","Enable Unix Lines Mode":"false","Maximum Buffer Size":"1 MB","Enable Canonical Equivalence":"false","Enable Case-insensitive Matching":"false","Enable Multiline Mode":"false","Maximum Capture Group Length":"1024","Enable Literal Parsing of the Pattern":"false","Character Set":"UTF-8","Include Capture Group 0":"false"},"propertyDescriptors":{},"style":{},"schedulingPeriod":"0 sec","schedulingStrategy":"TIMER_DRIVEN","executionNode":"ALL","penaltyDuration":"30 sec","yieldDuration":"1 sec","bulletinLevel":"WARN","runDurationMillis":0,"concurrentlySchedulableTaskCount":1,"autoTerminatedRelationships":["unmatched"],"scheduledState":"ENABLED","retryCount":10,"retriedRelationships":[],"backoffMechanism":"PENALIZE_FLOWFILE","maxBackoffPeriod":"10 mins","componentType":"PROCESSOR","groupIdentifier":"b3c0c7ff-385d-33e2-bb47-9ce8e5dcbe6e"},{"identifier":"db9cf755-e0f0-3cf3-90a4-025f19297ae9","instanceIdentifier":"de1f9362-f8a5-44a0-bef7-9a6f608e6e66","name":"ExtractText","comments":"","position":{"x":450.8543930053711,"y":550.7696887207031},"type":"org.apache.nifi.controller.service.mock.DummyProcessor","bundle":{"group":"default","artifact":"system","version":"unversioned"},"properties":{"Enable Unicode Predefined Character Classes":"false","Permit Whitespace and Comments in Pattern":"false","Enable Unicode-aware Case Folding":"false","data":"((?s:^.*$))","Enable DOTALL Mode":"false","Enable Unix Lines Mode":"false","Maximum Buffer Size":"1 MB","Enable Canonical Equivalence":"false","Enable Case-insensitive Matching":"false","Enable Multiline Mode":"false","Maximum Capture Group Length":"1024","Enable Literal Parsing of the Pattern":"false","Character Set":"UTF-8","Include Capture Group 0":"false"},"propertyDescriptors":{},"style":{},"schedulingPeriod":"0 sec","schedulingStrategy":"TIMER_DRIVEN","executionNode":"ALL","penaltyDuration":"30 sec","yieldDuration":"1 sec","bulletinLevel":"WARN","runDurationMillis":0,"concurrentlySchedulableTaskCount":1,"autoTerminatedRelationships":["unmatched"],"scheduledState":"ENABLED","retryCount":10,"retriedRelationships":[],"backoffMechanism":"PENALIZE_FLOWFILE","maxBackoffPeriod":"10 mins","componentType":"PROCESSOR","groupIdentifier":"b3c0c7ff-385d-33e2-bb47-9ce8e5dcbe6e"},{"identifier":"7d516c36-b449-3409-8745-400d6c43132c","instanceIdentifier":"3d47cfae-2ada-4992-8597-e7e814490749","name":"ReplaceText","comments":"","position":{"x":332.4707565307617,"y":1197.6259405517578},"type":"org.apache.nifi.controller.service.mock.DummyProcessor","bundle":{"group":"default","artifact":"system","version":"unversioned"},"properties":{"Regular Expression":"(?s:^.*$)","Replacement Value":"${nextInt()}","Evaluation Mode":"Entire text","Character Set":"UTF-8","Maximum Buffer Size":"1 MB","Replacement Strategy":"Regex Replace"},"propertyDescriptors":{},"style":{},"schedulingPeriod":"0 sec","schedulingStrategy":"TIMER_DRIVEN","executionNode":"ALL","penaltyDuration":"30 sec","yieldDuration":"1 sec","bulletinLevel":"WARN","runDurationMillis":0,"concurrentlySchedulableTaskCount":1,"autoTerminatedRelationships":["failure"],"scheduledState":"ENABLED","retryCount":10,"retriedRelationships":[],"backoffMechanism":"PENALIZE_FLOWFILE","maxBackoffPeriod":"10 mins","componentType":"PROCESSOR","groupIdentifier":"b3c0c7ff-385d-33e2-bb47-9ce8e5dcbe6e"},{"identifier":"31ffa34c-fe27-3a13-85aa-381fcbc79f11","instanceIdentifier":"92fd75c1-7d4c-4adc-bd2e-e3c33075dfb6","name":"ReplaceText","comments":"","position":{"x":2610.0236206054688,"y":1803.7930236816408},"type":"org.apache.nifi.controller.service.mock.DummyProcessor","bundle":{"group":"default","artifact":"system","version":"unversioned"},"properties":{"Regular Expression":"(?s:^.*$)","Replacement Value":"${nextInt()}","Evaluation Mode":"Entire text","Character Set":"UTF-8","Maximum Buffer Size":"1 MB","Replacement Strategy":"Regex Replace"},"propertyDescriptors":{},"style":{},"schedulingPeriod":"0 sec","schedulingStrategy":"TIMER_DRIVEN","executionNode":"ALL","penaltyDuration":"30 sec","yieldDuration":"1 sec","bulletinLevel":"WARN","runDurationMillis":0,"concurrentlySchedulableTaskCount":1,"autoTerminatedRelationships":["failure"],"scheduledState":"ENABLED","retryCount":10,"retriedRelationships":[],"backoffMechanism":"PENALIZE_FLOWFILE","maxBackoffPeriod":"10 mins","componentType":"PROCESSOR","groupIdentifier":"b3c0c7ff-385d-33e2-bb47-9ce8e5dcbe6e"}],"inputPorts":[],"outputPorts":[],"connections":[{"identifier":"3e97e327-1a18-31b3-bbf8-0dc593f73a03","instanceIdentifier":"2559c168-06e9-456c-9de7-72220127c35a","name":"","source":{"id":"dd01a121-badd-3d34-a805-47a743f1ba08","type":"FUNNEL","groupId":"b3c0c7ff-385d-33e2-bb47-9ce8e5dcbe6e","name":"Funnel","comments":"","instanceIdentifier":"2c272f59-c00b-44cb-a6b6-b4a6510ac3db"},"destination":{"id":"b9519c10-259e-3cc4-8512-ee20fd64ac38","type":"PROCESSOR","groupId":"b3c0c7ff-385d-33e2-bb47-9ce8e5dcbe6e","name":"ReplaceText","comments":"","instanceIdentifier":"74af00f4-5b4a-4236-bae0-ebc6bef5ae1f"},"labelIndex":1,"zIndex":0,"selectedRelationships":[""],"backPressureObjectThreshold":0,"backPressureDataSizeThreshold":"0 MB","flowFileExpiration":"0 sec","prioritizers":[],"bends":[],"loadBalanceStrategy":"DO_NOT_LOAD_BALANCE","loadBalanceCompression":"DO_NOT_COMPRESS","componentType":"CONNECTION","groupIdentifier":"b3c0c7ff-385d-33e2-bb47-9ce8e5dcbe6e"},{"identifier":"ca3d9cac-5e10-3da8-963d-f882f59e9536","instanceIdentifier":"e2f12e55-8b85-4e04-99c7-c33718b96495","name":"","source":{"id":"31ffa34c-fe27-3a13-85aa-381fcbc79f11","type":"PROCESSOR","groupId":"b3c0c7ff-385d-33e2-bb47-9ce8e5dcbe6e","name":"ReplaceText","comments":"","instanceIdentifier":"92fd75c1-7d4c-4adc-bd2e-e3c33075dfb6"},"destination":{"id":"c26f79a3-7ecf-38e2-8037-e15336ccbd0b","type":"PROCESSOR","groupId":"b3c0c7ff-385d-33e2-bb47-9ce8e5dcbe6e","name":"ExtractText","comments":"","instanceIdentifier":"a4b100b5-f3da-41ed-8ed9-a198d3f4aba3"},"labelIndex":1,"zIndex":0,"selectedRelationships":["success"],"backPressureObjectThreshold":0,"backPressureDataSizeThreshold":"0 MB","flowFileExpiration":"0 sec","prioritizers":[],"bends":[],"loadBalanceStrategy":"DO_NOT_LOAD_BALANCE","loadBalanceCompression":"DO_NOT_COMPRESS","componentType":"CONNECTION","groupIdentifier":"b3c0c7ff-385d-33e2-bb47-9ce8e5dcbe6e"},{"identifier":"929ffe39-552a-3329-b02c-f537156f21b4","instanceIdentifier":"69888bdf-518b-467c-b4aa-70dc97c5d5dc","name":"","source":{"id":"db9cf755-e0f0-3cf3-90a4-025f19297ae9","type":"PROCESSOR","groupId":"b3c0c7ff-385d-33e2-bb47-9ce8e5dcbe6e","name":"ExtractText","comments":"","instanceIdentifier":"de1f9362-f8a5-44a0-bef7-9a6f608e6e66"},"destination":{"id":"062d7339-50ae-3c65-a629-718e0cb29b2d","type":"PROCESSOR","groupId":"b3c0c7ff-385d-33e2-bb47-9ce8e5dcbe6e","name":"RouteOnAttribute","comments":"","instanceIdentifier":"dd6046fa-f346-461d-93a3-c65bcc38e3d8"},"labelIndex":1,"zIndex":0,"selectedRelationships":["matched"],"backPressureObjectThreshold":0,"backPressureDataSizeThreshold":"0 MB","flowFileExpiration":"0 sec","prioritizers":[],"bends":[],"loadBalanceStrategy":"DO_NOT_LOAD_BALANCE","loadBalanceCompression":"DO_NOT_COMPRESS","componentType":"CONNECTION","groupIdentifier":"b3c0c7ff-385d-33e2-bb47-9ce8e5dcbe6e"},{"identifier":"c1e9ebdf-085f-3e2d-9cff-e67ab54701cc","instanceIdentifier":"3a1725d4-4e55-448f-8f52-453d9dbfc40b","name":"data is odd","source":{"id":"ca07aec3-aa90-31a9-bdea-6bb999a2a608","type":"PROCESSOR","groupId":"b3c0c7ff-385d-33e2-bb47-9ce8e5dcbe6e","name":"RouteOnAttribute","comments":"","instanceIdentifier":"b4369bf2-e5d5-4df8-b397-4d63c1a11d29"},"destination":{"id":"34daddce-e121-3e80-a4b1-40d228a20018","type":"FUNNEL","groupId":"b3c0c7ff-385d-33e2-bb47-9ce8e5dcbe6e","name":"Funnel","comments":"","instanceIdentifier":"a008d0fa-ab77-48ff-bc46-09a92344b4a5"},"labelIndex":1,"zIndex":0,"selectedRelationships":["unmatched"],"backPressureObjectThreshold":0,"backPressureDataSizeThreshold":"0 MB","flowFileExpiration":"0 sec","prioritizers":[],"bends":[],"loadBalanceStrategy":"DO_NOT_LOAD_BALANCE","loadBalanceCompression":"DO_NOT_COMPRESS","componentType":"CONNECTION","groupIdentifier":"b3c0c7ff-385d-33e2-bb47-9ce8e5dcbe6e"},{"identifier":"07d8caab-a3dc-3911-bba3-1d3b4c9b52e8","instanceIdentifier":"c9f12359-99bb-47c4-bac7-9fc9e1f8779f","name":"","source":{"id":"96e61aa4-70af-375d-beb3-58e3b8256203","type":"PROCESSOR","groupId":"b3c0c7ff-385d-33e2-bb47-9ce8e5dcbe6e","name":"ExtractText","comments":"","instanceIdentifier":"2a1e8019-5264-48e7-bc61-8d71c4ecaa82"},"destination":{"id":"25db9f1b-cfc4-36dc-ab64-c5feeef81f41","type":"PROCESSOR","groupId":"b3c0c7ff-385d-33e2-bb47-9ce8e5dcbe6e","name":"RouteOnAttribute","comments":"","instanceIdentifier":"c697e51f-aa4f-4cbf-a4e7-25156970a360"},"labelIndex":1,"zIndex":0,"selectedRelationships":["matched"],"backPressureObjectThreshold":0,"backPressureDataSizeThreshold":"0 MB","flowFileExpiration":"0 sec","prioritizers":[],"bends":[],"loadBalanceStrategy":"DO_NOT_LOAD_BALANCE","loadBalanceCompression":"DO_NOT_COMPRESS","componentType":"CONNECTION","groupIdentifier":"b3c0c7ff-385d-33e2-bb47-9ce8e5dcbe6e"},{"identifier":"17ec0f9b-460f-3618-9290-0cc5bc8bcd1a","instanceIdentifier":"e2fef810-29c7-4c7c-a127-9fcf01b27634","name":"","source":{"id":"7d516c36-b449-3409-8745-400d6c43132c","type":"PROCESSOR","groupId":"b3c0c7ff-385d-33e2-bb47-9ce8e5dcbe6e","name":"ReplaceText","comments":"","instanceIdentifier":"3d47cfae-2ada-4992-8597-e7e814490749"},"destination":{"id":"1b852682-1499-38ff-a80d-33c7cfb59801","type":"PROCESSOR","groupId":"b3c0c7ff-385d-33e2-bb47-9ce8e5dcbe6e","name":"ExtractText","comments":"","instanceIdentifier":"0e49512d-67f4-4b74-9880-ea2e2fb3436b"},"labelIndex":1,"zIndex":0,"selectedRelationships":["success"],"backPressureObjectThreshold":0,"backPressureDataSizeThreshold":"0 MB","flowFileExpiration":"0 sec","prioritizers":[],"bends":[],"loadBalanceStrategy":"DO_NOT_LOAD_BALANCE","loadBalanceCompression":"DO_NOT_COMPRESS","componentType":"CONNECTION","groupIdentifier":"b3c0c7ff-385d-33e2-bb47-9ce8e5dcbe6e"},{"identifier":"67cf8272-2430-365d-ac3c-18288c87099c","instanceIdentifier":"9a53fe17-7b7b-4a29-b993-b19fc7ba3e9b","name":"","source":{"id":"54fb1fc6-1037-3254-ab0c-c1a430eae27f","type":"OUTPUT_PORT","groupId":"dd5b00c2-ae44-3a20-965b-291d897a795e","name":"Output","comments":"","instanceIdentifier":"b6e55392-154f-46d6-ba5a-4d1311659094"},"destination":{"id":"31ffa34c-fe27-3a13-85aa-381fcbc79f11","type":"PROCESSOR","groupId":"b3c0c7ff-385d-33e2-bb47-9ce8e5dcbe6e","name":"ReplaceText","comments":"","instanceIdentifier":"92fd75c1-7d4c-4adc-bd2e-e3c33075dfb6"},"labelIndex":1,"zIndex":0,"selectedRelationships":[""],"backPressureObjectThreshold":0,"backPressureDataSizeThreshold":"0 MB","flowFileExpiration":"0 sec","prioritizers":[],"bends":[],"loadBalanceStrategy":"DO_NOT_LOAD_BALANCE","loadBalanceCompression":"DO_NOT_COMPRESS","componentType":"CONNECTION","groupIdentifier":"b3c0c7ff-385d-33e2-bb47-9ce8e5dcbe6e"},{"identifier":"e56d57e9-58f8-3a11-9fbf-60af07fcf32f","instanceIdentifier":"657c6894-2e76-4dd1-bcfe-94e8beeb82ff","name":"","source":{"id":"b9519c10-259e-3cc4-8512-ee20fd64ac38","type":"PROCESSOR","groupId":"b3c0c7ff-385d-33e2-bb47-9ce8e5dcbe6e","name":"ReplaceText","comments":"","instanceIdentifier":"74af00f4-5b4a-4236-bae0-ebc6bef5ae1f"},"destination":{"id":"96e61aa4-70af-375d-beb3-58e3b8256203","type":"PROCESSOR","groupId":"b3c0c7ff-385d-33e2-bb47-9ce8e5dcbe6e","name":"ExtractText","comments":"","instanceIdentifier":"2a1e8019-5264-48e7-bc61-8d71c4ecaa82"},"labelIndex":1,"zIndex":0,"selectedRelationships":["success"],"backPressureObjectThreshold":0,"backPressureDataSizeThreshold":"0 MB","flowFileExpiration":"0 sec","prioritizers":[],"bends":[],"loadBalanceStrategy":"DO_NOT_LOAD_BALANCE","loadBalanceCompression":"DO_NOT_COMPRESS","componentType":"CONNECTION","groupIdentifier":"b3c0c7ff-385d-33e2-bb47-9ce8e5dcbe6e"},{"identifier":"e4b512b6-b3fc-3b3c-9179-bc050e8fd6b0","instanceIdentifier":"c08639c3-fdf4-465d-9f17-361fc2b8717f","name":"","source":{"id":"dd01a121-badd-3d34-a805-47a743f1ba08","type":"FUNNEL","groupId":"b3c0c7ff-385d-33e2-bb47-9ce8e5dcbe6e","name":"Funnel","comments":"","instanceIdentifier":"2c272f59-c00b-44cb-a6b6-b4a6510ac3db"},"destination":{"id":"4db7d310-54d1-3639-87a5-f0ebdf08e394","type":"PROCESSOR","groupId":"b3c0c7ff-385d-33e2-bb47-9ce8e5dcbe6e","name":"ReplaceText","comments":"","instanceIdentifier":"ab9e02ee-511f-48de-ac07-da46481c1a70"},"labelIndex":0,"zIndex":0,"selectedRelationships":[""],"backPressureObjectThreshold":0,"backPressureDataSizeThreshold":"0 MB","flowFileExpiration":"0 sec","prioritizers":[],"bends":[{"x":1503.3428649902344,"y":332.97529876708984}],"loadBalanceStrategy":"DO_NOT_LOAD_BALANCE","loadBalanceCompression":"DO_NOT_COMPRESS","componentType":"CONNECTION","groupIdentifier":"b3c0c7ff-385d-33e2-bb47-9ce8e5dcbe6e"},{"identifier":"9c035ce4-b6e4-3813-aa05-c721c89df2fb","instanceIdentifier":"fb2d7868-d1f6-4f9a-83f2-2431acd4cf41","name":"","source":{"id":"ca07aec3-aa90-31a9-bdea-6bb999a2a608","type":"PROCESSOR","groupId":"b3c0c7ff-385d-33e2-bb47-9ce8e5dcbe6e","name":"RouteOnAttribute","comments":"","instanceIdentifier":"b4369bf2-e5d5-4df8-b397-4d63c1a11d29"},"destination":{"id":"dd01a121-badd-3d34-a805-47a743f1ba08","type":"FUNNEL","groupId":"b3c0c7ff-385d-33e2-bb47-9ce8e5dcbe6e","name":"Funnel","comments":"","instanceIdentifier":"2c272f59-c00b-44cb-a6b6-b4a6510ac3db"},"labelIndex":1,"zIndex":0,"selectedRelationships":["data is even"],"backPressureObjectThreshold":0,"backPressureDataSizeThreshold":"0 MB","flowFileExpiration":"0 sec","prioritizers":[],"bends":[],"loadBalanceStrategy":"DO_NOT_LOAD_BALANCE","loadBalanceCompression":"DO_NOT_COMPRESS","componentType":"CONNECTION","groupIdentifier":"b3c0c7ff-385d-33e2-bb47-9ce8e5dcbe6e"},{"identifier":"f75ffe06-6634-303a-8982-cb7004fe4bb0","instanceIdentifier":"a50b2dc1-baa9-402a-abf7-20e39b0dba0c","name":"","source":{"id":"25db9f1b-cfc4-36dc-ab64-c5feeef81f41","type":"PROCESSOR","groupId":"b3c0c7ff-385d-33e2-bb47-9ce8e5dcbe6e","name":"RouteOnAttribute","comments":"","instanceIdentifier":"c697e51f-aa4f-4cbf-a4e7-25156970a360"},"destination":{"id":"fe85a88f-b03f-3bbf-beb6-b12a12bd5a83","type":"INPUT_PORT","groupId":"dd5b00c2-ae44-3a20-965b-291d897a795e","name":"Input","comments":"","instanceIdentifier":"5f2c8c29-4cea-4da9-9a48-23abe40bd08d"},"labelIndex":1,"zIndex":0,"selectedRelationships":["content is even","unmatched"],"backPressureObjectThreshold":0,"backPressureDataSizeThreshold":"0 MB","flowFileExpiration":"0 sec","prioritizers":[],"bends":[],"loadBalanceStrategy":"DO_NOT_LOAD_BALANCE","loadBalanceCompression":"DO_NOT_COMPRESS","componentType":"CONNECTION","groupIdentifier":"b3c0c7ff-385d-33e2-bb47-9ce8e5dcbe6e"},{"identifier":"faa70a20-6ae6-311a-86b6-2964a42f1226","instanceIdentifier":"8a6298e5-2b79-4099-8b21-ab35c2c663c4","name":"","source":{"id":"c81682f7-ac5e-39a4-a89c-c8f5f8416bfa","type":"PROCESSOR","groupId":"b3c0c7ff-385d-33e2-bb47-9ce8e5dcbe6e","name":"ExtractText","comments":"","instanceIdentifier":"d908041d-3f7a-4d07-bf76-cdee0c2e2c28"},"destination":{"id":"ba48f6ad-95ab-3eac-91de-6d4950bb1764","type":"PROCESSOR","groupId":"b3c0c7ff-385d-33e2-bb47-9ce8e5dcbe6e","name":"RouteOnAttribute","comments":"","instanceIdentifier":"b1f6cb42-dbce-49a3-be9f-cfc6c889bb8b"},"labelIndex":1,"zIndex":0,"selectedRelationships":["matched"],"backPressureObjectThreshold":0,"backPressureDataSizeThreshold":"0 MB","flowFileExpiration":"0 sec","prioritizers":[],"bends":[],"loadBalanceStrategy":"DO_NOT_LOAD_BALANCE","loadBalanceCompression":"DO_NOT_COMPRESS","componentType":"CONNECTION","groupIdentifier":"b3c0c7ff-385d-33e2-bb47-9ce8e5dcbe6e"},{"identifier":"1d9d7702-5bc5-316d-ba04-812cade9ccf1","instanceIdentifier":"be5e2f01-7a6f-49d2-80b0-2dbb7b4efea5","name":"","source":{"id":"1b852682-1499-38ff-a80d-33c7cfb59801","type":"PROCESSOR","groupId":"b3c0c7ff-385d-33e2-bb47-9ce8e5dcbe6e","name":"ExtractText","comments":"","instanceIdentifier":"0e49512d-67f4-4b74-9880-ea2e2fb3436b"},"destination":{"id":"3bbe0eed-14a2-3ee2-94a3-8817f65f200c","type":"PROCESSOR","groupId":"b3c0c7ff-385d-33e2-bb47-9ce8e5dcbe6e","name":"RouteOnAttribute","comments":"","instanceIdentifier":"3dae3e31-ab40-49ca-bb86-21e91f24ee97"},"labelIndex":1,"zIndex":0,"selectedRelationships":["matched"],"backPressureObjectThreshold":0,"backPressureDataSizeThreshold":"0 MB","flowFileExpiration":"0 sec","prioritizers":[],"bends":[],"loadBalanceStrategy":"DO_NOT_LOAD_BALANCE","loadBalanceCompression":"DO_NOT_COMPRESS","componentType":"CONNECTION","groupIdentifier":"b3c0c7ff-385d-33e2-bb47-9ce8e5dcbe6e"},{"identifier":"48016160-3101-38c7-80ec-59b89a0ddb5b","instanceIdentifier":"d7b307c3-9a81-4764-ac47-86f1b34e3789","name":"","source":{"id":"7e30b537-b10c-39a9-9dab-f0825174a33a","type":"PROCESSOR","groupId":"b3c0c7ff-385d-33e2-bb47-9ce8e5dcbe6e","name":"ReplaceText","comments":"","instanceIdentifier":"65140227-fe60-42ad-beb9-69c36bc75acb"},"destination":{"id":"8823fbfd-07ce-303a-b1be-1f59501eab08","type":"PROCESSOR","groupId":"b3c0c7ff-385d-33e2-bb47-9ce8e5dcbe6e","name":"ExtractText","comments":"","instanceIdentifier":"a05b4e5f-059f-4105-af89-a551098d2779"},"labelIndex":1,"zIndex":0,"selectedRelationships":["success"],"backPressureObjectThreshold":0,"backPressureDataSizeThreshold":"0 MB","flowFileExpiration":"0 sec","prioritizers":[],"bends":[],"loadBalanceStrategy":"DO_NOT_LOAD_BALANCE","loadBalanceCompression":"DO_NOT_COMPRESS","componentType":"CONNECTION","groupIdentifier":"b3c0c7ff-385d-33e2-bb47-9ce8e5dcbe6e"},{"identifier":"905742bf-a80b-396e-bbb9-2a6ea28bac98","instanceIdentifier":"805d91a6-aa21-4cd4-a14f-28063fd7da52","name":"","source":{"id":"c26f79a3-7ecf-38e2-8037-e15336ccbd0b","type":"PROCESSOR","groupId":"b3c0c7ff-385d-33e2-bb47-9ce8e5dcbe6e","name":"ExtractText","comments":"","instanceIdentifier":"a4b100b5-f3da-41ed-8ed9-a198d3f4aba3"},"destination":{"id":"260a9b16-2dfe-3f85-8a48-ee2aba60563b","type":"PROCESSOR","groupId":"b3c0c7ff-385d-33e2-bb47-9ce8e5dcbe6e","name":"RouteOnAttribute","comments":"","instanceIdentifier":"6461ece1-db83-481f-88ba-534cd3a67b1b"},"labelIndex":1,"zIndex":0,"selectedRelationships":["matched"],"backPressureObjectThreshold":0,"backPressureDataSizeThreshold":"0 MB","flowFileExpiration":"0 sec","prioritizers":[],"bends":[],"loadBalanceStrategy":"DO_NOT_LOAD_BALANCE","loadBalanceCompression":"DO_NOT_COMPRESS","componentType":"CONNECTION","groupIdentifier":"b3c0c7ff-385d-33e2-bb47-9ce8e5dcbe6e"},{"identifier":"0bcc6ef3-8f38-36d4-a1e4-052409cbee14","instanceIdentifier":"988349d3-faf0-4ffb-8403-dd9db2376958","name":"","source":{"id":"4db7d310-54d1-3639-87a5-f0ebdf08e394","type":"PROCESSOR","groupId":"b3c0c7ff-385d-33e2-bb47-9ce8e5dcbe6e","name":"ReplaceText","comments":"","instanceIdentifier":"ab9e02ee-511f-48de-ac07-da46481c1a70"},"destination":{"id":"c81682f7-ac5e-39a4-a89c-c8f5f8416bfa","type":"PROCESSOR","groupId":"b3c0c7ff-385d-33e2-bb47-9ce8e5dcbe6e","name":"ExtractText","comments":"","instanceIdentifier":"d908041d-3f7a-4d07-bf76-cdee0c2e2c28"},"labelIndex":1,"zIndex":0,"selectedRelationships":["success"],"backPressureObjectThreshold":0,"backPressureDataSizeThreshold":"0 MB","flowFileExpiration":"0 sec","prioritizers":[],"bends":[],"loadBalanceStrategy":"DO_NOT_LOAD_BALANCE","loadBalanceCompression":"DO_NOT_COMPRESS","componentType":"CONNECTION","groupIdentifier":"b3c0c7ff-385d-33e2-bb47-9ce8e5dcbe6e"},{"identifier":"48caf01b-c304-306b-b88c-bb235de6baac","instanceIdentifier":"2087d4da-e40c-4674-ac98-e11e8f0f1851","name":"","source":{"id":"7b92d941-b477-3d3e-90fc-4214753750af","type":"PROCESSOR","groupId":"b3c0c7ff-385d-33e2-bb47-9ce8e5dcbe6e","name":"GenerateFlowFile","comments":"","instanceIdentifier":"f3063c47-c8ae-4b25-bd71-98dffdf33e0a"},"destination":{"id":"7e30b537-b10c-39a9-9dab-f0825174a33a","type":"PROCESSOR","groupId":"b3c0c7ff-385d-33e2-bb47-9ce8e5dcbe6e","name":"ReplaceText","comments":"","instanceIdentifier":"65140227-fe60-42ad-beb9-69c36bc75acb"},"labelIndex":1,"zIndex":0,"selectedRelationships":["success"],"backPressureObjectThreshold":0,"backPressureDataSizeThreshold":"0 MB","flowFileExpiration":"0 sec","prioritizers":[],"bends":[],"loadBalanceStrategy":"DO_NOT_LOAD_BALANCE","loadBalanceCompression":"DO_NOT_COMPRESS","componentType":"CONNECTION","groupIdentifier":"b3c0c7ff-385d-33e2-bb47-9ce8e5dcbe6e"},{"identifier":"3dafe3fc-fc29-3c63-bd9c-40cfe769e0eb","instanceIdentifier":"69ab1907-f0b7-490a-9630-95f886b4f49f","name":"","source":{"id":"8d24ad16-0816-36f0-86a0-fff0e33749fc","type":"PROCESSOR","groupId":"b3c0c7ff-385d-33e2-bb47-9ce8e5dcbe6e","name":"ReplaceText","comments":"","instanceIdentifier":"4b771af6-af32-443f-b807-6b9f7c476e39"},"destination":{"id":"db9cf755-e0f0-3cf3-90a4-025f19297ae9","type":"PROCESSOR","groupId":"b3c0c7ff-385d-33e2-bb47-9ce8e5dcbe6e","name":"ExtractText","comments":"","instanceIdentifier":"de1f9362-f8a5-44a0-bef7-9a6f608e6e66"},"labelIndex":1,"zIndex":0,"selectedRelationships":["success"],"backPressureObjectThreshold":0,"backPressureDataSizeThreshold":"0 MB","flowFileExpiration":"0 sec","prioritizers":[],"bends":[],"loadBalanceStrategy":"DO_NOT_LOAD_BALANCE","loadBalanceCompression":"DO_NOT_COMPRESS","componentType":"CONNECTION","groupIdentifier":"b3c0c7ff-385d-33e2-bb47-9ce8e5dcbe6e"},{"identifier":"e22425c9-4074-3ed3-9372-7d3ce953af4c","instanceIdentifier":"a1cf21e6-2cf3-4cb6-b969-e3521035e5af","name":"","source":{"id":"34daddce-e121-3e80-a4b1-40d228a20018","type":"FUNNEL","groupId":"b3c0c7ff-385d-33e2-bb47-9ce8e5dcbe6e","name":"Funnel","comments":"","instanceIdentifier":"a008d0fa-ab77-48ff-bc46-09a92344b4a5"},"destination":{"id":"7d516c36-b449-3409-8745-400d6c43132c","type":"PROCESSOR","groupId":"b3c0c7ff-385d-33e2-bb47-9ce8e5dcbe6e","name":"ReplaceText","comments":"","instanceIdentifier":"3d47cfae-2ada-4992-8597-e7e814490749"},"labelIndex":1,"zIndex":0,"selectedRelationships":[""],"backPressureObjectThreshold":0,"backPressureDataSizeThreshold":"0 MB","flowFileExpiration":"0 sec","prioritizers":[],"bends":[],"loadBalanceStrategy":"DO_NOT_LOAD_BALANCE","loadBalanceCompression":"DO_NOT_COMPRESS","componentType":"CONNECTION","groupIdentifier":"b3c0c7ff-385d-33e2-bb47-9ce8e5dcbe6e"},{"identifier":"e8eaef62-1d2a-32c1-94b5-16fed6aa6e20","instanceIdentifier":"a08466d6-8c3e-43d7-9d36-0f5019845342","name":"","source":{"id":"34daddce-e121-3e80-a4b1-40d228a20018","type":"FUNNEL","groupId":"b3c0c7ff-385d-33e2-bb47-9ce8e5dcbe6e","name":"Funnel","comments":"","instanceIdentifier":"a008d0fa-ab77-48ff-bc46-09a92344b4a5"},"destination":{"id":"8d24ad16-0816-36f0-86a0-fff0e33749fc","type":"PROCESSOR","groupId":"b3c0c7ff-385d-33e2-bb47-9ce8e5dcbe6e","name":"ReplaceText","comments":"","instanceIdentifier":"4b771af6-af32-443f-b807-6b9f7c476e39"},"labelIndex":0,"zIndex":0,"selectedRelationships":[""],"backPressureObjectThreshold":0,"backPressureDataSizeThreshold":"0 MB","flowFileExpiration":"0 sec","prioritizers":[],"bends":[{"x":1003.0369262695312,"y":335.6515774536133}],"loadBalanceStrategy":"DO_NOT_LOAD_BALANCE","loadBalanceCompression":"DO_NOT_COMPRESS","componentType":"CONNECTION","groupIdentifier":"b3c0c7ff-385d-33e2-bb47-9ce8e5dcbe6e"},{"identifier":"a2b286dd-3258-3974-b375-2001ce80de16","instanceIdentifier":"2c06978d-8cc0-46f2-84c0-bf448029cdd9","name":"","source":{"id":"8823fbfd-07ce-303a-b1be-1f59501eab08","type":"PROCESSOR","groupId":"b3c0c7ff-385d-33e2-bb47-9ce8e5dcbe6e","name":"ExtractText","comments":"","instanceIdentifier":"a05b4e5f-059f-4105-af89-a551098d2779"},"destination":{"id":"ca07aec3-aa90-31a9-bdea-6bb999a2a608","type":"PROCESSOR","groupId":"b3c0c7ff-385d-33e2-bb47-9ce8e5dcbe6e","name":"RouteOnAttribute","comments":"","instanceIdentifier":"b4369bf2-e5d5-4df8-b397-4d63c1a11d29"},"labelIndex":1,"zIndex":0,"selectedRelationships":["matched"],"backPressureObjectThreshold":0,"backPressureDataSizeThreshold":"0 MB","flowFileExpiration":"0 sec","prioritizers":[],"bends":[],"loadBalanceStrategy":"DO_NOT_LOAD_BALANCE","loadBalanceCompression":"DO_NOT_COMPRESS","componentType":"CONNECTION","groupIdentifier":"b3c0c7ff-385d-33e2-bb47-9ce8e5dcbe6e"}],"labels":[{"identifier":"708cc2a8-ff60-358f-85e9-d45a91a4c404","instanceIdentifier":"4b34ece6-21eb-4191-8284-6d478cc6548b","position":{"x":1044.0781339291925,"y":259.6300635927613},"label":"Position Scaling Test Flow","zIndex":0,"width":268.2286682128906,"height":33.134117126464844,"style":{"font-size":"20px"},"componentType":"LABEL","groupIdentifier":"b3c0c7ff-385d-33e2-bb47-9ce8e5dcbe6e"}],"funnels":[{"identifier":"dd01a121-badd-3d34-a805-47a743f1ba08","instanceIdentifier":"2c272f59-c00b-44cb-a6b6-b4a6510ac3db","position":{"x":1492.8511047363281,"y":1097.0516369628906},"componentType":"FUNNEL","groupIdentifier":"b3c0c7ff-385d-33e2-bb47-9ce8e5dcbe6e"},{"identifier":"34daddce-e121-3e80-a4b1-40d228a20018","instanceIdentifier":"a008d0fa-ab77-48ff-bc46-09a92344b4a5","position":{"x":926.6429593829063,"y":1094.3790700295367},"componentType":"FUNNEL","groupIdentifier":"b3c0c7ff-385d-33e2-bb47-9ce8e5dcbe6e"}],"controllerServices":[],"variables":{},"defaultFlowFileExpiration":"0 sec","defaultBackPressureObjectThreshold":10000,"defaultBackPressureDataSizeThreshold":"1 GB","componentType":"PROCESS_GROUP","flowFileOutboundPolicy":"STREAM_WHEN_AVAILABLE","flowFileConcurrency":"UNBOUNDED"}} \ No newline at end of file diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/resources/conf/scale-positions-flow-0.7.0.xml b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/resources/conf/scale-positions-flow-0.7.0.xml deleted file mode 100644 index 7ebad5000d..0000000000 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/resources/conf/scale-positions-flow-0.7.0.xml +++ /dev/null @@ -1,1533 +0,0 @@ - - - - 10 - 5 - - 32b64df5-b2f5-4c47-9880-3d0bd5c8ff15 - NiFi Flow - - - - a05b4e5f-059f-4105-af89-a551098d2779 - ExtractText - - - - org.apache.nifi.controller.service.mock.DummyProcessor - 1 - 0 sec - 30 sec - 1 sec - WARN - false - RUNNING - TIMER_DRIVEN - 0 - - Character Set - UTF-8 - - - Maximum Buffer Size - 1 MB - - - Maximum Capture Group Length - 1024 - - - Enable Canonical Equivalence - false - - - Enable Case-insensitive Matching - false - - - Permit Whitespace and Comments in Pattern - false - - - Enable DOTALL Mode - false - - - Enable Literal Parsing of the Pattern - false - - - Enable Multiline Mode - false - - - Enable Unicode-aware Case Folding - false - - - Enable Unicode Predefined Character Classes - false - - - Enable Unix Lines Mode - false - - - Include Capture Group 0 - false - - - data - ((?s:^.*$)) - - unmatched - - - 74af00f4-5b4a-4236-bae0-ebc6bef5ae1f - ReplaceText - - - - org.apache.nifi.controller.service.mock.DummyProcessor - 1 - 0 sec - 30 sec - 1 sec - WARN - false - RUNNING - TIMER_DRIVEN - 0 - - Regular Expression - (?s:^.*$) - - - Replacement Value - ${nextInt()} - - - Character Set - UTF-8 - - - Maximum Buffer Size - 1 MB - - - Replacement Strategy - Regex Replace - - - Evaluation Mode - Entire text - - failure - - - b4369bf2-e5d5-4df8-b397-4d63c1a11d29 - RouteOnAttribute - - - - org.apache.nifi.controller.service.mock.DummyProcessor - 1 - 0 sec - 30 sec - 1 sec - WARN - false - RUNNING - TIMER_DRIVEN - 0 - - Routing Strategy - Route to Property name - - - data is even - ${data:mod(2):equals(0)} - - - - b1f6cb42-dbce-49a3-be9f-cfc6c889bb8b - RouteOnAttribute - - - - org.apache.nifi.controller.service.mock.DummyProcessor - 1 - 0 sec - 30 sec - 1 sec - WARN - false - RUNNING - TIMER_DRIVEN - 0 - - Routing Strategy - Route to Property name - - - content is even - ${data:mod(2):equals(0)} - - content is even - unmatched - - - 0e49512d-67f4-4b74-9880-ea2e2fb3436b - ExtractText - - - - org.apache.nifi.controller.service.mock.DummyProcessor - 1 - 0 sec - 30 sec - 1 sec - WARN - false - RUNNING - TIMER_DRIVEN - 0 - - Character Set - UTF-8 - - - Maximum Buffer Size - 1 MB - - - Maximum Capture Group Length - 1024 - - - Enable Canonical Equivalence - false - - - Enable Case-insensitive Matching - false - - - Permit Whitespace and Comments in Pattern - false - - - Enable DOTALL Mode - false - - - Enable Literal Parsing of the Pattern - false - - - Enable Multiline Mode - false - - - Enable Unicode-aware Case Folding - false - - - Enable Unicode Predefined Character Classes - false - - - Enable Unix Lines Mode - false - - - Include Capture Group 0 - false - - - data - ((?s:^.*$)) - - unmatched - - - c697e51f-aa4f-4cbf-a4e7-25156970a360 - RouteOnAttribute - - - - org.apache.nifi.controller.service.mock.DummyProcessor - 1 - 0 sec - 30 sec - 1 sec - WARN - false - RUNNING - TIMER_DRIVEN - 0 - - Routing Strategy - Route to Property name - - - content is even - ${data:mod(2):equals(0)} - - - - 2a1e8019-5264-48e7-bc61-8d71c4ecaa82 - ExtractText - - - - org.apache.nifi.controller.service.mock.DummyProcessor - 1 - 0 sec - 30 sec - 1 sec - WARN - false - RUNNING - TIMER_DRIVEN - 0 - - Character Set - UTF-8 - - - Maximum Buffer Size - 1 MB - - - Maximum Capture Group Length - 1024 - - - Enable Canonical Equivalence - false - - - Enable Case-insensitive Matching - false - - - Permit Whitespace and Comments in Pattern - false - - - Enable DOTALL Mode - false - - - Enable Literal Parsing of the Pattern - false - - - Enable Multiline Mode - false - - - Enable Unicode-aware Case Folding - false - - - Enable Unicode Predefined Character Classes - false - - - Enable Unix Lines Mode - false - - - Include Capture Group 0 - false - - - data - ((?s:^.*$)) - - unmatched - - - ab9e02ee-511f-48de-ac07-da46481c1a70 - ReplaceText - - - - org.apache.nifi.controller.service.mock.DummyProcessor - 1 - 0 sec - 30 sec - 1 sec - WARN - false - RUNNING - TIMER_DRIVEN - 0 - - Regular Expression - (?s:^.*$) - - - Replacement Value - ${nextInt()} - - - Character Set - UTF-8 - - - Maximum Buffer Size - 1 MB - - - Replacement Strategy - Regex Replace - - - Evaluation Mode - Entire text - - failure - - - a4b100b5-f3da-41ed-8ed9-a198d3f4aba3 - ExtractText - - - - org.apache.nifi.controller.service.mock.DummyProcessor - 1 - 0 sec - 30 sec - 1 sec - WARN - false - RUNNING - TIMER_DRIVEN - 0 - - Character Set - UTF-8 - - - Maximum Buffer Size - 1 MB - - - Maximum Capture Group Length - 1024 - - - Enable Canonical Equivalence - false - - - Enable Case-insensitive Matching - false - - - Permit Whitespace and Comments in Pattern - false - - - Enable DOTALL Mode - false - - - Enable Literal Parsing of the Pattern - false - - - Enable Multiline Mode - false - - - Enable Unicode-aware Case Folding - false - - - Enable Unicode Predefined Character Classes - false - - - Enable Unix Lines Mode - false - - - Include Capture Group 0 - false - - - data - ((?s:^.*$)) - - unmatched - - - 4b771af6-af32-443f-b807-6b9f7c476e39 - ReplaceText - - - - org.apache.nifi.controller.service.mock.DummyProcessor - 1 - 0 sec - 30 sec - 1 sec - WARN - false - RUNNING - TIMER_DRIVEN - 0 - - Regular Expression - (?s:^.*$) - - - Replacement Value - ${nextInt()} - - - Character Set - UTF-8 - - - Maximum Buffer Size - 1 MB - - - Replacement Strategy - Regex Replace - - - Evaluation Mode - Entire text - - failure - - - f3063c47-c8ae-4b25-bd71-98dffdf33e0a - GenerateFlowFile - - - - org.apache.nifi.controller.service.mock.DummyProcessor - 1 - 1 sec - 30 sec - 1 sec - WARN - false - RUNNING - TIMER_DRIVEN - 0 - - File Size - 0b - - - Batch Size - 1 - - - Data Format - Binary - - - Unique FlowFiles - false - - - - 3dae3e31-ab40-49ca-bb86-21e91f24ee97 - RouteOnAttribute - - - - org.apache.nifi.controller.service.mock.DummyProcessor - 1 - 0 sec - 30 sec - 1 sec - WARN - false - RUNNING - TIMER_DRIVEN - 0 - - Routing Strategy - Route to Property name - - - content is even - ${data:mod(2):equals(0)} - - content is even - unmatched - - - de1f9362-f8a5-44a0-bef7-9a6f608e6e66 - ExtractText - - - - org.apache.nifi.controller.service.mock.DummyProcessor - 1 - 0 sec - 30 sec - 1 sec - WARN - false - RUNNING - TIMER_DRIVEN - 0 - - Character Set - UTF-8 - - - Maximum Buffer Size - 1 MB - - - Maximum Capture Group Length - 1024 - - - Enable Canonical Equivalence - false - - - Enable Case-insensitive Matching - false - - - Permit Whitespace and Comments in Pattern - false - - - Enable DOTALL Mode - false - - - Enable Literal Parsing of the Pattern - false - - - Enable Multiline Mode - false - - - Enable Unicode-aware Case Folding - false - - - Enable Unicode Predefined Character Classes - false - - - Enable Unix Lines Mode - false - - - Include Capture Group 0 - false - - - data - ((?s:^.*$)) - - unmatched - - - 92fd75c1-7d4c-4adc-bd2e-e3c33075dfb6 - ReplaceText - - - - org.apache.nifi.controller.service.mock.DummyProcessor - 1 - 0 sec - 30 sec - 1 sec - WARN - false - RUNNING - TIMER_DRIVEN - 0 - - Regular Expression - (?s:^.*$) - - - Replacement Value - ${nextInt()} - - - Character Set - UTF-8 - - - Maximum Buffer Size - 1 MB - - - Replacement Strategy - Regex Replace - - - Evaluation Mode - Entire text - - failure - - - d908041d-3f7a-4d07-bf76-cdee0c2e2c28 - ExtractText - - - - org.apache.nifi.controller.service.mock.DummyProcessor - 1 - 0 sec - 30 sec - 1 sec - WARN - false - RUNNING - TIMER_DRIVEN - 0 - - Character Set - UTF-8 - - - Maximum Buffer Size - 1 MB - - - Maximum Capture Group Length - 1024 - - - Enable Canonical Equivalence - false - - - Enable Case-insensitive Matching - false - - - Permit Whitespace and Comments in Pattern - false - - - Enable DOTALL Mode - false - - - Enable Literal Parsing of the Pattern - false - - - Enable Multiline Mode - false - - - Enable Unicode-aware Case Folding - false - - - Enable Unicode Predefined Character Classes - false - - - Enable Unix Lines Mode - false - - - Include Capture Group 0 - false - - - data - ((?s:^.*$)) - - unmatched - - - dd6046fa-f346-461d-93a3-c65bcc38e3d8 - RouteOnAttribute - - - - org.apache.nifi.controller.service.mock.DummyProcessor - 1 - 0 sec - 30 sec - 1 sec - WARN - false - RUNNING - TIMER_DRIVEN - 0 - - Routing Strategy - Route to Property name - - - content is even - ${data:mod(2):equals(0)} - - content is even - unmatched - - - 6461ece1-db83-481f-88ba-534cd3a67b1b - RouteOnAttribute - - - - org.apache.nifi.controller.service.mock.DummyProcessor - 1 - 0 sec - 30 sec - 1 sec - WARN - false - RUNNING - TIMER_DRIVEN - 0 - - Routing Strategy - Route to Property name - - - content is even - ${data:mod(2):equals(0)} - - content is even - unmatched - - - 65140227-fe60-42ad-beb9-69c36bc75acb - ReplaceText - - - - org.apache.nifi.controller.service.mock.DummyProcessor - 1 - 0 sec - 30 sec - 1 sec - WARN - false - RUNNING - TIMER_DRIVEN - 0 - - Regular Expression - (?s:^.*$) - - - Replacement Value - ${nextInt()} - - - Character Set - UTF-8 - - - Maximum Buffer Size - 1 MB - - - Replacement Strategy - Regex Replace - - - Evaluation Mode - Entire text - - failure - - - 3d47cfae-2ada-4992-8597-e7e814490749 - ReplaceText - - - - org.apache.nifi.controller.service.mock.DummyProcessor - 1 - 0 sec - 30 sec - 1 sec - WARN - false - RUNNING - TIMER_DRIVEN - 0 - - Regular Expression - (?s:^.*$) - - - Replacement Value - ${nextInt()} - - - Character Set - UTF-8 - - - Maximum Buffer Size - 1 MB - - - Replacement Strategy - Regex Replace - - - Evaluation Mode - Entire text - - failure - - - - a008d0fa-ab77-48ff-bc46-09a92344b4a5 - - - - 2c272f59-c00b-44cb-a6b6-b4a6510ac3db - - - - e7e03999-bbb8-491f-bba0-dbf772ec1966 - Process Group - - - - aea93188-0832-4876-9bb8-38901a773922 - ExtractText - - - - org.apache.nifi.controller.service.mock.DummyProcessor - 1 - 0 sec - 30 sec - 1 sec - WARN - false - RUNNING - TIMER_DRIVEN - 0 - - Character Set - UTF-8 - - - Maximum Buffer Size - 1 MB - - - Maximum Capture Group Length - 1024 - - - Enable Canonical Equivalence - false - - - Enable Case-insensitive Matching - false - - - Permit Whitespace and Comments in Pattern - false - - - Enable DOTALL Mode - false - - - Enable Literal Parsing of the Pattern - false - - - Enable Multiline Mode - false - - - Enable Unicode-aware Case Folding - false - - - Enable Unicode Predefined Character Classes - false - - - Enable Unix Lines Mode - false - - - Include Capture Group 0 - false - - - data - ((?s:^.*$)) - - unmatched - - - 3b3957f9-91ff-4817-9a7f-fa80f78c2b72 - ReplaceText - - - - org.apache.nifi.controller.service.mock.DummyProcessor - 1 - 0 sec - 30 sec - 1 sec - WARN - false - RUNNING - TIMER_DRIVEN - 0 - - Regular Expression - (?s:^.*$) - - - Replacement Value - ${nextInt()} - - - Character Set - UTF-8 - - - Maximum Buffer Size - 1 MB - - - Replacement Strategy - Regex Replace - - - Evaluation Mode - Entire text - - failure - - - 79d49fec-3b95-4e7b-a949-0d414594e7fa - RouteOnAttribute - - - - org.apache.nifi.controller.service.mock.DummyProcessor - 1 - 0 sec - 30 sec - 1 sec - WARN - false - RUNNING - TIMER_DRIVEN - 0 - - Routing Strategy - Route to Property name - - - content is even - ${data:mod(2):equals(0)} - - unmatched - - - 5f2c8c29-4cea-4da9-9a48-23abe40bd08d - Input - - - RUNNING - - - b6e55392-154f-46d6-ba5a-4d1311659094 - Output - - - RUNNING - - - 7b7f871b-8f4b-4ce7-a9d4-2f810bf2b55f - - - 1 - 0 - aea93188-0832-4876-9bb8-38901a773922 - e7e03999-bbb8-491f-bba0-dbf772ec1966 - PROCESSOR - 79d49fec-3b95-4e7b-a949-0d414594e7fa - e7e03999-bbb8-491f-bba0-dbf772ec1966 - PROCESSOR - matched - 0 - 0 MB - 0 sec - - - 8bf960ca-ba33-4b5b-9830-1a808dd2db0e - - - 1 - 0 - 79d49fec-3b95-4e7b-a949-0d414594e7fa - e7e03999-bbb8-491f-bba0-dbf772ec1966 - PROCESSOR - b6e55392-154f-46d6-ba5a-4d1311659094 - e7e03999-bbb8-491f-bba0-dbf772ec1966 - OUTPUT_PORT - content is even - 0 - 0 MB - 0 sec - - - a82d78cb-de24-40e3-8fb9-d2117919cb31 - - - 1 - 0 - 3b3957f9-91ff-4817-9a7f-fa80f78c2b72 - e7e03999-bbb8-491f-bba0-dbf772ec1966 - PROCESSOR - aea93188-0832-4876-9bb8-38901a773922 - e7e03999-bbb8-491f-bba0-dbf772ec1966 - PROCESSOR - success - 0 - 0 MB - 0 sec - - - 197ffc73-76d3-4e83-84ca-a431fef50a8f - - - 1 - 0 - 5f2c8c29-4cea-4da9-9a48-23abe40bd08d - e7e03999-bbb8-491f-bba0-dbf772ec1966 - INPUT_PORT - 3b3957f9-91ff-4817-9a7f-fa80f78c2b72 - e7e03999-bbb8-491f-bba0-dbf772ec1966 - PROCESSOR - - 0 - 0 MB - 0 sec - - - - b1ebe9d0-56db-4f2e-8448-b78ec9daab66 - https://localhost:8080/nifi - - - https://localhost:8080/nifi - 30 sec - 10 sec - false - - - c08639c3-fdf4-465d-9f17-361fc2b8717f - - - - - 0 - 0 - 2c272f59-c00b-44cb-a6b6-b4a6510ac3db - 32b64df5-b2f5-4c47-9880-3d0bd5c8ff15 - FUNNEL - ab9e02ee-511f-48de-ac07-da46481c1a70 - 32b64df5-b2f5-4c47-9880-3d0bd5c8ff15 - PROCESSOR - - 0 - 0 MB - 0 sec - - - d7b307c3-9a81-4764-ac47-86f1b34e3789 - - - 1 - 0 - 65140227-fe60-42ad-beb9-69c36bc75acb - 32b64df5-b2f5-4c47-9880-3d0bd5c8ff15 - PROCESSOR - a05b4e5f-059f-4105-af89-a551098d2779 - 32b64df5-b2f5-4c47-9880-3d0bd5c8ff15 - PROCESSOR - success - 0 - 0 MB - 0 sec - - - 2559c168-06e9-456c-9de7-72220127c35a - - - 1 - 0 - 2c272f59-c00b-44cb-a6b6-b4a6510ac3db - 32b64df5-b2f5-4c47-9880-3d0bd5c8ff15 - FUNNEL - 74af00f4-5b4a-4236-bae0-ebc6bef5ae1f - 32b64df5-b2f5-4c47-9880-3d0bd5c8ff15 - PROCESSOR - - 0 - 0 MB - 0 sec - - - 805d91a6-aa21-4cd4-a14f-28063fd7da52 - - - 1 - 0 - a4b100b5-f3da-41ed-8ed9-a198d3f4aba3 - 32b64df5-b2f5-4c47-9880-3d0bd5c8ff15 - PROCESSOR - 6461ece1-db83-481f-88ba-534cd3a67b1b - 32b64df5-b2f5-4c47-9880-3d0bd5c8ff15 - PROCESSOR - matched - 0 - 0 MB - 0 sec - - - e2f12e55-8b85-4e04-99c7-c33718b96495 - - - 1 - 0 - 92fd75c1-7d4c-4adc-bd2e-e3c33075dfb6 - 32b64df5-b2f5-4c47-9880-3d0bd5c8ff15 - PROCESSOR - a4b100b5-f3da-41ed-8ed9-a198d3f4aba3 - 32b64df5-b2f5-4c47-9880-3d0bd5c8ff15 - PROCESSOR - success - 0 - 0 MB - 0 sec - - - 9a53fe17-7b7b-4a29-b993-b19fc7ba3e9b - - - 1 - 0 - b6e55392-154f-46d6-ba5a-4d1311659094 - e7e03999-bbb8-491f-bba0-dbf772ec1966 - OUTPUT_PORT - 92fd75c1-7d4c-4adc-bd2e-e3c33075dfb6 - 32b64df5-b2f5-4c47-9880-3d0bd5c8ff15 - PROCESSOR - - 0 - 0 MB - 0 sec - - - 657c6894-2e76-4dd1-bcfe-94e8beeb82ff - - - 1 - 0 - 74af00f4-5b4a-4236-bae0-ebc6bef5ae1f - 32b64df5-b2f5-4c47-9880-3d0bd5c8ff15 - PROCESSOR - 2a1e8019-5264-48e7-bc61-8d71c4ecaa82 - 32b64df5-b2f5-4c47-9880-3d0bd5c8ff15 - PROCESSOR - success - 0 - 0 MB - 0 sec - - - be5e2f01-7a6f-49d2-80b0-2dbb7b4efea5 - - - 1 - 0 - 0e49512d-67f4-4b74-9880-ea2e2fb3436b - 32b64df5-b2f5-4c47-9880-3d0bd5c8ff15 - PROCESSOR - 3dae3e31-ab40-49ca-bb86-21e91f24ee97 - 32b64df5-b2f5-4c47-9880-3d0bd5c8ff15 - PROCESSOR - matched - 0 - 0 MB - 0 sec - - - c9f12359-99bb-47c4-bac7-9fc9e1f8779f - - - 1 - 0 - 2a1e8019-5264-48e7-bc61-8d71c4ecaa82 - 32b64df5-b2f5-4c47-9880-3d0bd5c8ff15 - PROCESSOR - c697e51f-aa4f-4cbf-a4e7-25156970a360 - 32b64df5-b2f5-4c47-9880-3d0bd5c8ff15 - PROCESSOR - matched - 0 - 0 MB - 0 sec - - - 2087d4da-e40c-4674-ac98-e11e8f0f1851 - - - 1 - 0 - f3063c47-c8ae-4b25-bd71-98dffdf33e0a - 32b64df5-b2f5-4c47-9880-3d0bd5c8ff15 - PROCESSOR - 65140227-fe60-42ad-beb9-69c36bc75acb - 32b64df5-b2f5-4c47-9880-3d0bd5c8ff15 - PROCESSOR - success - 0 - 0 MB - 0 sec - - - 8a6298e5-2b79-4099-8b21-ab35c2c663c4 - - - 1 - 0 - d908041d-3f7a-4d07-bf76-cdee0c2e2c28 - 32b64df5-b2f5-4c47-9880-3d0bd5c8ff15 - PROCESSOR - b1f6cb42-dbce-49a3-be9f-cfc6c889bb8b - 32b64df5-b2f5-4c47-9880-3d0bd5c8ff15 - PROCESSOR - matched - 0 - 0 MB - 0 sec - - - 69888bdf-518b-467c-b4aa-70dc97c5d5dc - - - 1 - 0 - de1f9362-f8a5-44a0-bef7-9a6f608e6e66 - 32b64df5-b2f5-4c47-9880-3d0bd5c8ff15 - PROCESSOR - dd6046fa-f346-461d-93a3-c65bcc38e3d8 - 32b64df5-b2f5-4c47-9880-3d0bd5c8ff15 - PROCESSOR - matched - 0 - 0 MB - 0 sec - - - e2fef810-29c7-4c7c-a127-9fcf01b27634 - - - 1 - 0 - 3d47cfae-2ada-4992-8597-e7e814490749 - 32b64df5-b2f5-4c47-9880-3d0bd5c8ff15 - PROCESSOR - 0e49512d-67f4-4b74-9880-ea2e2fb3436b - 32b64df5-b2f5-4c47-9880-3d0bd5c8ff15 - PROCESSOR - success - 0 - 0 MB - 0 sec - - - 69ab1907-f0b7-490a-9630-95f886b4f49f - - - 1 - 0 - 4b771af6-af32-443f-b807-6b9f7c476e39 - 32b64df5-b2f5-4c47-9880-3d0bd5c8ff15 - PROCESSOR - de1f9362-f8a5-44a0-bef7-9a6f608e6e66 - 32b64df5-b2f5-4c47-9880-3d0bd5c8ff15 - PROCESSOR - success - 0 - 0 MB - 0 sec - - - fb2d7868-d1f6-4f9a-83f2-2431acd4cf41 - - - 1 - 0 - b4369bf2-e5d5-4df8-b397-4d63c1a11d29 - 32b64df5-b2f5-4c47-9880-3d0bd5c8ff15 - PROCESSOR - 2c272f59-c00b-44cb-a6b6-b4a6510ac3db - 32b64df5-b2f5-4c47-9880-3d0bd5c8ff15 - FUNNEL - data is even - 0 - 0 MB - 0 sec - - - 2c06978d-8cc0-46f2-84c0-bf448029cdd9 - - - 1 - 0 - a05b4e5f-059f-4105-af89-a551098d2779 - 32b64df5-b2f5-4c47-9880-3d0bd5c8ff15 - PROCESSOR - b4369bf2-e5d5-4df8-b397-4d63c1a11d29 - 32b64df5-b2f5-4c47-9880-3d0bd5c8ff15 - PROCESSOR - matched - 0 - 0 MB - 0 sec - - - a50b2dc1-baa9-402a-abf7-20e39b0dba0c - - - 1 - 0 - c697e51f-aa4f-4cbf-a4e7-25156970a360 - 32b64df5-b2f5-4c47-9880-3d0bd5c8ff15 - PROCESSOR - 5f2c8c29-4cea-4da9-9a48-23abe40bd08d - e7e03999-bbb8-491f-bba0-dbf772ec1966 - INPUT_PORT - content is even - unmatched - 0 - 0 MB - 0 sec - - - a1cf21e6-2cf3-4cb6-b969-e3521035e5af - - - 1 - 0 - a008d0fa-ab77-48ff-bc46-09a92344b4a5 - 32b64df5-b2f5-4c47-9880-3d0bd5c8ff15 - FUNNEL - 3d47cfae-2ada-4992-8597-e7e814490749 - 32b64df5-b2f5-4c47-9880-3d0bd5c8ff15 - PROCESSOR - - 0 - 0 MB - 0 sec - - - a08466d6-8c3e-43d7-9d36-0f5019845342 - - - - - 0 - 0 - a008d0fa-ab77-48ff-bc46-09a92344b4a5 - 32b64df5-b2f5-4c47-9880-3d0bd5c8ff15 - FUNNEL - 4b771af6-af32-443f-b807-6b9f7c476e39 - 32b64df5-b2f5-4c47-9880-3d0bd5c8ff15 - PROCESSOR - - 0 - 0 MB - 0 sec - - - 988349d3-faf0-4ffb-8403-dd9db2376958 - - - 1 - 0 - ab9e02ee-511f-48de-ac07-da46481c1a70 - 32b64df5-b2f5-4c47-9880-3d0bd5c8ff15 - PROCESSOR - d908041d-3f7a-4d07-bf76-cdee0c2e2c28 - 32b64df5-b2f5-4c47-9880-3d0bd5c8ff15 - PROCESSOR - success - 0 - 0 MB - 0 sec - - - 3a1725d4-4e55-448f-8f52-453d9dbfc40b - data is odd - - 1 - 0 - b4369bf2-e5d5-4df8-b397-4d63c1a11d29 - 32b64df5-b2f5-4c47-9880-3d0bd5c8ff15 - PROCESSOR - a008d0fa-ab77-48ff-bc46-09a92344b4a5 - 32b64df5-b2f5-4c47-9880-3d0bd5c8ff15 - FUNNEL - unmatched - 0 - 0 MB - 0 sec - - - - - diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/resources/conf/standard-flow.json b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/resources/conf/standard-flow.json new file mode 100644 index 0000000000..13d4401a5f --- /dev/null +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/resources/conf/standard-flow.json @@ -0,0 +1 @@ +{"encodingVersion":{"majorVersion":2,"minorVersion":0},"maxTimerDrivenThreadCount":66,"maxEventDrivenThreadCount":33,"registries":[],"parameterContexts":[],"parameterProviders":[],"controllerServices":[],"reportingTasks":[],"templates":[],"rootGroup":{"identifier":"81dc9bdb-52d0-3dc2-8036-dbd8313ed055","instanceIdentifier":"1234","name":"Root Group","comments":"","position":{"x":0.0,"y":0.0},"processGroups":[{"identifier":"d93591bd-f786-3e1e-8ee2-fca799911215","instanceIdentifier":"4321","name":"Child Group","comments":"","position":{"x":0.0,"y":0.0},"processGroups":[],"remoteProcessGroups":[],"processors":[{"identifier":"b59c67bf-196a-3758-991e-42f76670ceba","instanceIdentifier":"1111","name":"Data Destroyer","comments":"","position":{"x":0.0,"y":0.0},"type":"org.apache.nifi.processors.standard.TerminationFileProcessor","bundle":{"group":"default","artifact":"unknown","version":"unversioned"},"properties":{},"propertyDescriptors":{},"style":{},"schedulingStrategy":"TIMER_DRIVEN","executionNode":"ALL","penaltyDuration":"30 sec","yieldDuration":"1 sec","bulletinLevel":"WARN","runDurationMillis":0,"concurrentlySchedulableTaskCount":5,"autoTerminatedRelationships":[],"scheduledState":"ENABLED","retryCount":10,"retriedRelationships":[],"backoffMechanism":"PENALIZE_FLOWFILE","maxBackoffPeriod":"10 mins","componentType":"PROCESSOR","groupIdentifier":"d93591bd-f786-3e1e-8ee2-fca799911215"},{"identifier":"4a7d1ed4-1447-3e40-b3ac-29ccb8653d9b","instanceIdentifier":"0000","name":"Data Generator","comments":"","position":{"x":0.0,"y":0.0},"type":"org.apache.nifi.processors.standard.LocalSelectionFileProcessor","bundle":{"group":"default","artifact":"unknown","version":"unversioned"},"properties":{"polling.interval.secs":"enc{null}","keep.original":"enc{null}","root.directory.path":"enc{null}"},"propertyDescriptors":{},"style":{},"schedulingStrategy":"TIMER_DRIVEN","executionNode":"ALL","penaltyDuration":"30 sec","yieldDuration":"1 sec","bulletinLevel":"WARN","runDurationMillis":0,"concurrentlySchedulableTaskCount":5,"autoTerminatedRelationships":[],"scheduledState":"ENABLED","retryCount":10,"retriedRelationships":[],"backoffMechanism":"PENALIZE_FLOWFILE","maxBackoffPeriod":"10 mins","componentType":"PROCESSOR","groupIdentifier":"d93591bd-f786-3e1e-8ee2-fca799911215"}],"inputPorts":[],"outputPorts":[{"identifier":"6074c6aa-3488-33c2-9ddf-f2a7ca821aab","instanceIdentifier":"5555","name":"Output Port","position":{"x":0.0,"y":0.0},"type":"OUTPUT_PORT","concurrentlySchedulableTaskCount":1,"scheduledState":"ENABLED","allowRemoteAccess":false,"componentType":"OUTPUT_PORT","groupIdentifier":"d93591bd-f786-3e1e-8ee2-fca799911215"}],"connections":[{"identifier":"d79c8788-088c-3193-b024-4d8f1f36d2db","instanceIdentifier":"7777","name":"success","source":{"id":"4a7d1ed4-1447-3e40-b3ac-29ccb8653d9b","type":"PROCESSOR","groupId":"d93591bd-f786-3e1e-8ee2-fca799911215","name":"Data Generator","comments":"","instanceIdentifier":"0000"},"destination":{"id":"b59c67bf-196a-3758-991e-42f76670ceba","type":"PROCESSOR","groupId":"d93591bd-f786-3e1e-8ee2-fca799911215","name":"Data Destroyer","comments":"","instanceIdentifier":"1111"},"labelIndex":1,"zIndex":0,"selectedRelationships":["success"],"backPressureObjectThreshold":10000,"backPressureDataSizeThreshold":"1 GB","flowFileExpiration":"0 sec","prioritizers":[],"bends":[],"loadBalanceStrategy":"DO_NOT_LOAD_BALANCE","loadBalanceCompression":"DO_NOT_COMPRESS","componentType":"CONNECTION","groupIdentifier":"d93591bd-f786-3e1e-8ee2-fca799911215"},{"identifier":"cf79ae6a-ddba-30ad-8183-47359bd144d2","instanceIdentifier":"8888","name":"to.other.group","source":{"id":"4a7d1ed4-1447-3e40-b3ac-29ccb8653d9b","type":"PROCESSOR","groupId":"d93591bd-f786-3e1e-8ee2-fca799911215","name":"Data Generator","comments":"","instanceIdentifier":"0000"},"destination":{"id":"6074c6aa-3488-33c2-9ddf-f2a7ca821aab","type":"OUTPUT_PORT","groupId":"d93591bd-f786-3e1e-8ee2-fca799911215","name":"Output Port","instanceIdentifier":"5555"},"labelIndex":1,"zIndex":0,"selectedRelationships":["success"],"backPressureObjectThreshold":10000,"backPressureDataSizeThreshold":"1 GB","flowFileExpiration":"0 sec","prioritizers":[],"bends":[],"loadBalanceStrategy":"DO_NOT_LOAD_BALANCE","loadBalanceCompression":"DO_NOT_COMPRESS","componentType":"CONNECTION","groupIdentifier":"d93591bd-f786-3e1e-8ee2-fca799911215"}],"labels":[],"funnels":[],"controllerServices":[],"variables":{},"defaultFlowFileExpiration":"0 sec","defaultBackPressureObjectThreshold":10000,"defaultBackPressureDataSizeThreshold":"1 GB","componentType":"PROCESS_GROUP","flowFileConcurrency":"UNBOUNDED","flowFileOutboundPolicy":"STREAM_WHEN_AVAILABLE","groupIdentifier":"81dc9bdb-52d0-3dc2-8036-dbd8313ed055"},{"identifier":"fa246d02-62c3-3256-97b0-c72bb20eeb1d","instanceIdentifier":"9999","name":"Second Group","comments":"","position":{"x":0.0,"y":0.0},"processGroups":[],"remoteProcessGroups":[],"processors":[{"identifier":"934b5358-00b1-3ba8-b96a-5d72f72f1611","instanceIdentifier":"2222","name":"Logger","comments":"","position":{"x":0.0,"y":0.0},"type":"org.apache.nifi.processors.standard.AttributeLoggerProcessor","bundle":{"group":"default","artifact":"unknown","version":"unversioned"},"properties":{},"propertyDescriptors":{},"style":{},"schedulingStrategy":"TIMER_DRIVEN","executionNode":"ALL","penaltyDuration":"30 sec","yieldDuration":"1 sec","bulletinLevel":"WARN","runDurationMillis":0,"concurrentlySchedulableTaskCount":5,"autoTerminatedRelationships":[],"scheduledState":"ENABLED","retryCount":10,"retriedRelationships":[],"backoffMechanism":"PENALIZE_FLOWFILE","maxBackoffPeriod":"10 mins","componentType":"PROCESSOR","groupIdentifier":"fa246d02-62c3-3256-97b0-c72bb20eeb1d"},{"identifier":"2be9bd7a-3434-3703-8ca2-7d1918de58bd","instanceIdentifier":"3333","name":"Second Terminator","comments":"","position":{"x":0.0,"y":0.0},"type":"org.apache.nifi.processors.standard.TerminationFileProcessor","bundle":{"group":"default","artifact":"unknown","version":"unversioned"},"properties":{},"propertyDescriptors":{},"style":{},"schedulingStrategy":"TIMER_DRIVEN","executionNode":"ALL","penaltyDuration":"30 sec","yieldDuration":"1 sec","bulletinLevel":"WARN","runDurationMillis":0,"concurrentlySchedulableTaskCount":5,"autoTerminatedRelationships":[],"scheduledState":"ENABLED","retryCount":10,"retriedRelationships":[],"backoffMechanism":"PENALIZE_FLOWFILE","maxBackoffPeriod":"10 mins","componentType":"PROCESSOR","groupIdentifier":"fa246d02-62c3-3256-97b0-c72bb20eeb1d"}],"inputPorts":[{"identifier":"561eaa10-8c90-34d4-816d-a3186f2b9be0","instanceIdentifier":"32874","name":"Input Port","position":{"x":0.0,"y":0.0},"type":"INPUT_PORT","concurrentlySchedulableTaskCount":1,"scheduledState":"ENABLED","allowRemoteAccess":false,"componentType":"INPUT_PORT","groupIdentifier":"fa246d02-62c3-3256-97b0-c72bb20eeb1d"}],"outputPorts":[],"connections":[{"identifier":"e75b4fad-e838-3a3a-a7df-8793f0ca36fa","instanceIdentifier":"12378461946123976329","name":"success","source":{"id":"934b5358-00b1-3ba8-b96a-5d72f72f1611","type":"PROCESSOR","groupId":"fa246d02-62c3-3256-97b0-c72bb20eeb1d","name":"Logger","comments":"","instanceIdentifier":"2222"},"destination":{"id":"2be9bd7a-3434-3703-8ca2-7d1918de58bd","type":"PROCESSOR","groupId":"fa246d02-62c3-3256-97b0-c72bb20eeb1d","name":"Second Terminator","comments":"","instanceIdentifier":"3333"},"labelIndex":1,"zIndex":0,"selectedRelationships":["success"],"backPressureObjectThreshold":10000,"backPressureDataSizeThreshold":"1 GB","flowFileExpiration":"0 sec","prioritizers":[],"bends":[],"loadBalanceStrategy":"DO_NOT_LOAD_BALANCE","loadBalanceCompression":"DO_NOT_COMPRESS","componentType":"CONNECTION","groupIdentifier":"fa246d02-62c3-3256-97b0-c72bb20eeb1d"},{"identifier":"21214479-be8e-3ae9-9122-bb7aeb324f57","instanceIdentifier":"2347892","name":"Input Port Connection","source":{"id":"561eaa10-8c90-34d4-816d-a3186f2b9be0","type":"INPUT_PORT","groupId":"fa246d02-62c3-3256-97b0-c72bb20eeb1d","name":"Input Port","instanceIdentifier":"32874"},"destination":{"id":"934b5358-00b1-3ba8-b96a-5d72f72f1611","type":"PROCESSOR","groupId":"fa246d02-62c3-3256-97b0-c72bb20eeb1d","name":"Logger","comments":"","instanceIdentifier":"2222"},"labelIndex":1,"zIndex":0,"selectedRelationships":[""],"backPressureObjectThreshold":10000,"backPressureDataSizeThreshold":"1 GB","flowFileExpiration":"0 sec","prioritizers":[],"bends":[],"loadBalanceStrategy":"DO_NOT_LOAD_BALANCE","loadBalanceCompression":"DO_NOT_COMPRESS","componentType":"CONNECTION","groupIdentifier":"fa246d02-62c3-3256-97b0-c72bb20eeb1d"}],"labels":[],"funnels":[],"controllerServices":[],"variables":{},"defaultFlowFileExpiration":"0 sec","defaultBackPressureObjectThreshold":10000,"defaultBackPressureDataSizeThreshold":"1 GB","componentType":"PROCESS_GROUP","flowFileConcurrency":"UNBOUNDED","flowFileOutboundPolicy":"STREAM_WHEN_AVAILABLE","groupIdentifier":"81dc9bdb-52d0-3dc2-8036-dbd8313ed055"}],"remoteProcessGroups":[],"processors":[],"inputPorts":[],"outputPorts":[],"connections":[{"identifier":"3fdf5b1f-75b9-3ece-88c0-e7c5dee739f6","instanceIdentifier":"47823","name":"Output Port Connection","source":{"id":"6074c6aa-3488-33c2-9ddf-f2a7ca821aab","type":"OUTPUT_PORT","groupId":"d93591bd-f786-3e1e-8ee2-fca799911215","name":"Output Port","instanceIdentifier":"5555"},"destination":{"id":"561eaa10-8c90-34d4-816d-a3186f2b9be0","type":"INPUT_PORT","groupId":"fa246d02-62c3-3256-97b0-c72bb20eeb1d","name":"Input Port","instanceIdentifier":"32874"},"labelIndex":1,"zIndex":0,"selectedRelationships":[""],"backPressureObjectThreshold":10000,"backPressureDataSizeThreshold":"1 GB","flowFileExpiration":"0 sec","prioritizers":[],"bends":[],"loadBalanceStrategy":"DO_NOT_LOAD_BALANCE","loadBalanceCompression":"DO_NOT_COMPRESS","componentType":"CONNECTION","groupIdentifier":"81dc9bdb-52d0-3dc2-8036-dbd8313ed055"}],"labels":[],"funnels":[],"controllerServices":[],"variables":{},"defaultFlowFileExpiration":"0 sec","defaultBackPressureObjectThreshold":10000,"defaultBackPressureDataSizeThreshold":"1 GB","componentType":"PROCESS_GROUP","flowFileConcurrency":"UNBOUNDED","flowFileOutboundPolicy":"STREAM_WHEN_AVAILABLE"}} \ No newline at end of file diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/resources/conf/standard-flow.xml b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/resources/conf/standard-flow.xml deleted file mode 100644 index e019927e94..0000000000 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/resources/conf/standard-flow.xml +++ /dev/null @@ -1,196 +0,0 @@ - - - - 100 - - 1234 - Root Group - - - - - 4321 - Child Group - - - - - 0000 - Data Generator - - - - org.apache.nifi.processors.standard.LocalSelectionFileProcessor - 5 - 0 - 500 - true - - root.directory.path - C:/temp/youMayDeleteMe/lotsOf2MBFiles - - - keep.original - true - - - polling.interval.secs - 0 - - - - 1111 - Data Destroyer - - - - org.apache.nifi.processors.standard.TerminationFileProcessor - 5 - 0 - 20000 - true - - - 5555 - Output Port - - - - - 7777 - success - - 1 - 0 - - 0000 - 4321 - PROCESSOR - 1111 - 4321 - PROCESSOR - true - true - success - - - 8888 - to.other.group - - 1 - 0 - - 0000 - 4321 - PROCESSOR - 5555 - 4321 - OUTPUT_PORT - true - true - success - - - - 9999 - Second Group - - - - - 2222 - Logger - - - - org.apache.nifi.processors.standard.AttributeLoggerProcessor - 5 - 0 - 500 - true - - - 3333 - Second Terminator - - - - org.apache.nifi.processors.standard.TerminationFileProcessor - 5 - 0 - 20000 - true - - - 32874 - Input Port - - - - - 12378461946123976329 - success - - 1 - 0 - - 2222 - 9999 - PROCESSOR - 3333 - 9999 - PROCESSOR - true - true - success - - - 2347892 - Input Port Connection - - 1 - 0 - - 32874 - 9999 - INPUT_PORT - 2222 - 9999 - PROCESSOR - true - true - - - - - 47823 - Output Port Connection - - 1 - 0 - - 5555 - 4321 - OUTPUT_PORT - 32874 - 9999 - INPUT_PORT - true - true - - - - diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/resources/conf/taskConfig.xml b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/resources/conf/taskConfig.xml deleted file mode 100644 index dac771e14e..0000000000 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/resources/conf/taskConfig.xml +++ /dev/null @@ -1,17 +0,0 @@ - - - - \ No newline at end of file diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/resources/conf/termination-only.xml b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/resources/conf/termination-only.json similarity index 100% rename from nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/resources/conf/termination-only.xml rename to nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/resources/conf/termination-only.json diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/resources/flowcontrollertest.nifi.properties b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/resources/flowcontrollertest.nifi.properties index ebde513f07..484325558a 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/resources/flowcontrollertest.nifi.properties +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/resources/flowcontrollertest.nifi.properties @@ -14,7 +14,7 @@ # limitations under the License. # Core Properties # -nifi.flow.configuration.file=./target/flowcontrollertest/flow.xml.gz +nifi.flow.configuration.file=./target/flowcontrollertest/flow.json.gz nifi.flow.configuration.archive.dir=./target/flowcontrollertest/archive/ nifi.flowcontroller.autoResumeState=true nifi.flowcontroller.graceful.shutdown.period=10 sec diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/resources/nifi-with-remote.properties b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/resources/nifi-with-remote.properties index a4234d3dda..2ed45db046 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/resources/nifi-with-remote.properties +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/resources/nifi-with-remote.properties @@ -14,7 +14,7 @@ # limitations under the License. # Core Properties # -nifi.flow.configuration.file=./target/flow.xml.gz +nifi.flow.configuration.file=./target/flow.json.gz nifi.flow.configuration.archive.dir=./target/archive/ nifi.flowcontroller.autoResumeState=true nifi.flowcontroller.graceful.shutdown.period=10 sec diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/resources/nifi/fingerprint/flow1a.json b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/resources/nifi/fingerprint/flow1a.json new file mode 100644 index 0000000000..65dd294898 --- /dev/null +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/resources/nifi/fingerprint/flow1a.json @@ -0,0 +1,399 @@ +{ + "encodingVersion": { + "majorVersion": 2, + "minorVersion": 0 + }, + "maxTimerDrivenThreadCount": 10, + "maxEventDrivenThreadCount": 5, + "registries": [], + "parameterContexts": [ + { + "identifier": "5b2e44fe-fca6-3f19-995c-949bcf22b10d", + "instanceIdentifier": "8dafcb3b-82cb-30d4-b9e2-0174454e312e", + "name": "Parameter Context 1", + "parameters": [ + { + "name": "Parameter Name", + "description": "", + "sensitive": false, + "provided": false, + "value": "Parameter Value" + } + ], + "inheritedParameterContexts": [], + "description": "", + "componentType": "PARAMETER_CONTEXT" + } + ], + "parameterProviders": [], + "controllerServices": [], + "reportingTasks": [], + "templates": [], + "rootGroup": { + "identifier": "7c2de141-f9cc-3c73-89af-9caba6f946bc", + "instanceIdentifier": "e3909250-331d-420b-a9b3-cc54ad459401", + "name": "NiFi Flow", + "comments": "", + "position": { + "x": 0, + "y": 0 + }, + "processGroups": [ + { + "identifier": "30bbf1a5-3a58-354f-b2dd-1815ba8ace34", + "instanceIdentifier": "efeece05-3934-4298-a725-658eec116470", + "name": "Hello", + "comments": "", + "position": { + "x": 0, + "y": 0 + }, + "processGroups": [], + "remoteProcessGroups": [], + "processors": [ + { + "identifier": "e2809c14-dc1d-336a-85a5-4f65b45ffcfb", + "instanceIdentifier": "34caa1d6-cf14-4ec0-9f18-12859c37d55d", + "name": "LogAttribute", + "comments": "", + "position": { + "x": 0, + "y": 0 + }, + "type": "org.apache.nifi.processors.standard.LogAttribute", + "bundle": { + "group": "default", + "artifact": "unknown", + "version": "unversioned" + }, + "properties": {}, + "propertyDescriptors": {}, + "style": {}, + "annotationData": "", + "schedulingPeriod": "0 s", + "executionNode": "ALL", + "runDurationMillis": 0, + "concurrentlySchedulableTaskCount": 1, + "autoTerminatedRelationships": [], + "retryCount": 10, + "retriedRelationships": [], + "backoffMechanism": "PENALIZE_FLOWFILE", + "maxBackoffPeriod": "10 mins", + "componentType": "PROCESSOR", + "groupIdentifier": "30bbf1a5-3a58-354f-b2dd-1815ba8ace34" + } + ], + "inputPorts": [ + { + "identifier": "506632ef-38fe-3f1c-91b3-b1e631b46360", + "instanceIdentifier": "f37849cc-0177-1000-0000-00007d8cb44d", + "name": "Public-In", + "comments": "", + "position": { + "x": 0, + "y": 0 + }, + "type": "INPUT_PORT", + "concurrentlySchedulableTaskCount": 1, + "scheduledState": "ENABLED", + "allowRemoteAccess": true, + "componentType": "INPUT_PORT", + "groupIdentifier": "30bbf1a5-3a58-354f-b2dd-1815ba8ace34" + }, + { + "identifier": "1eb0d393-25fd-33bc-b478-466e5ba5a04f", + "instanceIdentifier": "91fae6d8-ad95-47cf-aa83-a6dfc742b7cb", + "name": "In", + "comments": "", + "position": { + "x": 0, + "y": 0 + }, + "type": "INPUT_PORT", + "concurrentlySchedulableTaskCount": 1, + "allowRemoteAccess": false, + "componentType": "INPUT_PORT", + "groupIdentifier": "30bbf1a5-3a58-354f-b2dd-1815ba8ace34" + } + ], + "outputPorts": [ + { + "identifier": "830a6c52-dcfb-3318-9f69-2419294d3d6c", + "instanceIdentifier": "fa543349-0178-1000-ffff-ffffd7f12ba4", + "name": "Public-Out", + "comments": "", + "position": { + "x": 0, + "y": 0 + }, + "type": "OUTPUT_PORT", + "concurrentlySchedulableTaskCount": 1, + "scheduledState": "ENABLED", + "allowRemoteAccess": true, + "componentType": "OUTPUT_PORT", + "groupIdentifier": "30bbf1a5-3a58-354f-b2dd-1815ba8ace34" + }, + { + "identifier": "ddd9a00b-e1b5-32c6-99a8-2883e4c6bf60", + "instanceIdentifier": "a65695bb-a938-4d3d-bf5d-f70a335268ec", + "name": "Out", + "comments": "", + "position": { + "x": 0, + "y": 0 + }, + "type": "OUTPUT_PORT", + "concurrentlySchedulableTaskCount": 1, + "scheduledState": "ENABLED", + "allowRemoteAccess": false, + "componentType": "OUTPUT_PORT", + "groupIdentifier": "30bbf1a5-3a58-354f-b2dd-1815ba8ace34" + } + ], + "connections": [ + { + "identifier": "d9558af4-bfe7-3f5b-ab16-f950e2e728db", + "instanceIdentifier": "908afab7-8777-4acf-a807-24f684f7aa9f", + "name": "", + "source": { + "id": "e2809c14-dc1d-336a-85a5-4f65b45ffcfb", + "type": "PROCESSOR", + "groupId": "30bbf1a5-3a58-354f-b2dd-1815ba8ace34", + "name": "LogAttribute", + "comments": "", + "instanceIdentifier": "34caa1d6-cf14-4ec0-9f18-12859c37d55d" + }, + "destination": { + "id": "ddd9a00b-e1b5-32c6-99a8-2883e4c6bf60", + "type": "OUTPUT_PORT", + "groupId": "30bbf1a5-3a58-354f-b2dd-1815ba8ace34", + "name": "Out", + "comments": "", + "instanceIdentifier": "a65695bb-a938-4d3d-bf5d-f70a335268ec" + }, + "labelIndex": 1, + "zIndex": 0, + "selectedRelationships": [ + "success" + ], + "backPressureObjectThreshold": 0, + "backPressureDataSizeThreshold": "1 GB", + "flowFileExpiration": "0 s", + "prioritizers": [], + "bends": [], + "loadBalanceStrategy": "DO_NOT_LOAD_BALANCE", + "loadBalanceCompression": "DO_NOT_COMPRESS", + "componentType": "CONNECTION", + "groupIdentifier": "30bbf1a5-3a58-354f-b2dd-1815ba8ace34" + }, + { + "identifier": "3c933742-c9d6-3f3a-9e69-549b7b1ba006", + "instanceIdentifier": "b25c3c8f-8dfe-4dda-950e-b6edfb6c99f4", + "name": "In Connection", + "source": { + "id": "1eb0d393-25fd-33bc-b478-466e5ba5a04f", + "type": "INPUT_PORT", + "groupId": "30bbf1a5-3a58-354f-b2dd-1815ba8ace34", + "name": "In", + "comments": "", + "instanceIdentifier": "91fae6d8-ad95-47cf-aa83-a6dfc742b7cb" + }, + "destination": { + "id": "e2809c14-dc1d-336a-85a5-4f65b45ffcfb", + "type": "PROCESSOR", + "groupId": "30bbf1a5-3a58-354f-b2dd-1815ba8ace34", + "name": "LogAttribute", + "comments": "", + "instanceIdentifier": "34caa1d6-cf14-4ec0-9f18-12859c37d55d" + }, + "labelIndex": 1, + "zIndex": 0, + "selectedRelationships": [ + "" + ], + "backPressureObjectThreshold": 0, + "backPressureDataSizeThreshold": "1 GB", + "flowFileExpiration": "0 s", + "prioritizers": [], + "bends": [], + "loadBalanceStrategy": "DO_NOT_LOAD_BALANCE", + "loadBalanceCompression": "DO_NOT_COMPRESS", + "componentType": "CONNECTION", + "groupIdentifier": "30bbf1a5-3a58-354f-b2dd-1815ba8ace34" + } + ], + "labels": [], + "funnels": [], + "controllerServices": [], + "variables": {}, + "defaultFlowFileExpiration": "0 sec", + "defaultBackPressureObjectThreshold": 10000, + "defaultBackPressureDataSizeThreshold": "1 GB", + "componentType": "PROCESS_GROUP", + "flowFileOutboundPolicy": "STREAM_WHEN_AVAILABLE", + "flowFileConcurrency": "UNBOUNDED", + "groupIdentifier": "7c2de141-f9cc-3c73-89af-9caba6f946bc" + } + ], + "remoteProcessGroups": [], + "processors": [ + { + "identifier": "761ffb7b-ab7a-3e99-9b10-79587718355c", + "instanceIdentifier": "d89ada5d-35fb-44ff-83f1-4cc00b48b2df", + "name": "GenerateFlowFile", + "comments": "", + "position": { + "x": 0, + "y": 0 + }, + "type": "org.apache.nifi.processors.standard.GenerateFlowFile", + "bundle": { + "group": "org.apache.nifi", + "artifact": "nifi-standard-nar", + "version": "1.3.0-SNAPSHOT" + }, + "properties": { + "generate-ff-custom-text": "enc{null}", + "file.size": "enc{null}" + }, + "propertyDescriptors": {}, + "style": {}, + "annotationData": "", + "schedulingPeriod": "0 sec", + "schedulingStrategy": "TIMER_DRIVEN", + "executionNode": "ALL", + "penaltyDuration": "30 sec", + "yieldDuration": "1 sec", + "bulletinLevel": "WARN", + "runDurationMillis": 0, + "concurrentlySchedulableTaskCount": 1, + "autoTerminatedRelationships": [], + "scheduledState": "ENABLED", + "retryCount": 10, + "retriedRelationships": [], + "backoffMechanism": "PENALIZE_FLOWFILE", + "maxBackoffPeriod": "10 mins", + "componentType": "PROCESSOR", + "groupIdentifier": "7c2de141-f9cc-3c73-89af-9caba6f946bc" + }, + { + "identifier": "ffa852e4-5291-31dd-bdcc-dc82a531bf9e", + "instanceIdentifier": "e520797a-dddb-4930-9034-2092d3e816a6", + "name": "LogAttribute", + "comments": "", + "position": { + "x": 0, + "y": 0 + }, + "type": "org.apache.nifi.processors.standard.LogAttribute", + "bundle": { + "group": "default", + "artifact": "unknown", + "version": "unversioned" + }, + "properties": {}, + "propertyDescriptors": {}, + "style": {}, + "annotationData": "", + "executionNode": "ALL", + "runDurationMillis": 0, + "concurrentlySchedulableTaskCount": 1, + "autoTerminatedRelationships": [ + "success" + ], + "scheduledState": "ENABLED", + "retryCount": 10, + "retriedRelationships": [], + "backoffMechanism": "PENALIZE_FLOWFILE", + "maxBackoffPeriod": "10 mins", + "componentType": "PROCESSOR", + "groupIdentifier": "7c2de141-f9cc-3c73-89af-9caba6f946bc" + } + ], + "inputPorts": [], + "outputPorts": [], + "connections": [ + { + "identifier": "f9b8ad05-d347-3060-8d55-dd7697930841", + "instanceIdentifier": "03f4f5bf-baa5-47fa-9b1a-b77860d67d4f", + "name": "", + "source": { + "id": "761ffb7b-ab7a-3e99-9b10-79587718355c", + "type": "PROCESSOR", + "groupId": "7c2de141-f9cc-3c73-89af-9caba6f946bc", + "name": "GenerateFlowFile", + "comments": "", + "instanceIdentifier": "d89ada5d-35fb-44ff-83f1-4cc00b48b2df" + }, + "destination": { + "id": "1eb0d393-25fd-33bc-b478-466e5ba5a04f", + "type": "INPUT_PORT", + "groupId": "30bbf1a5-3a58-354f-b2dd-1815ba8ace34", + "name": "In", + "comments": "", + "instanceIdentifier": "91fae6d8-ad95-47cf-aa83-a6dfc742b7cb" + }, + "labelIndex": 1, + "zIndex": 0, + "selectedRelationships": [ + "success" + ], + "backPressureObjectThreshold": 0, + "backPressureDataSizeThreshold": "1 GB", + "flowFileExpiration": "0 s", + "prioritizers": [], + "bends": [], + "loadBalanceStrategy": "DO_NOT_LOAD_BALANCE", + "loadBalanceCompression": "DO_NOT_COMPRESS", + "componentType": "CONNECTION", + "groupIdentifier": "7c2de141-f9cc-3c73-89af-9caba6f946bc" + }, + { + "identifier": "64c3117c-752c-3ffb-b19a-455c9f3fcbd8", + "instanceIdentifier": "5bd05300-f03d-4511-a13f-6a36afe2bcc5", + "name": "", + "source": { + "id": "ddd9a00b-e1b5-32c6-99a8-2883e4c6bf60", + "type": "OUTPUT_PORT", + "groupId": "30bbf1a5-3a58-354f-b2dd-1815ba8ace34", + "name": "Out", + "comments": "", + "instanceIdentifier": "a65695bb-a938-4d3d-bf5d-f70a335268ec" + }, + "destination": { + "id": "ffa852e4-5291-31dd-bdcc-dc82a531bf9e", + "type": "PROCESSOR", + "groupId": "7c2de141-f9cc-3c73-89af-9caba6f946bc", + "name": "LogAttribute", + "comments": "", + "instanceIdentifier": "e520797a-dddb-4930-9034-2092d3e816a6" + }, + "labelIndex": 1, + "zIndex": 0, + "selectedRelationships": [ + "" + ], + "backPressureObjectThreshold": 0, + "backPressureDataSizeThreshold": "1 GB", + "flowFileExpiration": "0 s", + "prioritizers": [], + "bends": [], + "loadBalanceStrategy": "DO_NOT_LOAD_BALANCE", + "loadBalanceCompression": "DO_NOT_COMPRESS", + "componentType": "CONNECTION", + "groupIdentifier": "7c2de141-f9cc-3c73-89af-9caba6f946bc" + } + ], + "labels": [], + "funnels": [], + "controllerServices": [], + "variables": {}, + "parameterContextName": "Parameter Context 1", + "defaultFlowFileExpiration": "0 sec", + "defaultBackPressureObjectThreshold": 10000, + "defaultBackPressureDataSizeThreshold": "1 GB", + "componentType": "PROCESS_GROUP", + "flowFileOutboundPolicy": "STREAM_WHEN_AVAILABLE", + "flowFileConcurrency": "UNBOUNDED" + } +} \ No newline at end of file diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/resources/nifi/fingerprint/flow1a.xml b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/resources/nifi/fingerprint/flow1a.xml deleted file mode 100644 index 422aa7ffae..0000000000 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/resources/nifi/fingerprint/flow1a.xml +++ /dev/null @@ -1,215 +0,0 @@ - - - - 15 - - - 8dafcb3b-82cb-30d4-b9e2-0174454e312e - Parameter Context 1 - - - Parameter Name - - false - Parameter Value - - - - - e3909250-331d-420b-a9b3-cc54ad459401 - NiFi Flow - - - - org.apache.nifi.processors.standard.LogAttribute - 1 - 0 s - false - false - - success - - - efeece05-3934-4298-a725-658eec116470 - Hello - - - - UNBOUNDED - STREAM_WHEN_AVAILABLE - - 34caa1d6-cf14-4ec0-9f18-12859c37d55d - LogAttribute - - - - org.apache.nifi.processors.standard.LogAttribute - 1 - 0 s - false - false - - - - 91fae6d8-ad95-47cf-aa83-a6dfc742b7cb - In - - - - false - - - f37849cc-0177-1000-0000-00007d8cb44d - Public-In - - - STOPPED - 1 - true - user1 - group1 - - - a65695bb-a938-4d3d-bf5d-f70a335268ec - Out - - - - false - - - fa543349-0178-1000-ffff-ffffd7f12ba4 - Public-Out - - - STOPPED - 1 - true - user1 - group1 - - - b25c3c8f-8dfe-4dda-950e-b6edfb6c99f4 - In Connection - - 1 - 0 - - - this is a comment - org.apache.nifi.processors.standard.GenerateFlowFile - - org.apache.nifi - nifi-standard-nar - 1.3.0-SNAPSHOT - - 1 - 0 sec - 30 sec - 1 sec - WARN - false - DISABLED - TIMER_DRIVEN - ALL - 0 - - file.size - 5 - - - generate-ff-custom-text - #{Custom_Text} - - - - - e520797a-dddb-4930-9034-2092d3e816a6 - LogAttribute - - - - org.apache.nifi.processors.standard.LogAttribute - 1 - 0 s - false - false - - success - - - efeece05-3934-4298-a725-658eec116470 - Hello2 - - - - UNBOUNDED - STREAM_WHEN_AVAILABLE - - 34caa1d6-cf14-4ec0-9f18-12859c37d55d - LogAttribute - - - - org.apache.nifi.processors.standard.LogAttribute - 1 - 0 s - false - false - - - - 91fae6d8-ad95-47cf-aa83-a6dfc742b7cb - In - - - foobar - false - - - f37849cc-0177-1000-0000-00007d8cb44d - Public-In - - Public Port for Site to Site - STOPPED - 1 - true - user1 - group1 - - - a65695bb-a938-4d3d-bf5d-f70a335268ec - Out - - - - false - - - fa543349-0178-1000-ffff-ffffd7f12ba4 - Public-Out - - - RUNNING - 1 - true - user1 - group1 - - - b25c3c8f-8dfe-4dda-950e-b6edfb6c99f4 - In Connection - - 1 - 0 - - - org.apache.nifi.processors.standard.LogAttribute - 1 - 0 s - false - false - - success - - - efeece05-3934-4298-a725-658eec116470 - Hello - - - - UNBOUNDED - STREAM_WHEN_AVAILABLE - - 34caa1d6-cf14-4ec0-9f18-12859c37d55d - LogAttribute - - - - org.apache.nifi.processors.standard.LogAttribute - 1 - 0 s - false - false - - - - 91fae6d8-ad95-47cf-aa83-a6dfc742b7cb - In - - - - false - - - a65695bb-a938-4d3d-bf5d-f70a335268ec - Out - - - - false - - - b25c3c8f-8dfe-4dda-950e-b6edfb6c99f4 - In - - 1 - 0 - - - org.apache.nifi.processors.standard.GenerateFlowFile - - org.apache.nifi - nifi-standard-nar - 1.0 - - 1 - 0 sec - 30 sec - 1 sec - WARN - false - STOPPED - TIMER_DRIVEN - ALL - 0 - - file.size - 5 - - - - - diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/resources/nifi/fingerprint/flow3-with-bundle-2.json b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/resources/nifi/fingerprint/flow3-with-bundle-2.json new file mode 100644 index 0000000000..b49561d69d --- /dev/null +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/resources/nifi/fingerprint/flow3-with-bundle-2.json @@ -0,0 +1,79 @@ +{ + "encodingVersion": { + "majorVersion": 2, + "minorVersion": 0 + }, + "maxTimerDrivenThreadCount": 10, + "maxEventDrivenThreadCount": 5, + "registries": [], + "parameterContexts": [], + "parameterProviders": [], + "controllerServices": [], + "reportingTasks": [], + "templates": [], + "rootGroup": { + "identifier": "7c2de141-f9cc-3c73-89af-9caba6f946bc", + "instanceIdentifier": "e3909250-331d-420b-a9b3-cc54ad459401", + "name": "NiFi Flow", + "comments": "", + "position": { + "x": 0, + "y": 0 + }, + "processGroups": [], + "remoteProcessGroups": [], + "processors": [ + { + "identifier": "761ffb7b-ab7a-3e99-9b10-79587718355c", + "instanceIdentifier": "d89ada5d-35fb-44ff-83f1-4cc00b48b2df", + "name": "GenerateFlowFile", + "comments": "", + "position": { + "x": 0, + "y": 0 + }, + "type": "org.apache.nifi.processors.standard.GenerateFlowFile", + "bundle": { + "group": "org.apache.nifi", + "artifact": "nifi-standard-nar", + "version": "2.0" + }, + "properties": { + "file.size": "enc{null}" + }, + "propertyDescriptors": {}, + "style": {}, + "annotationData": "", + "schedulingPeriod": "0 sec", + "schedulingStrategy": "TIMER_DRIVEN", + "executionNode": "ALL", + "penaltyDuration": "30 sec", + "yieldDuration": "1 sec", + "bulletinLevel": "WARN", + "runDurationMillis": 0, + "concurrentlySchedulableTaskCount": 1, + "autoTerminatedRelationships": [], + "scheduledState": "ENABLED", + "retryCount": 10, + "retriedRelationships": [], + "backoffMechanism": "PENALIZE_FLOWFILE", + "maxBackoffPeriod": "10 mins", + "componentType": "PROCESSOR", + "groupIdentifier": "7c2de141-f9cc-3c73-89af-9caba6f946bc" + } + ], + "inputPorts": [], + "outputPorts": [], + "connections": [], + "labels": [], + "funnels": [], + "controllerServices": [], + "variables": {}, + "defaultFlowFileExpiration": "0 sec", + "defaultBackPressureObjectThreshold": 10000, + "defaultBackPressureDataSizeThreshold": "1 GB", + "componentType": "PROCESS_GROUP", + "flowFileConcurrency": "UNBOUNDED", + "flowFileOutboundPolicy": "STREAM_WHEN_AVAILABLE" + } +} \ No newline at end of file diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/resources/nifi/fingerprint/flow3-with-bundle-2.xml b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/resources/nifi/fingerprint/flow3-with-bundle-2.xml deleted file mode 100644 index bfce55029d..0000000000 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/resources/nifi/fingerprint/flow3-with-bundle-2.xml +++ /dev/null @@ -1,55 +0,0 @@ - - - - 15 - - e3909250-331d-420b-a9b3-cc54ad459401 - NiFi Flow - - - - org.apache.nifi.processors.standard.GenerateFlowFile - - org.apache.nifi - nifi-standard-nar - 2.0 - - 1 - 0 sec - 30 sec - 1 sec - WARN - false - STOPPED - TIMER_DRIVEN - ALL - 0 - - file.size - 5 - - - - - diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/resources/nifi/fingerprint/flow3-with-missing-bundle.json b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/resources/nifi/fingerprint/flow3-with-missing-bundle.json new file mode 100644 index 0000000000..6dbf597c30 --- /dev/null +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/resources/nifi/fingerprint/flow3-with-missing-bundle.json @@ -0,0 +1,79 @@ +{ + "encodingVersion": { + "majorVersion": 2, + "minorVersion": 0 + }, + "maxTimerDrivenThreadCount": 10, + "maxEventDrivenThreadCount": 5, + "registries": [], + "parameterContexts": [], + "parameterProviders": [], + "controllerServices": [], + "reportingTasks": [], + "templates": [], + "rootGroup": { + "identifier": "7c2de141-f9cc-3c73-89af-9caba6f946bc", + "instanceIdentifier": "e3909250-331d-420b-a9b3-cc54ad459401", + "name": "NiFi Flow", + "comments": "", + "position": { + "x": 0, + "y": 0 + }, + "processGroups": [], + "remoteProcessGroups": [], + "processors": [ + { + "identifier": "761ffb7b-ab7a-3e99-9b10-79587718355c", + "instanceIdentifier": "d89ada5d-35fb-44ff-83f1-4cc00b48b2df", + "name": "GenerateFlowFile", + "comments": "", + "position": { + "x": 0, + "y": 0 + }, + "type": "org.apache.nifi.processors.standard.GenerateFlowFile", + "bundle": { + "group": "missing", + "artifact": "missing", + "version": "missing" + }, + "properties": { + "file.size": "enc{null}" + }, + "propertyDescriptors": {}, + "style": {}, + "annotationData": "", + "schedulingPeriod": "0 sec", + "schedulingStrategy": "TIMER_DRIVEN", + "executionNode": "ALL", + "penaltyDuration": "30 sec", + "yieldDuration": "1 sec", + "bulletinLevel": "WARN", + "runDurationMillis": 0, + "concurrentlySchedulableTaskCount": 1, + "autoTerminatedRelationships": [], + "scheduledState": "ENABLED", + "retryCount": 10, + "retriedRelationships": [], + "backoffMechanism": "PENALIZE_FLOWFILE", + "maxBackoffPeriod": "10 mins", + "componentType": "PROCESSOR", + "groupIdentifier": "7c2de141-f9cc-3c73-89af-9caba6f946bc" + } + ], + "inputPorts": [], + "outputPorts": [], + "connections": [], + "labels": [], + "funnels": [], + "controllerServices": [], + "variables": {}, + "defaultFlowFileExpiration": "0 sec", + "defaultBackPressureObjectThreshold": 10000, + "defaultBackPressureDataSizeThreshold": "1 GB", + "componentType": "PROCESS_GROUP", + "flowFileConcurrency": "UNBOUNDED", + "flowFileOutboundPolicy": "STREAM_WHEN_AVAILABLE" + } +} \ No newline at end of file diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/resources/nifi/fingerprint/flow3-with-missing-bundle.xml b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/resources/nifi/fingerprint/flow3-with-missing-bundle.xml deleted file mode 100644 index 926efb93cc..0000000000 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/resources/nifi/fingerprint/flow3-with-missing-bundle.xml +++ /dev/null @@ -1,55 +0,0 @@ - - - - 15 - - e3909250-331d-420b-a9b3-cc54ad459401 - NiFi Flow - - - - org.apache.nifi.processors.standard.GenerateFlowFile - - missing - missing - missing - - 1 - 0 sec - 30 sec - 1 sec - WARN - false - STOPPED - TIMER_DRIVEN - ALL - 0 - - file.size - 5 - - - - - diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/resources/nifi/fingerprint/flow3-with-no-bundle.json b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/resources/nifi/fingerprint/flow3-with-no-bundle.json new file mode 100644 index 0000000000..4e64d90056 --- /dev/null +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/resources/nifi/fingerprint/flow3-with-no-bundle.json @@ -0,0 +1,79 @@ +{ + "encodingVersion": { + "majorVersion": 2, + "minorVersion": 0 + }, + "maxTimerDrivenThreadCount": 10, + "maxEventDrivenThreadCount": 5, + "registries": [], + "parameterContexts": [], + "parameterProviders": [], + "controllerServices": [], + "reportingTasks": [], + "templates": [], + "rootGroup": { + "identifier": "7c2de141-f9cc-3c73-89af-9caba6f946bc", + "instanceIdentifier": "e3909250-331d-420b-a9b3-cc54ad459401", + "name": "NiFi Flow", + "comments": "", + "position": { + "x": 0, + "y": 0 + }, + "processGroups": [], + "remoteProcessGroups": [], + "processors": [ + { + "identifier": "761ffb7b-ab7a-3e99-9b10-79587718355c", + "instanceIdentifier": "d89ada5d-35fb-44ff-83f1-4cc00b48b2df", + "name": "GenerateFlowFile", + "comments": "", + "position": { + "x": 0, + "y": 0 + }, + "type": "org.apache.nifi.processors.standard.GenerateFlowFile", + "bundle": { + "group": "default", + "artifact": "unknown", + "version": "unversioned" + }, + "properties": { + "file.size": "enc{null}" + }, + "propertyDescriptors": {}, + "style": {}, + "annotationData": "", + "schedulingPeriod": "0 sec", + "schedulingStrategy": "TIMER_DRIVEN", + "executionNode": "ALL", + "penaltyDuration": "30 sec", + "yieldDuration": "1 sec", + "bulletinLevel": "WARN", + "runDurationMillis": 0, + "concurrentlySchedulableTaskCount": 1, + "autoTerminatedRelationships": [], + "scheduledState": "ENABLED", + "retryCount": 10, + "retriedRelationships": [], + "backoffMechanism": "PENALIZE_FLOWFILE", + "maxBackoffPeriod": "10 mins", + "componentType": "PROCESSOR", + "groupIdentifier": "7c2de141-f9cc-3c73-89af-9caba6f946bc" + } + ], + "inputPorts": [], + "outputPorts": [], + "connections": [], + "labels": [], + "funnels": [], + "controllerServices": [], + "variables": {}, + "defaultFlowFileExpiration": "0 sec", + "defaultBackPressureObjectThreshold": 10000, + "defaultBackPressureDataSizeThreshold": "1 GB", + "componentType": "PROCESS_GROUP", + "flowFileConcurrency": "UNBOUNDED", + "flowFileOutboundPolicy": "STREAM_WHEN_AVAILABLE" + } +} \ No newline at end of file diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/resources/nifi/fingerprint/flow3-with-no-bundle.xml b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/resources/nifi/fingerprint/flow3-with-no-bundle.xml deleted file mode 100644 index a451a07975..0000000000 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/resources/nifi/fingerprint/flow3-with-no-bundle.xml +++ /dev/null @@ -1,50 +0,0 @@ - - - - 15 - - e3909250-331d-420b-a9b3-cc54ad459401 - NiFi Flow - - - - org.apache.nifi.processors.standard.GenerateFlowFile - 1 - 0 sec - 30 sec - 1 sec - WARN - false - STOPPED - TIMER_DRIVEN - ALL - 0 - - file.size - 5 - - - - - diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/resources/nifi/fingerprint/flow4-with-different-bundle.json b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/resources/nifi/fingerprint/flow4-with-different-bundle.json new file mode 100644 index 0000000000..5917498b10 --- /dev/null +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/resources/nifi/fingerprint/flow4-with-different-bundle.json @@ -0,0 +1,79 @@ +{ + "encodingVersion": { + "majorVersion": 2, + "minorVersion": 0 + }, + "maxTimerDrivenThreadCount": 10, + "maxEventDrivenThreadCount": 5, + "registries": [], + "parameterContexts": [], + "parameterProviders": [], + "controllerServices": [], + "reportingTasks": [], + "templates": [], + "rootGroup": { + "identifier": "7c2de141-f9cc-3c73-89af-9caba6f946bc", + "instanceIdentifier": "e3909250-331d-420b-a9b3-cc54ad459401", + "name": "NiFi Flow", + "comments": "", + "position": { + "x": 0, + "y": 0 + }, + "processGroups": [], + "remoteProcessGroups": [], + "processors": [ + { + "identifier": "761ffb7b-ab7a-3e99-9b10-79587718355c", + "instanceIdentifier": "d89ada5d-35fb-44ff-83f1-4cc00b48b2df", + "name": "GenerateFlowFile", + "comments": "", + "position": { + "x": 0, + "y": 0 + }, + "type": "org.apache.nifi.controller.service.mock.DummyProcessor", + "bundle": { + "group": "default", + "artifact": "system", + "version": "v2" + }, + "properties": { + "file.size": "5" + }, + "propertyDescriptors": {}, + "style": {}, + "annotationData": "", + "schedulingPeriod": "0 sec", + "schedulingStrategy": "TIMER_DRIVEN", + "executionNode": "ALL", + "penaltyDuration": "30 sec", + "yieldDuration": "1 sec", + "bulletinLevel": "WARN", + "runDurationMillis": 0, + "concurrentlySchedulableTaskCount": 1, + "autoTerminatedRelationships": [], + "scheduledState": "ENABLED", + "retryCount": 10, + "retriedRelationships": [], + "backoffMechanism": "PENALIZE_FLOWFILE", + "maxBackoffPeriod": "10 mins", + "componentType": "PROCESSOR", + "groupIdentifier": "7c2de141-f9cc-3c73-89af-9caba6f946bc" + } + ], + "inputPorts": [], + "outputPorts": [], + "connections": [], + "labels": [], + "funnels": [], + "controllerServices": [], + "variables": {}, + "defaultFlowFileExpiration": "0 sec", + "defaultBackPressureObjectThreshold": 10000, + "defaultBackPressureDataSizeThreshold": "1 GB", + "componentType": "PROCESS_GROUP", + "flowFileConcurrency": "UNBOUNDED", + "flowFileOutboundPolicy": "STREAM_WHEN_AVAILABLE" + } +} \ No newline at end of file diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/resources/nifi/fingerprint/flow4-with-different-bundle.xml b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/resources/nifi/fingerprint/flow4-with-different-bundle.xml deleted file mode 100644 index 315a97c8f8..0000000000 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/resources/nifi/fingerprint/flow4-with-different-bundle.xml +++ /dev/null @@ -1,55 +0,0 @@ - - - - 15 - - e3909250-331d-420b-a9b3-cc54ad459401 - NiFi Flow - - - - org.apache.nifi.controller.service.mock.DummyProcessor - - default - system - v2 - - 1 - 0 sec - 30 sec - 1 sec - WARN - false - STOPPED - TIMER_DRIVEN - ALL - 0 - - file.size - 5 - - - - - diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/resources/nifi/fingerprint/flow4.json b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/resources/nifi/fingerprint/flow4.json new file mode 100644 index 0000000000..e3600bd1b0 --- /dev/null +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/resources/nifi/fingerprint/flow4.json @@ -0,0 +1,79 @@ +{ + "encodingVersion": { + "majorVersion": 2, + "minorVersion": 0 + }, + "maxTimerDrivenThreadCount": 10, + "maxEventDrivenThreadCount": 5, + "registries": [], + "parameterContexts": [], + + "controllerServices": [], + "reportingTasks": [], + "templates": [], + "rootGroup": { + "identifier": "7c2de141-f9cc-3c73-89af-9caba6f946bc", + "instanceIdentifier": "e3909250-331d-420b-a9b3-cc54ad459401", + "name": "NiFi Flow", + "comments": "", + "position": { + "x": 0, + "y": 0 + }, + "processGroups": [], + "remoteProcessGroups": [], + "processors": [ + { + "identifier": "761ffb7b-ab7a-3e99-9b10-79587718355c", + "instanceIdentifier": "d89ada5d-35fb-44ff-83f1-4cc00b48b2df", + "name": "GenerateFlowFile", + "comments": "", + "position": { + "x": 0, + "y": 0 + }, + "type": "org.apache.nifi.controller.service.mock.DummyProcessor", + "bundle": { + "group": "default", + "artifact": "system", + "version": "unversioned" + }, + "properties": { + "file.size": "5" + }, + "propertyDescriptors": {}, + "style": {}, + "annotationData": "", + "schedulingPeriod": "0 sec", + "schedulingStrategy": "TIMER_DRIVEN", + "executionNode": "ALL", + "penaltyDuration": "30 sec", + "yieldDuration": "1 sec", + "bulletinLevel": "WARN", + "runDurationMillis": 0, + "concurrentlySchedulableTaskCount": 1, + "autoTerminatedRelationships": [], + "scheduledState": "ENABLED", + "retryCount": 10, + "retriedRelationships": [], + "backoffMechanism": "PENALIZE_FLOWFILE", + "maxBackoffPeriod": "10 mins", + "componentType": "PROCESSOR", + "groupIdentifier": "7c2de141-f9cc-3c73-89af-9caba6f946bc" + } + ], + "inputPorts": [], + "outputPorts": [], + "connections": [], + "labels": [], + "funnels": [], + "controllerServices": [], + "variables": {}, + "defaultFlowFileExpiration": "0 sec", + "defaultBackPressureObjectThreshold": 10000, + "defaultBackPressureDataSizeThreshold": "1 GB", + "componentType": "PROCESS_GROUP", + "flowFileConcurrency": "UNBOUNDED", + "flowFileOutboundPolicy": "STREAM_WHEN_AVAILABLE" + } +} \ No newline at end of file diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/resources/nifi/fingerprint/flow4.xml b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/resources/nifi/fingerprint/flow4.xml deleted file mode 100644 index d02080a0d0..0000000000 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/resources/nifi/fingerprint/flow4.xml +++ /dev/null @@ -1,55 +0,0 @@ - - - - 15 - - e3909250-331d-420b-a9b3-cc54ad459401 - NiFi Flow - - - - org.apache.nifi.controller.service.mock.DummyProcessor - - default - system - unversioned - - 1 - 0 sec - 30 sec - 1 sec - WARN - false - STOPPED - TIMER_DRIVEN - ALL - 0 - - file.size - 5 - - - - - diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/resources/standardflowserializertest.nifi.properties b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/resources/standardflowserializertest.nifi.properties index 345dffb3b0..1a5515bd99 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/resources/standardflowserializertest.nifi.properties +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/resources/standardflowserializertest.nifi.properties @@ -14,7 +14,7 @@ # limitations under the License. # Core Properties # -nifi.flow.configuration.file=./target/standardflowserializertest/flow.xml.gz +nifi.flow.configuration.file=./target/standardflowserializertest/flow.json.gz nifi.flow.configuration.archive.dir=./target/standardflowserializertest/archive/ nifi.flowcontroller.autoResumeState=true nifi.flowcontroller.graceful.shutdown.period=10 sec diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/resources/standardflowsynchronizerspec.nifi.properties b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/resources/standardflowsynchronizerspec.nifi.properties index adedc66840..303c2e9cca 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/resources/standardflowsynchronizerspec.nifi.properties +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/resources/standardflowsynchronizerspec.nifi.properties @@ -14,7 +14,7 @@ # limitations under the License. # Core Properties # -nifi.flow.configuration.file=./target/standardflowsynchronizerspec/flow.xml.gz +nifi.flow.configuration.file=./target/standardflowsynchronizerspec/flow.json.gz nifi.flow.configuration.archive.dir=./target/standardflowsynchronizerspec/archive/ nifi.flowcontroller.autoResumeState=true nifi.flowcontroller.graceful.shutdown.period=10 sec diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/resources/standardprocessschedulertest.nifi.properties b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/resources/standardprocessschedulertest.nifi.properties index 2c8927f2a7..5bb0baf925 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/resources/standardprocessschedulertest.nifi.properties +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/resources/standardprocessschedulertest.nifi.properties @@ -14,7 +14,7 @@ # limitations under the License. # Core Properties # -nifi.flow.configuration.file=./target/standardprocessschedulertest/flow.xml.gz +nifi.flow.configuration.file=./target/standardprocessschedulertest/flow.json.gz nifi.flow.configuration.archive.dir=./target/standardprocessschedulertest/archive/ nifi.flowcontroller.autoResumeState=true nifi.flowcontroller.graceful.shutdown.period=10 sec diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-nar-loading-utils/src/test/resources/conf/nifi.nar_with_native_lib.properties b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-nar-loading-utils/src/test/resources/conf/nifi.nar_with_native_lib.properties index 9e36d6bcad..fa3be8a5ac 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-nar-loading-utils/src/test/resources/conf/nifi.nar_with_native_lib.properties +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-nar-loading-utils/src/test/resources/conf/nifi.nar_with_native_lib.properties @@ -14,7 +14,7 @@ # limitations under the License. # Core Properties # -nifi.flow.configuration.file=./target/flow.xml.gz +nifi.flow.configuration.file=./target/flow.json.gz nifi.flow.configuration.archive.dir=./target/archive/ nifi.flowcontroller.autoResumeState=true nifi.flowcontroller.graceful.shutdown.period=10 sec diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-nar-loading-utils/src/test/resources/conf/nifi.nar_without_native_lib.properties b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-nar-loading-utils/src/test/resources/conf/nifi.nar_without_native_lib.properties index a0d7966223..b018334b53 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-nar-loading-utils/src/test/resources/conf/nifi.nar_without_native_lib.properties +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-nar-loading-utils/src/test/resources/conf/nifi.nar_without_native_lib.properties @@ -14,7 +14,7 @@ # limitations under the License. # Core Properties # -nifi.flow.configuration.file=./target/flow.xml.gz +nifi.flow.configuration.file=./target/flow.json.gz nifi.flow.configuration.archive.dir=./target/archive/ nifi.flowcontroller.autoResumeState=true nifi.flowcontroller.graceful.shutdown.period=10 sec diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-nar-loading-utils/src/test/resources/conf/nifi.properties b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-nar-loading-utils/src/test/resources/conf/nifi.properties index e95c89350d..534e47b729 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-nar-loading-utils/src/test/resources/conf/nifi.properties +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-nar-loading-utils/src/test/resources/conf/nifi.properties @@ -14,7 +14,7 @@ # limitations under the License. # Core Properties # -nifi.flow.configuration.file=./target/flow.xml.gz +nifi.flow.configuration.file=./target/flow.json.gz nifi.flow.configuration.archive.dir=./target/archive/ nifi.flowcontroller.autoResumeState=true nifi.flowcontroller.graceful.shutdown.period=10 sec diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-nar-utils/src/test/resources/NarUnpacker/conf/nifi.properties b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-nar-utils/src/test/resources/NarUnpacker/conf/nifi.properties index 508d78565a..4183ce1b9f 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-nar-utils/src/test/resources/NarUnpacker/conf/nifi.properties +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-nar-utils/src/test/resources/NarUnpacker/conf/nifi.properties @@ -14,7 +14,7 @@ # limitations under the License. # Core Properties # -nifi.flow.configuration.file=./target/flow.xml.gz +nifi.flow.configuration.file=./target/flow.json.gz nifi.flow.configuration.archive.dir=./target/archive/ nifi.flowcontroller.autoResumeState=true nifi.flowcontroller.graceful.shutdown.period=10 sec diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-nar-utils/src/test/resources/nifi.properties b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-nar-utils/src/test/resources/nifi.properties index 62af4eae68..2a63a02187 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-nar-utils/src/test/resources/nifi.properties +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-nar-utils/src/test/resources/nifi.properties @@ -14,7 +14,7 @@ # limitations under the License. # Core Properties # -nifi.flow.configuration.file=./conf/flow.xml.gz +nifi.flow.configuration.file=./conf/flow.json.gz nifi.flow.configuration.archive.enabled=true nifi.flow.configuration.archive.dir=./conf/archive/ nifi.flow.configuration.archive.max.time=30 days diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-headless-server/src/main/java/org/apache/nifi/headless/HeadlessNiFiServer.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-headless-server/src/main/java/org/apache/nifi/headless/HeadlessNiFiServer.java index b94e51e159..592b708472 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-headless-server/src/main/java/org/apache/nifi/headless/HeadlessNiFiServer.java +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-headless-server/src/main/java/org/apache/nifi/headless/HeadlessNiFiServer.java @@ -30,7 +30,6 @@ import org.apache.nifi.authorization.exception.AuthorizerDestructionException; import org.apache.nifi.bundle.Bundle; import org.apache.nifi.controller.DecommissionTask; import org.apache.nifi.controller.FlowController; -import org.apache.nifi.controller.FlowSerializationStrategy; import org.apache.nifi.controller.StandardFlowService; import org.apache.nifi.controller.flow.FlowManager; import org.apache.nifi.controller.repository.FlowFileEventRepository; @@ -150,8 +149,7 @@ public class HeadlessNiFiServer implements NiFiServer { flowController, props, null, // revision manager - authorizer, - getFlowSerializationStrategy()); + authorizer); diagnosticsFactory = new BootstrapDiagnosticsFactory(); ((BootstrapDiagnosticsFactory) diagnosticsFactory).setFlowController(flowController); @@ -262,10 +260,6 @@ public class HeadlessNiFiServer implements NiFiServer { return ExtensionManagerHolder.getExtensionManager().getBundles(bundleClass); } - protected FlowSerializationStrategy getFlowSerializationStrategy() { - return FlowSerializationStrategy.WRITE_XML_ONLY; - } - private static class ThreadDumpDiagnosticsFactory implements DiagnosticsFactory { @Override public DiagnosticsDump create(final boolean verbose) { diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-nar-utils/src/test/resources/nifi.properties b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-nar-utils/src/test/resources/nifi.properties index 557a2e8c41..f11eaba2c3 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-nar-utils/src/test/resources/nifi.properties +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-nar-utils/src/test/resources/nifi.properties @@ -14,7 +14,7 @@ # limitations under the License. # Core Properties # -nifi.flow.configuration.file=./conf/flow.xml.gz +nifi.flow.configuration.file=./conf/flow.json.gz nifi.flow.configuration.archive.enabled=true nifi.flow.configuration.archive.dir=./conf/archive/ nifi.flow.configuration.archive.max.time=30 days diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-properties-loader/src/main/java/org/apache/nifi/properties/NiFiPropertiesLoader.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-properties-loader/src/main/java/org/apache/nifi/properties/NiFiPropertiesLoader.java index d893e8fd01..57a9aea696 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-properties-loader/src/main/java/org/apache/nifi/properties/NiFiPropertiesLoader.java +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-properties-loader/src/main/java/org/apache/nifi/properties/NiFiPropertiesLoader.java @@ -233,15 +233,9 @@ public class NiFiPropertiesLoader { throw new SensitivePropertyProtectionException(PROPERTIES_KEY_MESSAGE); } - final File jsonFile = defaultProperties.getFlowConfigurationJsonFile(); - if (jsonFile != null && jsonFile.exists()) { - logger.error("Flow Configuration [{}] Found: Migration Required for blank Sensitive Properties Key [{}]", jsonFile, NiFiProperties.SENSITIVE_PROPS_KEY); - throw new SensitivePropertyProtectionException(PROPERTIES_KEY_MESSAGE); - } - - final File xmlFile = defaultProperties.getFlowConfigurationFile(); - if (xmlFile.exists()) { - logger.error("Flow Configuration [{}] Found: Migration Required for blank Sensitive Properties Key [{}]", xmlFile, NiFiProperties.SENSITIVE_PROPS_KEY); + final File flowConfigurationFile = defaultProperties.getFlowConfigurationFile(); + if (flowConfigurationFile.exists()) { + logger.error("Flow Configuration [{}] Found: Migration Required for blank Sensitive Properties Key [{}]", flowConfigurationFile, NiFiProperties.SENSITIVE_PROPS_KEY); throw new SensitivePropertyProtectionException(PROPERTIES_KEY_MESSAGE); } diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-properties-loader/src/test/resources/properties/conf/duplicates.nifi.properties b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-properties-loader/src/test/resources/properties/conf/duplicates.nifi.properties index 1ad45f1617..718e115486 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-properties-loader/src/test/resources/properties/conf/duplicates.nifi.properties +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-properties-loader/src/test/resources/properties/conf/duplicates.nifi.properties @@ -13,7 +13,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -nifi.flow.configuration.file=./conf/flow.xml.gz nifi.flow.configuration.file=./conf/flow.json.gz +nifi.flow.configuration.file=./conf/flow2.json.gz another.duplicate=foo another.duplicate=bar \ No newline at end of file diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-properties-loader/src/test/resources/properties/conf/flow.nifi.properties b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-properties-loader/src/test/resources/properties/conf/flow.nifi.properties index eeac375b29..77ebe04142 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-properties-loader/src/test/resources/properties/conf/flow.nifi.properties +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-properties-loader/src/test/resources/properties/conf/flow.nifi.properties @@ -13,8 +13,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -nifi.flow.configuration.file=./conf/flow.xml.gz -nifi.flow.configuration.json.file=./conf/flow.json.gz +nifi.flow.configuration.file=./conf/flow.json.gz # duplicate with the same value intentionally added to verify this is valid # while bad form, it will generate only a warning, not a critical error -nifi.flow.configuration.json.file=./conf/flow.json.gz +nifi.flow.configuration.file=./conf/flow.json.gz diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-properties-loader/src/test/resources/properties/conf/protected.nifi.properties b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-properties-loader/src/test/resources/properties/conf/protected.nifi.properties index 44fd450c3d..208e6e31d0 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-properties-loader/src/test/resources/properties/conf/protected.nifi.properties +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-properties-loader/src/test/resources/properties/conf/protected.nifi.properties @@ -13,8 +13,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -nifi.flow.configuration.file=./conf/flow.xml.gz -nifi.flow.configuration.json.file=./conf/flow.json.gz +nifi.flow.configuration.file=./conf/flow.json.gz nifi.security.keystorePasswd=Ycac+pAe3AdHbCAC||ImnArC6KZJ+WXDMwJw2cjpOpNJFk1s5XyfbkrB8= nifi.security.keystorePasswd.protected=aes/gcm/128 diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-resources/pom.xml b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-resources/pom.xml index b916372014..b6be9e8261 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-resources/pom.xml +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-resources/pom.xml @@ -37,8 +37,7 @@ 10000 1 GB - ./conf/flow.xml.gz - ./conf/flow.json.gz + ./conf/flow.json.gz true ./conf/archive/ 30 days diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-resources/src/main/resources/conf/nifi.properties b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-resources/src/main/resources/conf/nifi.properties index 1cea6f301d..aea5a693dc 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-resources/src/main/resources/conf/nifi.properties +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-resources/src/main/resources/conf/nifi.properties @@ -15,7 +15,6 @@ # Core Properties # nifi.flow.configuration.file=${nifi.flow.configuration.file} -nifi.flow.configuration.json.file=${nifi.flow.configuration.json.file} nifi.flow.configuration.archive.enabled=${nifi.flow.configuration.archive.enabled} nifi.flow.configuration.archive.dir=${nifi.flow.configuration.archive.dir} nifi.flow.configuration.archive.max.time=${nifi.flow.configuration.archive.max.time} diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-runtime/src/test/resources/NiFiProperties/conf/encrypted.nifi.properties b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-runtime/src/test/resources/NiFiProperties/conf/encrypted.nifi.properties index fad8f1331e..b554ebb245 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-runtime/src/test/resources/NiFiProperties/conf/encrypted.nifi.properties +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-runtime/src/test/resources/NiFiProperties/conf/encrypted.nifi.properties @@ -14,7 +14,7 @@ # limitations under the License. # Core Properties # -nifi.flow.configuration.file=./target/conf/flow.xml.gz +nifi.flow.configuration.file=./target/conf/flow.json.gz nifi.flow.configuration.archive.enabled=true nifi.flow.configuration.archive.dir=./target/conf/archive/ nifi.flow.configuration.archive.max.time=30 days diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-runtime/src/test/resources/NiFiProperties/conf/nifi.properties b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-runtime/src/test/resources/NiFiProperties/conf/nifi.properties index 3aa6a84ade..374f5b6e97 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-runtime/src/test/resources/NiFiProperties/conf/nifi.properties +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-runtime/src/test/resources/NiFiProperties/conf/nifi.properties @@ -14,7 +14,7 @@ # limitations under the License. # Core Properties # -nifi.flow.configuration.file=./target/conf/flow.xml.gz +nifi.flow.configuration.file=./target/conf/flow.json.gz nifi.flow.configuration.archive.enabled=true nifi.flow.configuration.archive.dir=./target/conf/archive/ nifi.flow.configuration.archive.max.time=30 days diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-site-to-site/src/main/java/org/apache/nifi/remote/StandardPublicPort.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-site-to-site/src/main/java/org/apache/nifi/remote/StandardPublicPort.java index 52b3300438..bd05945dc2 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-site-to-site/src/main/java/org/apache/nifi/remote/StandardPublicPort.java +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-site-to-site/src/main/java/org/apache/nifi/remote/StandardPublicPort.java @@ -16,23 +16,6 @@ */ package org.apache.nifi.remote; -import static java.util.Objects.requireNonNull; - -import java.io.IOException; -import java.net.SocketTimeoutException; -import java.util.ArrayList; -import java.util.Collection; -import java.util.Collections; -import java.util.HashSet; -import java.util.List; -import java.util.Set; -import java.util.concurrent.ArrayBlockingQueue; -import java.util.concurrent.BlockingQueue; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicBoolean; -import java.util.concurrent.atomic.AtomicReference; -import java.util.concurrent.locks.Lock; -import java.util.concurrent.locks.ReentrantLock; import org.apache.commons.lang3.builder.ToStringBuilder; import org.apache.commons.lang3.builder.ToStringStyle; import org.apache.nifi.authorization.AuthorizationResult; @@ -76,15 +59,27 @@ import org.apache.nifi.util.NiFiProperties; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.io.IOException; +import java.net.SocketTimeoutException; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.HashSet; +import java.util.List; +import java.util.Set; +import java.util.concurrent.ArrayBlockingQueue; +import java.util.concurrent.BlockingQueue; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.locks.Lock; +import java.util.concurrent.locks.ReentrantLock; + public class StandardPublicPort extends AbstractPort implements PublicPort { private static final String CATEGORY = "Site to Site"; private static final Logger logger = LoggerFactory.getLogger(StandardPublicPort.class); - - private final AtomicReference> groupAccessControl = new AtomicReference<>(new HashSet<>()); - private final AtomicReference> userAccessControl = new AtomicReference<>(new HashSet<>()); private final boolean secure; private final Authorizer authorizer; private final List identityMappings; @@ -310,26 +305,6 @@ public class StandardPublicPort extends AbstractPort implements PublicPort { } } - @Override - public void setGroupAccessControl(Set groups) { - groupAccessControl.set(new HashSet<>(requireNonNull(groups))); - } - - @Override - public Set getGroupAccessControl() { - return Collections.unmodifiableSet(groupAccessControl.get()); - } - - @Override - public void setUserAccessControl(Set users) { - userAccessControl.set(new HashSet<>(requireNonNull(users))); - } - - @Override - public Set getUserAccessControl() { - return Collections.unmodifiableSet(userAccessControl.get()); - } - @Override public void shutdown() { super.shutdown(); diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-site-to-site/src/test/resources/nifi.properties b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-site-to-site/src/test/resources/nifi.properties index d7088dce70..403deae865 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-site-to-site/src/test/resources/nifi.properties +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-site-to-site/src/test/resources/nifi.properties @@ -14,7 +14,7 @@ # limitations under the License. # core properties # -nifi.flow.configuration.file=./conf/flow.xml.gz +nifi.flow.configuration.file=./conf/flow.json.gz nifi.flow.configuration.archive.dir=./conf/archive/ nifi.task.configuration.file=./conf/reporting-tasks.xml nifi.service.configuration.file=./conf/controller-services.xml diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/audit/PortAuditor.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/audit/PortAuditor.java index efedfd9414..3ae44be1c6 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/audit/PortAuditor.java +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/audit/PortAuditor.java @@ -16,19 +16,18 @@ */ package org.apache.nifi.audit; -import org.apache.commons.lang3.StringUtils; import org.apache.nifi.action.Action; import org.apache.nifi.action.Component; import org.apache.nifi.action.FlowChangeAction; import org.apache.nifi.action.Operation; import org.apache.nifi.action.details.ActionDetails; import org.apache.nifi.action.details.FlowChangeConfigureDetails; +import org.apache.nifi.authorization.user.NiFiUser; import org.apache.nifi.authorization.user.NiFiUserUtils; import org.apache.nifi.connectable.ConnectableType; import org.apache.nifi.connectable.Port; import org.apache.nifi.controller.ScheduledState; import org.apache.nifi.remote.PublicPort; -import org.apache.nifi.authorization.user.NiFiUser; import org.apache.nifi.web.api.dto.PortDTO; import org.apache.nifi.web.dao.PortDAO; import org.aspectj.lang.ProceedingJoinPoint; @@ -40,8 +39,6 @@ import org.slf4j.LoggerFactory; import java.util.ArrayList; import java.util.Collection; import java.util.Date; -import java.util.HashSet; -import java.util.Set; @Aspect public class PortAuditor extends NiFiAuditor { @@ -92,15 +89,7 @@ public class PortAuditor extends NiFiAuditor { final String comments = port.getComments(); final int maxConcurrentTasks = port.getMaxConcurrentTasks(); - final Set existingUsers = new HashSet<>(); - final Set existingGroups = new HashSet<>(); - boolean isPublicPort = false; - if (port instanceof PublicPort) { - isPublicPort = true; - final PublicPort publicPort = (PublicPort) port; - existingUsers.addAll(publicPort.getUserAccessControl()); - existingGroups.addAll(publicPort.getGroupAccessControl()); - } + boolean isPublicPort = port instanceof PublicPort; // perform the underlying operation final Port updatedPort = (Port) proceedingJoinPoint.proceed(); @@ -145,46 +134,6 @@ public class PortAuditor extends NiFiAuditor { configurationDetails.add(configDetails); } - - // if user access control was specified in the request - if (portDTO.getUserAccessControl() != null) { - final Set newUsers = new HashSet<>(portDTO.getUserAccessControl()); - newUsers.removeAll(existingUsers); - - final Set removedUsers = new HashSet<>(existingUsers); - removedUsers.removeAll(portDTO.getUserAccessControl()); - - // if users were added/removed - if (newUsers.size() > 0 || removedUsers.size() > 0) { - // create the config details - FlowChangeConfigureDetails configDetails = new FlowChangeConfigureDetails(); - configDetails.setName("User Access Control"); - configDetails.setValue(StringUtils.join(portDTO.getUserAccessControl(), ", ")); - configDetails.setPreviousValue(StringUtils.join(existingUsers, ", ")); - - configurationDetails.add(configDetails); - } - } - - // if group access control was specified in the request - if (portDTO.getGroupAccessControl() != null) { - final Set newGroups = new HashSet<>(portDTO.getGroupAccessControl()); - newGroups.removeAll(existingGroups); - - final Set removedGroups = new HashSet<>(existingGroups); - removedGroups.removeAll(portDTO.getGroupAccessControl()); - - // if groups were added/removed - if (newGroups.size() > 0 || removedGroups.size() > 0) { - // create the config details - FlowChangeConfigureDetails configDetails = new FlowChangeConfigureDetails(); - configDetails.setName("Group Access Control"); - configDetails.setValue(StringUtils.join(portDTO.getGroupAccessControl(), ", ")); - configDetails.setPreviousValue(StringUtils.join(existingGroups, ", ")); - - configurationDetails.add(configDetails); - } - } } final Collection actions = new ArrayList<>(); diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/api/dto/DtoFactory.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/api/dto/DtoFactory.java index 7a79cadb22..0723aa3fac 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/api/dto/DtoFactory.java +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/api/dto/DtoFactory.java @@ -1417,9 +1417,13 @@ public final class DtoFactory { final PublicPort publicPort = (PublicPort) port; dto.setAllowRemoteAccess(true); dto.setTransmitting(publicPort.isTransmitting()); - dto.setGroupAccessControl(publicPort.getGroupAccessControl()); - dto.setUserAccessControl(publicPort.getUserAccessControl()); } + // if this port is remotely accessible, determine if its actually connected to another nifi + if (port instanceof PublicPort) { + final PublicPort publicPort = (PublicPort) port; + dto.setAllowRemoteAccess(true); + dto.setTransmitting(publicPort.isTransmitting()); + } final Collection validationErrors = port.getValidationErrors(); if (validationErrors != null && !validationErrors.isEmpty()) { @@ -4471,8 +4475,6 @@ public final class DtoFactory { copy.setType(original.getType()); copy.setTransmitting(original.isTransmitting()); copy.setConcurrentlySchedulableTaskCount(original.getConcurrentlySchedulableTaskCount()); - copy.setUserAccessControl(copy(original.getUserAccessControl())); - copy.setGroupAccessControl(copy(original.getGroupAccessControl())); copy.setValidationErrors(copy(original.getValidationErrors())); copy.setVersionedComponentId(original.getVersionedComponentId()); copy.setAllowRemoteAccess(original.getAllowRemoteAccess()); diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/dao/impl/AbstractPortDAO.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/dao/impl/AbstractPortDAO.java index 345b2502b8..ed73f74d92 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/dao/impl/AbstractPortDAO.java +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/dao/impl/AbstractPortDAO.java @@ -79,8 +79,7 @@ public abstract class AbstractPortDAO extends ComponentDAO implements PortDAO { } // see what's be modified - if (isAnyNotNull(portDTO.getUserAccessControl(), - portDTO.getGroupAccessControl(), + if (isAnyNotNull( portDTO.getConcurrentlySchedulableTaskCount(), portDTO.getName(), portDTO.getComments(), @@ -161,16 +160,6 @@ public abstract class AbstractPortDAO extends ComponentDAO implements PortDAO { } } - if (port instanceof PublicPort) { - final PublicPort publicPort = (PublicPort) port; - if (isNotNull(portDTO.getGroupAccessControl())) { - publicPort.setGroupAccessControl(portDTO.getGroupAccessControl()); - } - if (isNotNull(portDTO.getUserAccessControl())) { - publicPort.setUserAccessControl(portDTO.getUserAccessControl()); - } - } - // update the port final String name = portDTO.getName(); final String comments = portDTO.getComments(); diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/search/attributematchers/PublicPortMatcher.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/search/attributematchers/PublicPortMatcher.java deleted file mode 100644 index 417ee38146..0000000000 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/search/attributematchers/PublicPortMatcher.java +++ /dev/null @@ -1,42 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.nifi.web.search.attributematchers; - -import org.apache.nifi.connectable.Port; -import org.apache.nifi.remote.PublicPort; -import org.apache.nifi.web.search.query.SearchQuery; - -import java.util.List; - -import static org.apache.nifi.web.search.attributematchers.AttributeMatcher.addIfMatching; - -public class PublicPortMatcher implements AttributeMatcher { - private static final String LABEL_USER = "User access control"; - private static final String LABEL_GROUP = "Group access control"; - - @Override - public void match(final Port component, final SearchQuery query, final List matches) { - final String searchTerm = query.getTerm(); - - if (component instanceof PublicPort) { - final PublicPort publicPort = (PublicPort) component; - - publicPort.getUserAccessControl().forEach(control -> addIfMatching(searchTerm, control, LABEL_USER, matches)); - publicPort.getGroupAccessControl().forEach(control -> addIfMatching(searchTerm, control, LABEL_GROUP, matches)); - } - } -} diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/resources/nifi-web-api-context.xml b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/resources/nifi-web-api-context.xml index 2c56e9aeeb..5d9fe21d38 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/resources/nifi-web-api-context.xml +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/resources/nifi-web-api-context.xml @@ -104,7 +104,6 @@ - @@ -164,7 +163,6 @@ - diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/test/java/org/apache/nifi/web/controller/ComponentMockUtil.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/test/java/org/apache/nifi/web/controller/ComponentMockUtil.java index 51dbd4fd24..909f1f45cd 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/test/java/org/apache/nifi/web/controller/ComponentMockUtil.java +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/test/java/org/apache/nifi/web/controller/ComponentMockUtil.java @@ -325,12 +325,8 @@ public class ComponentMockUtil { final String comments, final ScheduledState scheduledState, final boolean isValid, - final boolean isAuthorized, - final List userAccessControl, - final List groupAccessControl) { + final boolean isAuthorized) { final PublicPort result = getPort(PublicPort.class, id, name, Optional.empty(), comments, scheduledState, isValid, isAuthorized); - Mockito.when(result.getUserAccessControl()).thenReturn(new HashSet<>(userAccessControl)); - Mockito.when(result.getGroupAccessControl()).thenReturn(new HashSet<>(groupAccessControl)); return result; } @@ -341,12 +337,8 @@ public class ComponentMockUtil { final String versionedId, final ScheduledState scheduledState, final boolean isValid, - final boolean isAuthorized, - final List userAccessControl, - final List groupAccessControl) { + final boolean isAuthorized) { final PublicPort result = getPort(PublicPort.class, id, name, Optional.of(versionedId), comments, scheduledState, isValid, isAuthorized); - Mockito.when(result.getUserAccessControl()).thenReturn(new HashSet<>(userAccessControl)); - Mockito.when(result.getGroupAccessControl()).thenReturn(new HashSet<>(groupAccessControl)); return result; } diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/test/java/org/apache/nifi/web/controller/ControllerSearchServiceIntegrationTest.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/test/java/org/apache/nifi/web/controller/ControllerSearchServiceIntegrationTest.java index 7830e2541d..52ae9cd07b 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/test/java/org/apache/nifi/web/controller/ControllerSearchServiceIntegrationTest.java +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/test/java/org/apache/nifi/web/controller/ControllerSearchServiceIntegrationTest.java @@ -31,7 +31,6 @@ import org.junit.jupiter.api.Test; import org.mockito.Mockito; import java.util.ArrayList; -import java.util.Arrays; import java.util.HashMap; import java.util.HashSet; import java.util.List; @@ -45,7 +44,6 @@ import static org.apache.nifi.web.controller.ComponentMockUtil.getConnection; import static org.apache.nifi.web.controller.ComponentMockUtil.getFunnel; import static org.apache.nifi.web.controller.ComponentMockUtil.getPort; import static org.apache.nifi.web.controller.ComponentMockUtil.getProcessorNode; -import static org.apache.nifi.web.controller.ComponentMockUtil.getPublicPort; import static org.apache.nifi.web.controller.ComponentMockUtil.getRemoteProcessGroup; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertTrue; @@ -603,20 +601,4 @@ public class ControllerSearchServiceIntegrationTest extends AbstractControllerSe .ofRemoteProcessGroup(getSimpleResultFromRoot("remote2", "remoteName2", "Transmission: Off")) .validate(results); } - - @Test - public void testSearchBasedOnPortPublicity() { - // given - givenRootProcessGroup() - .withInputPort(getPublicPort("port", "portName", "", ScheduledState.RUNNING, true, AUTHORIZED, - Arrays.asList("accessAllowed1"), Arrays.asList("accessAllowed2"))); - - // when - whenExecuteSearch("allowed"); - - // then - thenResultConsists() - .ofInputPort(getSimpleResultFromRoot("port", "portName", "User access control: accessAllowed1", "Group access control: accessAllowed2")) - .validate(results); - } } diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/test/java/org/apache/nifi/web/controller/ControllerSearchServiceRegressionTest.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/test/java/org/apache/nifi/web/controller/ControllerSearchServiceRegressionTest.java index f8eaaf396a..698a876b06 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/test/java/org/apache/nifi/web/controller/ControllerSearchServiceRegressionTest.java +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/test/java/org/apache/nifi/web/controller/ControllerSearchServiceRegressionTest.java @@ -178,20 +178,14 @@ public class ControllerSearchServiceRegressionTest extends AbstractControllerSea "outpublicport1_versionedId_omniMatch", ScheduledState.STOPPED, true, - AUTHORIZED, - Arrays.asList("outpublicport1_userAccessControl1_omniMatch", "outpublicport1_userAccessControl2_omniMatch"), - Arrays.asList("outpublicport1_groupAccessControl1_omniMatch", "outpublicport1_groupAccessControl2_omniMatch") + AUTHORIZED ); final ComponentSearchResultDTO outputPublicPort1Result = getSimpleResultFromRoot("outpublicport1_id_omniMatch", "outpublicport1_name_omniMatch", "Id: outpublicport1_id_omniMatch", "Version Control ID: outpublicport1_versionedId_omniMatch", "Name: outpublicport1_name_omniMatch", - "Comments: outpublicport1_comments_omniMatch", - "User access control: outpublicport1_userAccessControl1_omniMatch", - "User access control: outpublicport1_userAccessControl2_omniMatch", - "Group access control: outpublicport1_groupAccessControl1_omniMatch", - "Group access control: outpublicport1_groupAccessControl2_omniMatch" + "Comments: outpublicport1_comments_omniMatch" ); final Label label1 = getLabel("label1_id_omniMatch", "label1_value_omniMatch", true); diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/test/java/org/apache/nifi/web/search/attributematchers/PublicPortMatcherTest.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/test/java/org/apache/nifi/web/search/attributematchers/PublicPortMatcherTest.java deleted file mode 100644 index 6afced2741..0000000000 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/test/java/org/apache/nifi/web/search/attributematchers/PublicPortMatcherTest.java +++ /dev/null @@ -1,65 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.nifi.web.search.attributematchers; - -import org.apache.nifi.connectable.Port; -import org.apache.nifi.remote.PublicPort; -import org.junit.jupiter.api.Test; -import org.mockito.Mock; -import org.mockito.Mockito; - -import java.util.Arrays; -import java.util.HashSet; - -public class PublicPortMatcherTest extends AbstractAttributeMatcherTest { - - @Mock - private Port port; - - @Mock - private PublicPort publicPort; - - @Test - public void testNonPublicPort() { - // given - final PublicPortMatcher testSubject = new PublicPortMatcher(); - givenDefaultSearchTerm(); - // when - testSubject.match(port, searchQuery, matches); - - // then - thenNoMatches(); - } - - @Test - public void testPublicPort() { - // given - final PublicPortMatcher testSubject = new PublicPortMatcher(); - givenDefaultSearchTerm(); - Mockito.when(publicPort.getUserAccessControl()).thenReturn(new HashSet<>(Arrays.asList("user1Lorem", "user2Lorem"))); - Mockito.when(publicPort.getGroupAccessControl()).thenReturn(new HashSet<>(Arrays.asList("group1Lorem", "group2Lorem"))); - - // when - testSubject.match(publicPort, searchQuery, matches); - - // then - thenMatchConsistsOf("User access control: user1Lorem", - "User access control: user2Lorem", - "Group access control: group1Lorem", - "Group access control: group1Lorem"); - } -} \ No newline at end of file diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/test/resources/access-control/nifi-anonymous-allowed.properties b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/test/resources/access-control/nifi-anonymous-allowed.properties index b1119efed5..3a76886725 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/test/resources/access-control/nifi-anonymous-allowed.properties +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/test/resources/access-control/nifi-anonymous-allowed.properties @@ -14,7 +14,7 @@ # limitations under the License. # Core Properties # -nifi.flow.configuration.file=target/test-classes/access-control/flow.xml.gz +nifi.flow.configuration.file=target/test-classes/access-control/flow.json.gz nifi.flow.configuration.archive.dir=target/archive nifi.flowcontroller.autoResumeState=true nifi.flowcontroller.graceful.shutdown.period=10 sec diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/test/resources/access-control/nifi-flow.properties b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/test/resources/access-control/nifi-flow.properties index 1ab4f81c2b..5f2dce9716 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/test/resources/access-control/nifi-flow.properties +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/test/resources/access-control/nifi-flow.properties @@ -14,7 +14,7 @@ # limitations under the License. # Core Properties # -nifi.flow.configuration.file=target/test-classes/access-control/flow.xml.gz +nifi.flow.configuration.file=target/test-classes/access-control/flow.json.gz nifi.flow.configuration.archive.dir=target/archive nifi.flowcontroller.autoResumeState=true nifi.flowcontroller.graceful.shutdown.period=10 sec diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/test/resources/access-control/nifi-mapped-identities.properties b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/test/resources/access-control/nifi-mapped-identities.properties index 4b119876f1..99cc893bf8 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/test/resources/access-control/nifi-mapped-identities.properties +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/test/resources/access-control/nifi-mapped-identities.properties @@ -16,7 +16,7 @@ # Core Properties # -nifi.flow.configuration.file=target/test-classes/access-control/flow.xml.gz +nifi.flow.configuration.file=target/test-classes/access-control/flow.json.gz nifi.flow.configuration.archive.dir=target/archive nifi.flowcontroller.autoResumeState=true nifi.flowcontroller.graceful.shutdown.period=10 sec diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/test/resources/access-control/nifi.properties b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/test/resources/access-control/nifi.properties index 748ff713ea..82acb5d0fa 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/test/resources/access-control/nifi.properties +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/test/resources/access-control/nifi.properties @@ -14,7 +14,7 @@ # limitations under the License. # Core Properties # -nifi.flow.configuration.file=target/test-classes/access-control/flow.xml.gz +nifi.flow.configuration.file=target/test-classes/access-control/flow.json.gz nifi.flow.configuration.archive.dir=target/archive nifi.flowcontroller.autoResumeState=true nifi.flowcontroller.graceful.shutdown.period=10 sec diff --git a/nifi-nar-bundles/nifi-kafka-bundle/nifi-kafka-2-6-processors/src/main/resources/docs/org.apache.nifi.processors.kafka.pubsub.ConsumeKafkaRecord_2_6/additionalDetails.html b/nifi-nar-bundles/nifi-kafka-bundle/nifi-kafka-2-6-processors/src/main/resources/docs/org.apache.nifi.processors.kafka.pubsub.ConsumeKafkaRecord_2_6/additionalDetails.html index 3b1d2d33a3..5e53c6a4c0 100644 --- a/nifi-nar-bundles/nifi-kafka-bundle/nifi-kafka-2-6-processors/src/main/resources/docs/org.apache.nifi.processors.kafka.pubsub.ConsumeKafkaRecord_2_6/additionalDetails.html +++ b/nifi-nar-bundles/nifi-kafka-bundle/nifi-kafka-2-6-processors/src/main/resources/docs/org.apache.nifi.processors.kafka.pubsub.ConsumeKafkaRecord_2_6/additionalDetails.html @@ -194,7 +194,7 @@ username="nifi" password="nifi-password"; - NOTE: The dynamic properties of this processor are not secured and as a result the password entered when utilizing sasl.jaas.config will be stored in the flow.xml.gz file in plain-text, and will be saved to NiFi Registry if using versioned flows. + NOTE: The dynamic properties of this processor are not secured and as a result the password entered when utilizing sasl.jaas.config will be stored in the flow.json.gz file in plain-text, and will be saved to NiFi Registry if using versioned flows.

@@ -234,7 +234,7 @@ username="nifi" password="nifi-password"; - NOTE: The dynamic properties of this processor are not secured and as a result the password entered when utilizing sasl.jaas.config will be stored in the flow.xml.gz file in plain-text, and will be saved to NiFi Registry if using versioned flows. + NOTE: The dynamic properties of this processor are not secured and as a result the password entered when utilizing sasl.jaas.config will be stored in the flow.json.gz file in plain-text, and will be saved to NiFi Registry if using versioned flows.

NOTE: The Kerberos Service Name is not required for SASL mechanism of SCRAM-SHA-256 or SCRAM-SHA-512. However, processor warns saying this attribute has to be filled with non empty string. You can choose to fill any random string, such as "null". diff --git a/nifi-nar-bundles/nifi-kafka-bundle/nifi-kafka-2-6-processors/src/main/resources/docs/org.apache.nifi.processors.kafka.pubsub.ConsumeKafka_2_6/additionalDetails.html b/nifi-nar-bundles/nifi-kafka-bundle/nifi-kafka-2-6-processors/src/main/resources/docs/org.apache.nifi.processors.kafka.pubsub.ConsumeKafka_2_6/additionalDetails.html index be2f380a72..159800898c 100644 --- a/nifi-nar-bundles/nifi-kafka-bundle/nifi-kafka-2-6-processors/src/main/resources/docs/org.apache.nifi.processors.kafka.pubsub.ConsumeKafka_2_6/additionalDetails.html +++ b/nifi-nar-bundles/nifi-kafka-bundle/nifi-kafka-2-6-processors/src/main/resources/docs/org.apache.nifi.processors.kafka.pubsub.ConsumeKafka_2_6/additionalDetails.html @@ -194,7 +194,7 @@ username="nifi" password="nifi-password"; - NOTE: The dynamic properties of this processor are not secured and as a result the password entered when utilizing sasl.jaas.config will be stored in the flow.xml.gz file in plain-text, and will be saved to NiFi Registry if using versioned flows. + NOTE: The dynamic properties of this processor are not secured and as a result the password entered when utilizing sasl.jaas.config will be stored in the flow.json.gz file in plain-text, and will be saved to NiFi Registry if using versioned flows.

@@ -234,7 +234,7 @@ username="nifi" password="nifi-password"; - NOTE: The dynamic properties of this processor are not secured and as a result the password entered when utilizing sasl.jaas.config will be stored in the flow.xml.gz file in plain-text, and will be saved to NiFi Registry if using versioned flows. + NOTE: The dynamic properties of this processor are not secured and as a result the password entered when utilizing sasl.jaas.config will be stored in the flow.json.gz file in plain-text, and will be saved to NiFi Registry if using versioned flows.

NOTE: The Kerberos Service Name is not required for SASL mechanism of SCRAM-SHA-256 or SCRAM-SHA-512. However, processor warns saying this attribute has to be filled with non empty string. You can choose to fill any random string, such as "null". diff --git a/nifi-nar-bundles/nifi-kafka-bundle/nifi-kafka-2-6-processors/src/main/resources/docs/org.apache.nifi.processors.kafka.pubsub.PublishKafkaRecord_2_6/additionalDetails.html b/nifi-nar-bundles/nifi-kafka-bundle/nifi-kafka-2-6-processors/src/main/resources/docs/org.apache.nifi.processors.kafka.pubsub.PublishKafkaRecord_2_6/additionalDetails.html index e125b0f067..8a7544348f 100644 --- a/nifi-nar-bundles/nifi-kafka-bundle/nifi-kafka-2-6-processors/src/main/resources/docs/org.apache.nifi.processors.kafka.pubsub.PublishKafkaRecord_2_6/additionalDetails.html +++ b/nifi-nar-bundles/nifi-kafka-bundle/nifi-kafka-2-6-processors/src/main/resources/docs/org.apache.nifi.processors.kafka.pubsub.PublishKafkaRecord_2_6/additionalDetails.html @@ -127,7 +127,7 @@ username="nifi" password="nifi-password"; - NOTE: The dynamic properties of this processor are not secured and as a result the password entered when utilizing sasl.jaas.config will be stored in the flow.xml.gz file in plain-text, and will be saved to NiFi Registry if using versioned flows. + NOTE: The dynamic properties of this processor are not secured and as a result the password entered when utilizing sasl.jaas.config will be stored in the flow.json.gz file in plain-text, and will be saved to NiFi Registry if using versioned flows.

@@ -169,7 +169,7 @@ username="nifi" password="nifi-password"; - NOTE: The dynamic properties of this processor are not secured and as a result the password entered when utilizing sasl.jaas.config will be stored in the flow.xml.gz file in plain-text, and will be saved to NiFi Registry if using versioned flows. + NOTE: The dynamic properties of this processor are not secured and as a result the password entered when utilizing sasl.jaas.config will be stored in the flow.json.gz file in plain-text, and will be saved to NiFi Registry if using versioned flows. NOTE: The Kerberos Service Name is not required for SASL mechanism of SCRAM-SHA-256 or SCRAM-SHA-512. However, processor warns saying this attribute has to be filled with non empty string. You can choose to fill any random string, such as "null".

diff --git a/nifi-nar-bundles/nifi-kafka-bundle/nifi-kafka-2-6-processors/src/main/resources/docs/org.apache.nifi.processors.kafka.pubsub.PublishKafka_2_6/additionalDetails.html b/nifi-nar-bundles/nifi-kafka-bundle/nifi-kafka-2-6-processors/src/main/resources/docs/org.apache.nifi.processors.kafka.pubsub.PublishKafka_2_6/additionalDetails.html index 177fff9959..3f8d34985f 100644 --- a/nifi-nar-bundles/nifi-kafka-bundle/nifi-kafka-2-6-processors/src/main/resources/docs/org.apache.nifi.processors.kafka.pubsub.PublishKafka_2_6/additionalDetails.html +++ b/nifi-nar-bundles/nifi-kafka-bundle/nifi-kafka-2-6-processors/src/main/resources/docs/org.apache.nifi.processors.kafka.pubsub.PublishKafka_2_6/additionalDetails.html @@ -139,7 +139,7 @@ username="nifi" password="nifi-password"; - NOTE: The dynamic properties of this processor are not secured and as a result the password entered when utilizing sasl.jaas.config will be stored in the flow.xml.gz file in plain-text, and will be saved to NiFi Registry if using versioned flows. + NOTE: The dynamic properties of this processor are not secured and as a result the password entered when utilizing sasl.jaas.config will be stored in the flow.json.gz file in plain-text, and will be saved to NiFi Registry if using versioned flows.

@@ -181,7 +181,7 @@ username="nifi" password="nifi-password"; - NOTE: The dynamic properties of this processor are not secured and as a result the password entered when utilizing sasl.jaas.config will be stored in the flow.xml.gz file in plain-text, and will be saved to NiFi Registry if using versioned flows. + NOTE: The dynamic properties of this processor are not secured and as a result the password entered when utilizing sasl.jaas.config will be stored in the flow.json.gz file in plain-text, and will be saved to NiFi Registry if using versioned flows. NOTE: The Kerberos Service Name is not required for SASL mechanism of SCRAM-SHA-256 or SCRAM-SHA-512. However, processor warns saying this attribute has to be filled with non empty string. You can choose to fill any random string, such as "null".

diff --git a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-reporting-tasks/src/test/resources/nifi.properties b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-reporting-tasks/src/test/resources/nifi.properties index 459398488b..34fdb8aa85 100644 --- a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-reporting-tasks/src/test/resources/nifi.properties +++ b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-reporting-tasks/src/test/resources/nifi.properties @@ -14,7 +14,7 @@ # limitations under the License. # Core Properties # -nifi.flow.configuration.file=./target/flow.xml.gz +nifi.flow.configuration.file=./target/flow.json.gz nifi.flow.configuration.archive.dir=./target/archive/ nifi.reporting.task.configuration.file=./target/reporting-tasks.xml diff --git a/nifi-stateless/nifi-stateless-assembly/README.md b/nifi-stateless/nifi-stateless-assembly/README.md index e887860ca7..c853f46389 100644 --- a/nifi-stateless/nifi-stateless-assembly/README.md +++ b/nifi-stateless/nifi-stateless-assembly/README.md @@ -288,7 +288,7 @@ configuration provides it with the necessary information for what flow to run. The flow's location must be provided either by specifying a NiFi Registry URL, Bucket ID, and Flow ID (and optional version); by specifying a local filename for the flow; by specifying a URL for the flow; or by including a "stringified" version of the JSON flow definition itself. Note that if using a local filename, the format of the file is not the same as -the `flow.xml.gz` file that NiFi uses but rather is the `Versioned Flow Snapshot` format that is used by the NiFi Registry. +the `flow.json.gz` file that NiFi uses but rather is the `Versioned Flow Snapshot` format that is used by the NiFi Registry. The easiest way to export a flow from NiFi onto local disk for use by Stateless NiFi is to right-click on a Process Group or the canvas in NiFi and choose `Downlaod Flow`. diff --git a/nifi-system-tests/nifi-system-test-suite/pom.xml b/nifi-system-tests/nifi-system-test-suite/pom.xml index 58451408b8..eab29b13fd 100644 --- a/nifi-system-tests/nifi-system-test-suite/pom.xml +++ b/nifi-system-tests/nifi-system-test-suite/pom.xml @@ -109,10 +109,10 @@ apache-rat-plugin - src/test/resources/flows/mismatched-flows/flow1.xml.gz - src/test/resources/flows/mismatched-flows/flow2.xml.gz - src/test/resources/flows/missing-connection/with-connection.xml.gz - src/test/resources/flows/missing-connection/without-connection.xml.gz + src/test/resources/flows/mismatched-flows/flow1.json.gz + src/test/resources/flows/mismatched-flows/flow2.json.gz + src/test/resources/flows/missing-connection/with-connection.json.gz + src/test/resources/flows/missing-connection/without-connection.json.gz src/test/resources/keystore.jks src/test/resources/truststore.jks diff --git a/nifi-system-tests/nifi-system-test-suite/src/test/java/org/apache/nifi/tests/system/InstanceConfiguration.java b/nifi-system-tests/nifi-system-test-suite/src/test/java/org/apache/nifi/tests/system/InstanceConfiguration.java index 3d42426425..880c5c8e3a 100644 --- a/nifi-system-tests/nifi-system-test-suite/src/test/java/org/apache/nifi/tests/system/InstanceConfiguration.java +++ b/nifi-system-tests/nifi-system-test-suite/src/test/java/org/apache/nifi/tests/system/InstanceConfiguration.java @@ -25,7 +25,7 @@ import java.util.Objects; public class InstanceConfiguration { private final File bootstrapConfigFile; private final File instanceDirectory; - private final File flowXmlGz; + private final File flowJsonGz; private final File stateDirectory; private final boolean autoStart; private final Map nifiPropertiesOverrides; @@ -34,7 +34,7 @@ public class InstanceConfiguration { private InstanceConfiguration(Builder builder) { this.bootstrapConfigFile = builder.bootstrapConfigFile; this.instanceDirectory = builder.instanceDirectory; - this.flowXmlGz = builder.flowXmlGz; + this.flowJsonGz = builder.flowJsonGz; this.stateDirectory = builder.stateDirectory; this.autoStart = builder.autoStart; this.nifiPropertiesOverrides = builder.nifiPropertiesOverrides; @@ -49,8 +49,8 @@ public class InstanceConfiguration { return instanceDirectory; } - public File getFlowXmlGz() { - return flowXmlGz; + public File getFlowJsonGz() { + return flowJsonGz; } public File getStateDirectory() { @@ -81,19 +81,19 @@ public class InstanceConfiguration { final InstanceConfiguration that = (InstanceConfiguration) other; return autoStart == that.autoStart && unpackPythonExtensions == that.unpackPythonExtensions && Objects.equals(bootstrapConfigFile, that.bootstrapConfigFile) - && Objects.equals(instanceDirectory, that.instanceDirectory) && Objects.equals(flowXmlGz, that.flowXmlGz) + && Objects.equals(instanceDirectory, that.instanceDirectory) && Objects.equals(flowJsonGz, that.flowJsonGz) && Objects.equals(stateDirectory, that.stateDirectory) && Objects.equals(nifiPropertiesOverrides, that.nifiPropertiesOverrides); } @Override public int hashCode() { - return Objects.hash(bootstrapConfigFile, instanceDirectory, flowXmlGz, stateDirectory, autoStart, nifiPropertiesOverrides, unpackPythonExtensions); + return Objects.hash(bootstrapConfigFile, instanceDirectory, flowJsonGz, stateDirectory, autoStart, nifiPropertiesOverrides, unpackPythonExtensions); } public static class Builder { private File bootstrapConfigFile; private File instanceDirectory; - private File flowXmlGz; + private File flowJsonGz; private File stateDirectory; private boolean autoStart = true; private boolean unpackPythonExtensions = false; @@ -130,13 +130,13 @@ public class InstanceConfiguration { return instanceDirectory(new File(instanceDirName)); } - public Builder flowXml(final File flowXml) { - this.flowXmlGz = flowXml; + public Builder flowJson(final File flowJsonGz) { + this.flowJsonGz = flowJsonGz; return this; } - public Builder flowXml(final String flowXmlFilename) { - return flowXml(new File(flowXmlFilename)); + public Builder flowJson(final String flowJsonFilename) { + return flowJson(new File(flowJsonFilename)); } public Builder stateDirectory(final File stateDirectory) { diff --git a/nifi-system-tests/nifi-system-test-suite/src/test/java/org/apache/nifi/tests/system/SpawnedStandaloneNiFiInstanceFactory.java b/nifi-system-tests/nifi-system-test-suite/src/test/java/org/apache/nifi/tests/system/SpawnedStandaloneNiFiInstanceFactory.java index bc1047f927..4c77714b75 100644 --- a/nifi-system-tests/nifi-system-test-suite/src/test/java/org/apache/nifi/tests/system/SpawnedStandaloneNiFiInstanceFactory.java +++ b/nifi-system-tests/nifi-system-test-suite/src/test/java/org/apache/nifi/tests/system/SpawnedStandaloneNiFiInstanceFactory.java @@ -174,10 +174,10 @@ public class SpawnedStandaloneNiFiInstanceFactory implements NiFiInstanceFactory } NiFiSystemKeyStoreProvider.configureKeyStores(destinationCertsDir); - final File flowXmlGz = instanceConfiguration.getFlowXmlGz(); - if (flowXmlGz != null) { - final File destinationFlowXmlGz = new File(destinationConf, "flow.xml.gz"); - Files.copy(flowXmlGz.toPath(), destinationFlowXmlGz.toPath()); + final File flowJsonGz = instanceConfiguration.getFlowJsonGz(); + if (flowJsonGz != null) { + final File destinationFlowJsonGz = new File(destinationConf, "flow.json.gz"); + Files.copy(flowJsonGz.toPath(), destinationFlowJsonGz.toPath()); } // Write out any Property overrides diff --git a/nifi-system-tests/nifi-system-test-suite/src/test/java/org/apache/nifi/tests/system/clustering/FlowSynchronizationIT.java b/nifi-system-tests/nifi-system-test-suite/src/test/java/org/apache/nifi/tests/system/clustering/FlowSynchronizationIT.java index 48c426bfc7..b7e93d355d 100644 --- a/nifi-system-tests/nifi-system-test-suite/src/test/java/org/apache/nifi/tests/system/clustering/FlowSynchronizationIT.java +++ b/nifi-system-tests/nifi-system-test-suite/src/test/java/org/apache/nifi/tests/system/clustering/FlowSynchronizationIT.java @@ -425,7 +425,7 @@ public class FlowSynchronizationIT extends NiFiSystemIT { @Test public void testRestartWithFlowJsonGzNoXml() throws NiFiClientException, IOException { - restartWithOnlySingleFlowPersistenceFile("flow.xml.gz"); + restartWithOnlySingleFlowPersistenceFile("flow.json.gz"); } private void restartWithOnlySingleFlowPersistenceFile(final String filenameToDelete) throws NiFiClientException, IOException { @@ -437,7 +437,6 @@ public class FlowSynchronizationIT extends NiFiSystemIT { node2.stop(); final File confDir = new File(node2.getInstanceDirectory(), "conf"); - assertEquals(1, confDir.listFiles(file -> file.getName().equals("flow.xml.gz")).length); assertEquals(1, confDir.listFiles(file -> file.getName().equals("flow.json.gz")).length); final File jsonFile = new File(confDir, filenameToDelete); diff --git a/nifi-system-tests/nifi-system-test-suite/src/test/java/org/apache/nifi/tests/system/clustering/JoinClusterAdjustStateIT.java b/nifi-system-tests/nifi-system-test-suite/src/test/java/org/apache/nifi/tests/system/clustering/JoinClusterAdjustStateIT.java index 8c29fdc3c1..b7f7bfe53b 100644 --- a/nifi-system-tests/nifi-system-test-suite/src/test/java/org/apache/nifi/tests/system/clustering/JoinClusterAdjustStateIT.java +++ b/nifi-system-tests/nifi-system-test-suite/src/test/java/org/apache/nifi/tests/system/clustering/JoinClusterAdjustStateIT.java @@ -83,8 +83,8 @@ public class JoinClusterAdjustStateIT extends NiFiSystemIT { final NiFiInstance node2Instance = getNiFiInstance().getNodeInstance(2); // Copy the flow from Node 1 to Node 2. - final File node1Flow = new File(firstNodeInstanceDir, "conf/flow.xml.gz"); - final File node2Flow = new File(node2Instance.getInstanceDirectory(), "conf/flow.xml.gz"); + final File node1Flow = new File(firstNodeInstanceDir, "conf/flow.json.gz"); + final File node2Flow = new File(node2Instance.getInstanceDirectory(), "conf/flow.json.gz"); Thread.sleep(2000L); // Wait a bit before copying it, since the flow is written out in the background, and we want to ensure that the flow is up-to-date. Files.copy(node1Flow.toPath(), node2Flow.toPath()); diff --git a/nifi-system-tests/nifi-system-test-suite/src/test/java/org/apache/nifi/tests/system/clustering/JoinClusterWithDifferentFlow.java b/nifi-system-tests/nifi-system-test-suite/src/test/java/org/apache/nifi/tests/system/clustering/JoinClusterWithDifferentFlow.java index d42c1f5c7c..7f652712a8 100644 --- a/nifi-system-tests/nifi-system-test-suite/src/test/java/org/apache/nifi/tests/system/clustering/JoinClusterWithDifferentFlow.java +++ b/nifi-system-tests/nifi-system-test-suite/src/test/java/org/apache/nifi/tests/system/clustering/JoinClusterWithDifferentFlow.java @@ -16,40 +16,30 @@ */ package org.apache.nifi.tests.system.clustering; -import org.apache.nifi.controller.serialization.FlowEncodingVersion; -import org.apache.nifi.controller.serialization.FlowFromDOMFactory; -import org.apache.nifi.encrypt.PropertyEncryptor; -import org.apache.nifi.encrypt.PropertyEncryptorBuilder; +import com.fasterxml.jackson.databind.ObjectMapper; +import org.apache.nifi.controller.flow.VersionedDataflow; +import org.apache.nifi.flow.VersionedProcessGroup; +import org.apache.nifi.flow.VersionedProcessor; import org.apache.nifi.tests.system.InstanceConfiguration; import org.apache.nifi.tests.system.NiFiInstance; import org.apache.nifi.tests.system.NiFiInstanceFactory; import org.apache.nifi.tests.system.NiFiSystemIT; import org.apache.nifi.tests.system.SpawnedClusterNiFiInstanceFactory; -import org.apache.nifi.toolkit.cli.impl.client.nifi.NiFiClient; +import org.apache.nifi.toolkit.cli.impl.client.nifi.FlowClient; import org.apache.nifi.toolkit.cli.impl.client.nifi.NiFiClientException; -import org.apache.nifi.util.NiFiProperties; import org.apache.nifi.web.api.dto.AffectedComponentDTO; -import org.apache.nifi.web.api.dto.FlowSnippetDTO; -import org.apache.nifi.web.api.dto.NodeDTO; import org.apache.nifi.web.api.dto.ParameterContextDTO; import org.apache.nifi.web.api.dto.ParameterContextReferenceDTO; -import org.apache.nifi.web.api.dto.ProcessGroupDTO; -import org.apache.nifi.web.api.dto.ProcessorDTO; import org.apache.nifi.web.api.dto.flow.FlowDTO; import org.apache.nifi.web.api.dto.flow.ProcessGroupFlowDTO; import org.apache.nifi.web.api.entity.AffectedComponentEntity; import org.apache.nifi.web.api.entity.ControllerServiceEntity; import org.apache.nifi.web.api.entity.ControllerServicesEntity; -import org.apache.nifi.web.api.entity.NodeEntity; import org.apache.nifi.web.api.entity.ParameterEntity; import org.apache.nifi.web.api.entity.ProcessorEntity; -import org.apache.nifi.xml.processing.parsers.StandardDocumentProvider; import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; -import org.w3c.dom.Document; -import org.w3c.dom.Element; -import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; import java.io.File; import java.io.FileInputStream; @@ -63,38 +53,34 @@ import java.util.Arrays; import java.util.Collections; import java.util.List; import java.util.Map; -import java.util.Properties; import java.util.Set; import java.util.zip.GZIPInputStream; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; -@Disabled("This test needs some love. It had an issue where it assumed that Node 1 would have its flow elected the 'winner' in the flow election. That caused intermittent failures. Updated the test" + - " to instead startup both nodes with flow 1, then shutdown node 2, replace its flow, and startup again. However, this has caused its own set of problems because now the backup file that gets" + - " written out is JSON, not XML. Rather than going down the rabbit hole, just marking the test as Disabled for now.") +@Disabled("https://issues.apache.org/jira/browse/NIFI-12203") public class JoinClusterWithDifferentFlow extends NiFiSystemIT { @Override public NiFiInstanceFactory getInstanceFactory() { - final Map propertyOverrides = Collections.singletonMap("nifi.cluster.flow.serialization.format", "XML"); + final Map propertyOverrides = Collections.singletonMap("nifi.cluster.flow.serialization.format", "JSON"); return new SpawnedClusterNiFiInstanceFactory( new InstanceConfiguration.Builder() .bootstrapConfig("src/test/resources/conf/clustered/node1/bootstrap.conf") .instanceDirectory("target/node1") .overrideNifiProperties(propertyOverrides) - .flowXml(new File("src/test/resources/flows/mismatched-flows/flow1.xml.gz")) + .flowJson(new File("src/test/resources/flows/mismatched-flows/flow1.json.gz")) .build(), new InstanceConfiguration.Builder() .bootstrapConfig("src/test/resources/conf/clustered/node2/bootstrap.conf") .instanceDirectory("target/node2") - .flowXml(new File("src/test/resources/flows/mismatched-flows/flow1.xml.gz")) + .flowJson(new File("src/test/resources/flows/mismatched-flows/flow1.json.gz")) .overrideNifiProperties(propertyOverrides) .build() ); } - @Test public void testStartupWithDifferentFlow() throws IOException, NiFiClientException, InterruptedException { // Once we've started up, we want to have node 2 startup with a different flow. We cannot simply startup both nodes at the same time with @@ -106,102 +92,58 @@ public class JoinClusterWithDifferentFlow extends NiFiSystemIT { node2.stop(); final File node2ConfDir = new File(node2.getInstanceDirectory(), "conf"); - final File flowXmlFile = new File(node2ConfDir, "flow.xml.gz"); - Files.deleteIfExists(flowXmlFile.toPath()); - Files.copy(Paths.get("src/test/resources/flows/mismatched-flows/flow2.xml.gz"), flowXmlFile.toPath()); + final File flowJsonFile = new File(node2ConfDir, "flow.json.gz"); + Files.deleteIfExists(flowJsonFile.toPath()); + Files.copy(Paths.get("src/test/resources/flows/mismatched-flows/flow2.json.gz"), flowJsonFile.toPath()); node2.start(true); + waitForAllNodesConnected(); + switchClientToNode(2); + final File backupFile = getBackupFile(node2ConfDir); - final NodeDTO node2Dto = getNodeDtoByApiPort(5672); - verifyFlowContentsOnDisk(backupFile); - disconnectNode(node2Dto); verifyInMemoryFlowContents(); - - // Reconnect the node so that we can properly shutdown - reconnectNode(node2Dto); } - - private List getFlowXmlFiles(final File confDir) { - final File[] flowXmlFileArray = confDir.listFiles(file -> file.getName().startsWith("flow") && file.getName().endsWith(".xml.gz")); - final List flowXmlFiles = new ArrayList<>(Arrays.asList(flowXmlFileArray)); - return flowXmlFiles; + private List getFlowJsonFiles(final File confDir) { + final File[] flowJsonFileArray = confDir.listFiles(file -> file.getName().startsWith("flow") && file.getName().endsWith(".json.gz")); + final List flowJsonFiles = new ArrayList<>(Arrays.asList(flowJsonFileArray)); + return flowJsonFiles; } private File getBackupFile(final File confDir) throws InterruptedException { - waitFor(() -> getFlowXmlFiles(confDir).size() == 2); + waitFor(() -> getFlowJsonFiles(confDir).size() == 1); - final List flowXmlFiles = getFlowXmlFiles(confDir); - assertEquals(2, flowXmlFiles.size()); + final List flowJsonFiles = getFlowJsonFiles(confDir); + assertEquals(1, flowJsonFiles.size()); - flowXmlFiles.removeIf(file -> file.getName().equals("flow.xml.gz")); - - assertEquals(1, flowXmlFiles.size()); - final File backupFile = flowXmlFiles.get(0); - return backupFile; + return flowJsonFiles.iterator().next(); } private void verifyFlowContentsOnDisk(final File backupFile) throws IOException { - // Read the flow and make sure that the backup looks the same as the original. We don't just do a byte comparison because the compression may result in different - // gzipped bytes and because if the two flows do differ, we want to have the String representation so that we can compare to see how they are different. - final String flowXml = readFlow(backupFile); - final String expectedFlow = readFlow(new File("src/test/resources/flows/mismatched-flows/flow1.xml.gz")); - - assertEquals(expectedFlow, flowXml); - // Verify some of the values that were persisted to disk final File confDir = backupFile.getParentFile(); - final String loadedFlow = readFlow(new File(confDir, "flow.xml.gz")); + final String loadedFlow = readFlow(new File(confDir, "flow.json.gz")); - final StandardDocumentProvider documentProvider = new StandardDocumentProvider(); - final Document document = documentProvider.parse(new ByteArrayInputStream(loadedFlow.getBytes(StandardCharsets.UTF_8))); - final Element rootElement = (Element) document.getElementsByTagName("flowController").item(0); - final FlowEncodingVersion encodingVersion = FlowEncodingVersion.parse(rootElement); + final ObjectMapper objectMapper = new ObjectMapper(); + final VersionedDataflow versionedDataflow = objectMapper.readValue(loadedFlow, VersionedDataflow.class); + final VersionedProcessGroup rootGroup = versionedDataflow.getRootGroup(); + assertEquals(1, rootGroup.getProcessGroups().size()); - final NiFiInstance node2 = getNiFiInstance().getNodeInstance(2); - final PropertyEncryptor encryptor = createEncryptorFromProperties(node2.getProperties()); - final Element rootGroupElement = (Element) rootElement.getElementsByTagName("rootGroup").item(0); + final VersionedProcessGroup childGroup = rootGroup.getProcessGroups().iterator().next(); + assertFalse(childGroup.getIdentifier().endsWith("00")); - final ProcessGroupDTO groupDto = FlowFromDOMFactory.getProcessGroup(null, rootGroupElement, encryptor, encodingVersion); - final Set childGroupDtos = groupDto.getContents().getProcessGroups(); - assertEquals(1, childGroupDtos.size()); + assertEquals(1, childGroup.getProcessors().size()); + final VersionedProcessor childProcessor = childGroup.getProcessors().iterator().next(); - final ProcessGroupDTO childGroup = childGroupDtos.iterator().next(); - assertFalse(childGroup.getId().endsWith("00")); - final FlowSnippetDTO childContents = childGroup.getContents(); - - final Set childProcessors = childContents.getProcessors(); - assertEquals(1, childProcessors.size()); - - final ProcessorDTO procDto = childProcessors.iterator().next(); - assertFalse(procDto.getId().endsWith("00")); - assertFalse(procDto.getName().endsWith("00")); + assertFalse(childProcessor.getIdentifier().endsWith("00")); + assertFalse(childProcessor.getName().endsWith("00")); } - - private void disconnectNode(final NodeDTO nodeDto) throws NiFiClientException, IOException, InterruptedException { - // Disconnect Node 2 so that we can go to the node directly via the REST API and ensure that the flow is correct. - final NodeEntity nodeEntity = new NodeEntity(); - nodeEntity.setNode(nodeDto); - - getNifiClient().getControllerClient().disconnectNode(nodeDto.getNodeId(), nodeEntity); - - // Give the node a second to disconnect - Thread.sleep(1000L); - } - - private void reconnectNode(final NodeDTO nodeDto) throws NiFiClientException, IOException { - final NodeEntity nodeEntity = new NodeEntity(); - nodeEntity.setNode(nodeDto); - getNifiClient().getControllerClient().connectNode(nodeDto.getNodeId(), nodeEntity); - waitForAllNodesConnected(); - } - - private void verifyInMemoryFlowContents() throws NiFiClientException, IOException, InterruptedException { - final NiFiClient node2Client = createClient(5672); - final ProcessGroupFlowDTO rootGroupFlow = node2Client.getFlowClient().getProcessGroup("root").getProcessGroupFlow(); + private void verifyInMemoryFlowContents() throws NiFiClientException, IOException { + final FlowClient flowClient = getNifiClient().getFlowClient(DO_NOT_REPLICATE); + final ProcessGroupFlowDTO rootGroupFlow = flowClient.getProcessGroup("root").getProcessGroupFlow(); final FlowDTO flowDto = rootGroupFlow.getFlow(); assertEquals(1, flowDto.getProcessGroups().size()); @@ -209,21 +151,13 @@ public class JoinClusterWithDifferentFlow extends NiFiSystemIT { assertEquals("65b6403c-016e-1000-900b-357b13fcc7c4", paramContextReference.getId()); assertEquals("Context 1", paramContextReference.getName()); - ProcessorEntity generateFlowFileEntity = node2Client.getProcessorClient().getProcessor("65b8f293-016e-1000-7b8f-6c6752fa921b"); + final ProcessorEntity generateFlowFileEntity = getNifiClient().getProcessorClient(DO_NOT_REPLICATE).getProcessor("65b8f293-016e-1000-7b8f-6c6752fa921b"); final Map generateProperties = generateFlowFileEntity.getComponent().getConfig().getProperties(); assertEquals("01 B", generateProperties.get("File Size")); assertEquals("1", generateProperties.get("Batch Size")); - assertEquals("1 hour", generateFlowFileEntity.getComponent().getConfig().getSchedulingPeriod()); - String currentState = "RUNNING"; - while ("RUNNING".equals(currentState)) { - Thread.sleep(50L); - generateFlowFileEntity = node2Client.getProcessorClient().getProcessor("65b8f293-016e-1000-7b8f-6c6752fa921b"); - currentState = generateFlowFileEntity.getComponent().getState(); - } - - final ParameterContextDTO contextDto = node2Client.getParamContextClient().getParamContext(paramContextReference.getId(), false).getComponent(); + final ParameterContextDTO contextDto = getNifiClient().getParamContextClient().getParamContext(paramContextReference.getId(), false).getComponent(); assertEquals(2, contextDto.getBoundProcessGroups().size()); assertEquals(1, contextDto.getParameters().size()); final ParameterEntity parameterEntity = contextDto.getParameters().iterator().next(); @@ -237,21 +171,13 @@ public class JoinClusterWithDifferentFlow extends NiFiSystemIT { assertEquals(AffectedComponentDTO.COMPONENT_TYPE_PROCESSOR, affectedComponent.getReferenceType()); // The original Controller Service, whose UUID ended with 00 should be removed and a new one inherited. - final ControllerServicesEntity controllerLevelServices = node2Client.getFlowClient().getControllerServices(); + final ControllerServicesEntity controllerLevelServices = getNifiClient().getFlowClient(DO_NOT_REPLICATE).getControllerServices(); assertEquals(1, controllerLevelServices.getControllerServices().size()); final ControllerServiceEntity firstService = controllerLevelServices.getControllerServices().iterator().next(); assertFalse(firstService.getId().endsWith("00")); } - private PropertyEncryptor createEncryptorFromProperties(Properties properties) { - final NiFiProperties niFiProperties = NiFiProperties.createBasicNiFiProperties(null, properties); - - final String propertiesKey = niFiProperties.getProperty(NiFiProperties.SENSITIVE_PROPS_KEY); - final String propertiesAlgorithm = niFiProperties.getProperty(NiFiProperties.SENSITIVE_PROPS_ALGORITHM); - return new PropertyEncryptorBuilder(propertiesKey).setAlgorithm(propertiesAlgorithm).build(); - } - private String readFlow(final File file) throws IOException { final ByteArrayOutputStream baos = new ByteArrayOutputStream(); diff --git a/nifi-system-tests/nifi-system-test-suite/src/test/java/org/apache/nifi/tests/system/clustering/JoinClusterWithMissingConnectionNoData.java b/nifi-system-tests/nifi-system-test-suite/src/test/java/org/apache/nifi/tests/system/clustering/JoinClusterWithMissingConnectionNoData.java index dc69d12173..adb260b301 100644 --- a/nifi-system-tests/nifi-system-test-suite/src/test/java/org/apache/nifi/tests/system/clustering/JoinClusterWithMissingConnectionNoData.java +++ b/nifi-system-tests/nifi-system-test-suite/src/test/java/org/apache/nifi/tests/system/clustering/JoinClusterWithMissingConnectionNoData.java @@ -29,12 +29,12 @@ public class JoinClusterWithMissingConnectionNoData extends NiFiSystemIT { new InstanceConfiguration.Builder() .bootstrapConfig("src/test/resources/conf/clustered/node1/bootstrap.conf") .instanceDirectory("target/node1") - .flowXml("src/test/resources/flows/missing-connection/with-connection.xml.gz") + .flowJson("src/test/resources/flows/missing-connection/with-connection.json.gz") .build(), new InstanceConfiguration.Builder() .bootstrapConfig("src/test/resources/conf/clustered/node2/bootstrap.conf") .instanceDirectory("target/node2") - .flowXml("src/test/resources/flows/missing-connection/without-connection.xml.gz") + .flowJson("src/test/resources/flows/missing-connection/without-connection.json.gz") .build() ); } diff --git a/nifi-system-tests/nifi-system-test-suite/src/test/java/org/apache/nifi/tests/system/clustering/JoinClusterWithMissingConnectionWithData.java b/nifi-system-tests/nifi-system-test-suite/src/test/java/org/apache/nifi/tests/system/clustering/JoinClusterWithMissingConnectionWithData.java index f46b828b66..ebaede7a9e 100644 --- a/nifi-system-tests/nifi-system-test-suite/src/test/java/org/apache/nifi/tests/system/clustering/JoinClusterWithMissingConnectionWithData.java +++ b/nifi-system-tests/nifi-system-test-suite/src/test/java/org/apache/nifi/tests/system/clustering/JoinClusterWithMissingConnectionWithData.java @@ -41,12 +41,12 @@ public class JoinClusterWithMissingConnectionWithData extends NiFiSystemIT { new InstanceConfiguration.Builder() .bootstrapConfig("src/test/resources/conf/clustered/node1/bootstrap.conf") .instanceDirectory("target/node1") - .flowXml("src/test/resources/flows/missing-connection/with-connection.xml.gz") + .flowJson("src/test/resources/flows/missing-connection/with-connection.json.gz") .build(), new InstanceConfiguration.Builder() .bootstrapConfig("src/test/resources/conf/clustered/node2/bootstrap.conf") .instanceDirectory("target/node2") - .flowXml("src/test/resources/flows/missing-connection/with-connection.xml.gz") + .flowJson("src/test/resources/flows/missing-connection/with-connection.json.gz") .build() ); } diff --git a/nifi-system-tests/nifi-system-test-suite/src/test/java/org/apache/nifi/tests/system/restart/FlowFileRestorationIT.java b/nifi-system-tests/nifi-system-test-suite/src/test/java/org/apache/nifi/tests/system/restart/FlowFileRestorationIT.java index 4a19a07c05..fc02714396 100644 --- a/nifi-system-tests/nifi-system-test-suite/src/test/java/org/apache/nifi/tests/system/restart/FlowFileRestorationIT.java +++ b/nifi-system-tests/nifi-system-test-suite/src/test/java/org/apache/nifi/tests/system/restart/FlowFileRestorationIT.java @@ -63,8 +63,6 @@ public class FlowFileRestorationIT extends NiFiSystemIT { final File nifiHome = nifiInstance.getInstanceDirectory(); final File confDir = new File(nifiHome, "conf"); - final File flowXmlGz = new File(confDir, "flow.xml.gz"); - assertTrue(flowXmlGz.delete()); final File flowJsonGz = new File(confDir, "flow.json.gz"); final byte[] flowJsonGzBytes = Files.readAllBytes(flowJsonGz.toPath()); diff --git a/nifi-system-tests/nifi-system-test-suite/src/test/resources/conf/clustered/node1/nifi.properties b/nifi-system-tests/nifi-system-test-suite/src/test/resources/conf/clustered/node1/nifi.properties index 95d9693a2d..f0702251f1 100644 --- a/nifi-system-tests/nifi-system-test-suite/src/test/resources/conf/clustered/node1/nifi.properties +++ b/nifi-system-tests/nifi-system-test-suite/src/test/resources/conf/clustered/node1/nifi.properties @@ -14,7 +14,7 @@ # limitations under the License. # Core Properties # -nifi.flow.configuration.file=./conf/flow.xml.gz +nifi.flow.configuration.file=./conf/flow.json.gz nifi.flow.configuration.archive.enabled=true nifi.flow.configuration.archive.dir=./conf/archive/ nifi.flow.configuration.archive.max.time=30 days diff --git a/nifi-system-tests/nifi-system-test-suite/src/test/resources/conf/clustered/node2/nifi.properties b/nifi-system-tests/nifi-system-test-suite/src/test/resources/conf/clustered/node2/nifi.properties index 79b7d13026..0f98fe124b 100644 --- a/nifi-system-tests/nifi-system-test-suite/src/test/resources/conf/clustered/node2/nifi.properties +++ b/nifi-system-tests/nifi-system-test-suite/src/test/resources/conf/clustered/node2/nifi.properties @@ -14,7 +14,7 @@ # limitations under the License. # Core Properties # -nifi.flow.configuration.file=./conf/flow.xml.gz +nifi.flow.configuration.file=./conf/flow.json.gz nifi.flow.configuration.archive.enabled=true nifi.flow.configuration.archive.dir=./conf/archive/ nifi.flow.configuration.archive.max.time=30 days diff --git a/nifi-system-tests/nifi-system-test-suite/src/test/resources/conf/default/nifi.properties b/nifi-system-tests/nifi-system-test-suite/src/test/resources/conf/default/nifi.properties index 3b925e4317..bc50975d30 100644 --- a/nifi-system-tests/nifi-system-test-suite/src/test/resources/conf/default/nifi.properties +++ b/nifi-system-tests/nifi-system-test-suite/src/test/resources/conf/default/nifi.properties @@ -14,7 +14,7 @@ # limitations under the License. # Core Properties # -nifi.flow.configuration.file=./conf/flow.xml.gz +nifi.flow.configuration.file=./conf/flow.json.gz nifi.flow.configuration.archive.enabled=true nifi.flow.configuration.archive.dir=./conf/archive/ nifi.flow.configuration.archive.max.time=30 days diff --git a/nifi-system-tests/nifi-system-test-suite/src/test/resources/flows/mismatched-flows/flow1.json.gz b/nifi-system-tests/nifi-system-test-suite/src/test/resources/flows/mismatched-flows/flow1.json.gz new file mode 100644 index 0000000000..334d720c69 Binary files /dev/null and b/nifi-system-tests/nifi-system-test-suite/src/test/resources/flows/mismatched-flows/flow1.json.gz differ diff --git a/nifi-system-tests/nifi-system-test-suite/src/test/resources/flows/mismatched-flows/flow1.xml.gz b/nifi-system-tests/nifi-system-test-suite/src/test/resources/flows/mismatched-flows/flow1.xml.gz deleted file mode 100644 index d49f6cb4ba..0000000000 Binary files a/nifi-system-tests/nifi-system-test-suite/src/test/resources/flows/mismatched-flows/flow1.xml.gz and /dev/null differ diff --git a/nifi-system-tests/nifi-system-test-suite/src/test/resources/flows/mismatched-flows/flow2.json.gz b/nifi-system-tests/nifi-system-test-suite/src/test/resources/flows/mismatched-flows/flow2.json.gz new file mode 100644 index 0000000000..6116ecdbcd Binary files /dev/null and b/nifi-system-tests/nifi-system-test-suite/src/test/resources/flows/mismatched-flows/flow2.json.gz differ diff --git a/nifi-system-tests/nifi-system-test-suite/src/test/resources/flows/mismatched-flows/flow2.xml.gz b/nifi-system-tests/nifi-system-test-suite/src/test/resources/flows/mismatched-flows/flow2.xml.gz deleted file mode 100644 index 26f0f22162..0000000000 Binary files a/nifi-system-tests/nifi-system-test-suite/src/test/resources/flows/mismatched-flows/flow2.xml.gz and /dev/null differ diff --git a/nifi-system-tests/nifi-system-test-suite/src/test/resources/flows/missing-connection/with-connection.json.gz b/nifi-system-tests/nifi-system-test-suite/src/test/resources/flows/missing-connection/with-connection.json.gz new file mode 100644 index 0000000000..5db3f5d8b8 Binary files /dev/null and b/nifi-system-tests/nifi-system-test-suite/src/test/resources/flows/missing-connection/with-connection.json.gz differ diff --git a/nifi-system-tests/nifi-system-test-suite/src/test/resources/flows/missing-connection/with-connection.xml.gz b/nifi-system-tests/nifi-system-test-suite/src/test/resources/flows/missing-connection/with-connection.xml.gz deleted file mode 100644 index 9b07cfcfc8..0000000000 Binary files a/nifi-system-tests/nifi-system-test-suite/src/test/resources/flows/missing-connection/with-connection.xml.gz and /dev/null differ diff --git a/nifi-system-tests/nifi-system-test-suite/src/test/resources/flows/missing-connection/without-connection.json.gz b/nifi-system-tests/nifi-system-test-suite/src/test/resources/flows/missing-connection/without-connection.json.gz new file mode 100644 index 0000000000..d29e9879a4 Binary files /dev/null and b/nifi-system-tests/nifi-system-test-suite/src/test/resources/flows/missing-connection/without-connection.json.gz differ diff --git a/nifi-system-tests/nifi-system-test-suite/src/test/resources/flows/missing-connection/without-connection.xml.gz b/nifi-system-tests/nifi-system-test-suite/src/test/resources/flows/missing-connection/without-connection.xml.gz deleted file mode 100644 index b5615d7c4e..0000000000 Binary files a/nifi-system-tests/nifi-system-test-suite/src/test/resources/flows/missing-connection/without-connection.xml.gz and /dev/null differ diff --git a/nifi-toolkit/nifi-toolkit-encrypt-config/src/main/groovy/org/apache/nifi/properties/ConfigEncryptionTool.groovy b/nifi-toolkit/nifi-toolkit-encrypt-config/src/main/groovy/org/apache/nifi/properties/ConfigEncryptionTool.groovy index cbfe5b920a..4f4717bc8a 100644 --- a/nifi-toolkit/nifi-toolkit-encrypt-config/src/main/groovy/org/apache/nifi/properties/ConfigEncryptionTool.groovy +++ b/nifi-toolkit/nifi-toolkit-encrypt-config/src/main/groovy/org/apache/nifi/properties/ConfigEncryptionTool.groovy @@ -17,7 +17,6 @@ package org.apache.nifi.properties import groovy.io.GroovyPrintWriter - import groovy.xml.XmlSlurper import groovy.xml.XmlUtil import groovy.xml.slurpersupport.GPathResult @@ -32,7 +31,7 @@ import org.apache.commons.codec.binary.Hex import org.apache.nifi.encrypt.PropertyEncryptor import org.apache.nifi.encrypt.PropertyEncryptorBuilder import org.apache.nifi.flow.encryptor.FlowEncryptor -import org.apache.nifi.flow.encryptor.StandardFlowEncryptor +import org.apache.nifi.flow.encryptor.JsonFlowEncryptor import org.apache.nifi.properties.scheme.ProtectionScheme import org.apache.nifi.properties.scheme.StandardProtectionScheme import org.apache.nifi.properties.scheme.StandardProtectionSchemeResolver @@ -50,10 +49,8 @@ import org.xml.sax.SAXException import javax.crypto.BadPaddingException import javax.crypto.Cipher import java.nio.charset.StandardCharsets -import java.nio.file.Files import java.nio.file.Path import java.nio.file.Paths -import java.nio.file.StandardCopyOption import java.security.KeyException import java.util.function.Supplier import java.util.regex.Matcher @@ -71,8 +68,8 @@ class ConfigEncryptionTool { public String outputLoginIdentityProvidersPath public String authorizersPath public String outputAuthorizersPath - public static flowXmlPath - public String outputFlowXmlPath + public static flowJsonPath + public String outputFlowJsonPath static final ProtectionScheme DEFAULT_PROTECTION_SCHEME = new StandardProtectionScheme("aes/gcm") @@ -93,7 +90,7 @@ class ConfigEncryptionTool { private NiFiProperties niFiProperties private String loginIdentityProviders private String authorizers - private InputStream flowXmlInputStream + private InputStream flowJsonInputStream private boolean usingPassword = true private boolean usingPasswordMigration = true @@ -102,7 +99,7 @@ class ConfigEncryptionTool { private boolean handlingNiFiProperties = false private boolean handlingLoginIdentityProviders = false private boolean handlingAuthorizers = false - private boolean handlingFlowXml = false + private boolean handlingFlowJson = false private boolean ignorePropertiesFiles = false private boolean translatingCli = false @@ -115,8 +112,8 @@ class ConfigEncryptionTool { private static final String OUTPUT_LOGIN_IDENTITY_PROVIDERS_ARG = "outputLoginIdentityProviders" private static final String AUTHORIZERS_ARG = "authorizers" private static final String OUTPUT_AUTHORIZERS_ARG = "outputAuthorizers" - private static final String FLOW_XML_ARG = "flowXml" - private static final String OUTPUT_FLOW_XML_ARG = "outputFlowXml" + private static final String FLOW_JSON_ARG = "flowJson" + private static final String OUTPUT_FLOW_JSON_ARG = "outputFlowJson" private static final String KEY_ARG = "key" private static final String PROTECTION_SCHEME_ARG = "protectionScheme" private static final String PASSWORD_ARG = "password" @@ -126,7 +123,7 @@ class ConfigEncryptionTool { private static final String USE_KEY_ARG = "useRawKey" private static final String MIGRATION_ARG = "migrate" private static final String PROPS_KEY_ARG = "propsKey" - private static final String DO_NOT_ENCRYPT_NIFI_PROPERTIES_ARG = "encryptFlowXmlOnly" + private static final String DO_NOT_ENCRYPT_NIFI_PROPERTIES_ARG = "encryptFlowJsonOnly" private static final String NEW_FLOW_ALGORITHM_ARG = "newFlowAlgorithm" private static final String NEW_FLOW_PROVIDER_ARG = "newFlowProvider" private static final String TRANSLATE_CLI_ARG = "translateCli" @@ -162,7 +159,7 @@ class ConfigEncryptionTool { "prompts the user for a root key, and encrypts each value. It will replace the " + "plain value with the protected value in the same file (or write to a new file if " + "specified). It can also be used to migrate already-encrypted values in those " + - "files or in flow.xml.gz to be encrypted with a new key." + "files or in flow.json.gz to be encrypted with a new key." private static final String LDAP_PROVIDER_CLASS = "org.apache.nifi.ldap.LdapProvider" private static @@ -201,9 +198,7 @@ class ConfigEncryptionTool { /(?s)(?:(?!).)*?\s*org\.apache\.nifi\.authorization\.azure\.AzureGraphUserGroupProvider.*?<\/userGroupProvider>/ private static final String XML_DECLARATION_REGEX = /<\?xml version="1.0" encoding="UTF-8"\?>/ - private static final String WRAPPED_FLOW_XML_CIPHER_TEXT_REGEX = /enc\{[a-fA-F0-9]+?\}/ - private static final String DEFAULT_FLOW_ALGORITHM = "NIFI_PBKDF2_AES_GCM_256" private static final Map PROPERTY_KEY_MAP = [ "nifi.security.keystore" : "keystore", @@ -246,8 +241,8 @@ class ConfigEncryptionTool { options.addOption(Option.builder("i").longOpt(OUTPUT_LOGIN_IDENTITY_PROVIDERS_ARG).hasArg(true).argName("file").desc("The destination login-identity-providers.xml file containing protected config values (will not modify input login-identity-providers.xml)").build()) options.addOption(Option.builder("a").longOpt(AUTHORIZERS_ARG).hasArg(true).argName("file").desc("The authorizers.xml file containing unprotected config values (will be overwritten unless -u is specified)").build()) options.addOption(Option.builder("u").longOpt(OUTPUT_AUTHORIZERS_ARG).hasArg(true).argName("file").desc("The destination authorizers.xml file containing protected config values (will not modify input authorizers.xml)").build()) - options.addOption(Option.builder("f").longOpt(FLOW_XML_ARG).hasArg(true).argName("file").desc("The flow.xml.gz file currently protected with old password (will be overwritten unless -g is specified)").build()) - options.addOption(Option.builder("g").longOpt(OUTPUT_FLOW_XML_ARG).hasArg(true).argName("file").desc("The destination flow.xml.gz file containing protected config values (will not modify input flow.xml.gz)").build()) + options.addOption(Option.builder("f").longOpt(FLOW_JSON_ARG).hasArg(true).argName("file").desc("The flow.json.gz file currently protected with old password (will be overwritten unless -g is specified)").build()) + options.addOption(Option.builder("g").longOpt(OUTPUT_FLOW_JSON_ARG).hasArg(true).argName("file").desc("The destination flow.json.gz file containing protected config values (will not modify input flow.json.gz)").build()) options.addOption(Option.builder("b").longOpt(BOOTSTRAP_CONF_ARG).hasArg(true).argName("file").desc("The bootstrap.conf file to persist root key and to optionally provide any configuration for the protection scheme.").build()) options.addOption(Option.builder("S").longOpt(PROTECTION_SCHEME_ARG).hasArg(true).argName("protectionScheme").desc(PROTECTION_SCHEME_DESC).build()) options.addOption(Option.builder("k").longOpt(KEY_ARG).hasArg(true).argName("keyhex").desc("The raw hexadecimal key to use to encrypt the sensitive properties").build()) @@ -257,10 +252,10 @@ class ConfigEncryptionTool { options.addOption(Option.builder("w").longOpt(PASSWORD_MIGRATION_ARG).hasArg(true).argName("password").desc("The old password from which to derive the key during migration").build()) options.addOption(Option.builder("r").longOpt(USE_KEY_ARG).hasArg(false).desc("If provided, the secure console will prompt for the raw key value in hexadecimal form").build()) options.addOption(Option.builder("m").longOpt(MIGRATION_ARG).hasArg(false).desc("If provided, the nifi.properties and/or login-identity-providers.xml sensitive properties will be re-encrypted with the new scheme").build()) - options.addOption(Option.builder("x").longOpt(DO_NOT_ENCRYPT_NIFI_PROPERTIES_ARG).hasArg(false).desc("If provided, the properties in flow.xml.gz will be re-encrypted with a new key but the nifi.properties and/or login-identity-providers.xml files will not be modified").build()) - options.addOption(Option.builder("s").longOpt(PROPS_KEY_ARG).hasArg(true).argName("password|keyhex").desc("The password or key to use to encrypt the sensitive processor properties in flow.xml.gz").build()) - options.addOption(Option.builder("A").longOpt(NEW_FLOW_ALGORITHM_ARG).hasArg(true).argName("algorithm").desc("The algorithm to use to encrypt the sensitive processor properties in flow.xml.gz").build()) - options.addOption(Option.builder("P").longOpt(NEW_FLOW_PROVIDER_ARG).hasArg(true).argName("algorithm").desc("The security provider to use to encrypt the sensitive processor properties in flow.xml.gz").build()) + options.addOption(Option.builder("x").longOpt(DO_NOT_ENCRYPT_NIFI_PROPERTIES_ARG).hasArg(false).desc("If provided, the properties in flow.json.gz will be re-encrypted with a new key but the nifi.properties and/or login-identity-providers.xml files will not be modified").build()) + options.addOption(Option.builder("s").longOpt(PROPS_KEY_ARG).hasArg(true).argName("password|keyhex").desc("The password or key to use to encrypt the sensitive processor properties in flow.json.gz").build()) + options.addOption(Option.builder("A").longOpt(NEW_FLOW_ALGORITHM_ARG).hasArg(true).argName("algorithm").desc("The algorithm to use to encrypt the sensitive processor properties in flow.json.gz").build()) + options.addOption(Option.builder("P").longOpt(NEW_FLOW_PROVIDER_ARG).hasArg(true).argName("algorithm").desc("The security provider to use to encrypt the sensitive processor properties in flow.json.gz").build()) options.addOption(Option.builder("c").longOpt(TRANSLATE_CLI_ARG).hasArg(false).desc("Translates the nifi.properties file to a format suitable for the NiFi CLI tool").build()) options } @@ -383,24 +378,24 @@ class ConfigEncryptionTool { } } - if (commandLine.hasOption(FLOW_XML_ARG)) { + if (commandLine.hasOption(FLOW_JSON_ARG)) { if (isVerbose) { - logger.info("Handling encryption of flow.xml.gz") + logger.info("Handling encryption of flow.json.gz") } - flowXmlPath = commandLine.getOptionValue(FLOW_XML_ARG) - outputFlowXmlPath = commandLine.getOptionValue(OUTPUT_FLOW_XML_ARG, flowXmlPath) - handlingFlowXml = true + flowJsonPath = commandLine.getOptionValue(FLOW_JSON_ARG) + outputFlowJsonPath = commandLine.getOptionValue(OUTPUT_FLOW_JSON_ARG, flowJsonPath) + handlingFlowJson = true newFlowAlgorithm = commandLine.getOptionValue(NEW_FLOW_ALGORITHM_ARG) newFlowProvider = commandLine.getOptionValue(NEW_FLOW_PROVIDER_ARG) - if (flowXmlPath == outputFlowXmlPath) { + if (flowJsonPath == outputFlowJsonPath) { // TODO: Add confirmation pause and provide -y flag to offer no-interaction mode? - logger.warn("The source flow.xml.gz and destination flow.xml.gz are identical [${outputFlowXmlPath}] so the original will be overwritten") + logger.warn("The source flow.json.gz and destination flow.json.gz are identical [${outputFlowJsonPath}] so the original will be overwritten") } if (!commandLine.hasOption(NIFI_PROPERTIES_ARG)) { - printUsageAndThrow("In order to migrate a flow.xml.gz, a nifi.properties file must also be specified via '-n'/'--${NIFI_PROPERTIES_ARG}'.", ExitCode.INVALID_ARGS) + printUsageAndThrow("In order to migrate a flow.json.gz, a nifi.properties file must also be specified via '-n'/'--${NIFI_PROPERTIES_ARG}'.", ExitCode.INVALID_ARGS) } } @@ -412,8 +407,8 @@ class ConfigEncryptionTool { logger.info("(dest) login-identity-providers.xml: ${outputLoginIdentityProvidersPath}") logger.info("(src) authorizers.xml: ${authorizersPath}") logger.info("(dest) authorizers.xml: ${outputAuthorizersPath}") - logger.info("(src) flow.xml.gz: ${flowXmlPath}") - logger.info("(dest) flow.xml.gz: ${outputFlowXmlPath}") + logger.info("(src) flow.json.gz: ${flowJsonPath}") + logger.info("(dest) flow.json.gz: ${outputFlowJsonPath}") } if (!commandLine.hasOption(NIFI_PROPERTIES_ARG) @@ -681,11 +676,11 @@ class ConfigEncryptionTool { /** * Loads the flow definition from the provided file path, handling the GZIP file compression. Unlike {@link #loadLoginIdentityProviders()} this method does not decrypt the content (for performance and separation of concern reasons). * - * @param The path to the XML file + * @param The path to the JSON file * @return The file content - * @throw IOException if the flow.xml.gz file cannot be read + * @throw IOException if the flow.json.gz file cannot be read */ - private InputStream loadFlowXml(String filePath) throws IOException { + private InputStream loadFlowJson(String filePath) throws IOException { if (filePath && (new File(filePath)).exists()) { try { return new GZIPInputStream(new FileInputStream(filePath)) @@ -704,32 +699,32 @@ class ConfigEncryptionTool { } /** - * Scans XML content and decrypts each encrypted element, then re-encrypts it with the new key, and returns the final XML content. + * Scans JSON content and decrypts each encrypted element, then re-encrypts it with the new key, and returns the final JSON content. * - * @param flowXmlContent the original flow.xml.gz as an input stream + * @param flowJsonContent the original flow.json.gz as an input stream * @param existingFlowPassword the existing value of nifi.sensitive.props.key (not a raw key, but rather a password) * @param newFlowPassword the password to use to for encryption (not a raw key, but rather a password) * @param existingAlgorithm the KDF algorithm to use * @param existingProvider the {@link java.security.Provider} to use (defaults to BC) - * @return the encrypted XML content as an InputStream + * @return the encrypted JSON content as an InputStream */ - private InputStream migrateFlowXmlContent(InputStream flowXmlContent, String existingFlowPassword, String newFlowPassword, String existingAlgorithm = DEFAULT_FLOW_ALGORITHM, String newAlgorithm = DEFAULT_FLOW_ALGORITHM) { - File tempFlowXmlFile = new File(getTemporaryFlowXmlFile(outputFlowXmlPath).toString()) - final OutputStream flowOutputStream = getFlowOutputStream(tempFlowXmlFile, flowXmlContent instanceof GZIPInputStream) + private InputStream migrateFlowJsonContent(InputStream flowJsonContent, String existingFlowPassword, String newFlowPassword, String existingAlgorithm = DEFAULT_FLOW_ALGORITHM, String newAlgorithm = DEFAULT_FLOW_ALGORITHM) { + File tempFlowJsonFile = new File(getTemporaryFlowJsonFile(outputFlowJsonPath).toString()) + final OutputStream flowOutputStream = getFlowOutputStream(tempFlowJsonFile, flowJsonContent instanceof GZIPInputStream) final PropertyEncryptor inputEncryptor = new PropertyEncryptorBuilder(existingFlowPassword).setAlgorithm(existingAlgorithm).build() final PropertyEncryptor outputEncryptor = new PropertyEncryptorBuilder(newFlowPassword).setAlgorithm(newAlgorithm).build() - final FlowEncryptor flowEncryptor = new StandardFlowEncryptor() - flowEncryptor.processFlow(flowXmlContent, flowOutputStream, inputEncryptor, outputEncryptor) + final FlowEncryptor flowEncryptor = new JsonFlowEncryptor() + flowEncryptor.processFlow(flowJsonContent, flowOutputStream, inputEncryptor, outputEncryptor) // Overwrite the original flow file with the migrated flow file - Files.move(tempFlowXmlFile.toPath(), Paths.get(outputFlowXmlPath), StandardCopyOption.ATOMIC_MOVE) - loadFlowXml(outputFlowXmlPath) + Files.move(tempFlowJsonFile.toPath(), Paths.get(outputFlowJsonPath), StandardCopyOption.ATOMIC_MOVE) + loadFlowJson(outputFlowJsonPath) } - private static OutputStream getFlowOutputStream(File outputFlowXmlPath, boolean isFileGZipped) { - OutputStream flowOutputStream = new FileOutputStream(outputFlowXmlPath) + private static OutputStream getFlowOutputStream(File outputFlowJsonPath, boolean isFileGZipped) { + OutputStream flowOutputStream = new FileOutputStream(outputFlowJsonPath) if(isFileGZipped) { flowOutputStream = new GZIPOutputStream(flowOutputStream) } @@ -737,10 +732,10 @@ class ConfigEncryptionTool { } // Create a temporary output file we can write the stream to - private static Path getTemporaryFlowXmlFile(String originalOutputFlowXmlPath) { - String outputFilename = Paths.get(originalOutputFlowXmlPath).getFileName().toString() + private static Path getTemporaryFlowJsonFile(String originalOutputFlowJsonPath) { + String outputFilename = Paths.get(originalOutputFlowJsonPath).getFileName().toString() String migratedFileName = "migrated-${outputFilename}" - Paths.get(originalOutputFlowXmlPath).resolveSibling(migratedFileName) + Paths.get(originalOutputFlowJsonPath).resolveSibling(migratedFileName) } private SensitivePropertyProviderFactory getSensitivePropertyProviderFactory(final String keyHex) { @@ -1373,8 +1368,8 @@ class ConfigEncryptionTool { } boolean existingNiFiPropertiesAreEncrypted = tool.niFiPropertiesAreEncrypted() - if (!tool.ignorePropertiesFiles || (tool.handlingFlowXml && existingNiFiPropertiesAreEncrypted)) { - // If we are handling the flow.xml.gz and nifi.properties is already encrypted, try getting the key from bootstrap.conf rather than the console + if (!tool.ignorePropertiesFiles || (tool.handlingFlowJson && existingNiFiPropertiesAreEncrypted)) { + // If we are handling the flow.json.gz and nifi.properties is already encrypted, try getting the key from bootstrap.conf rather than the console if (tool.ignorePropertiesFiles) { tool.keyHex = NiFiBootstrapUtils.extractKeyFromBootstrapFile(tool.bootstrapConfPath) } else { @@ -1412,8 +1407,8 @@ class ConfigEncryptionTool { } String existingKeyHex = tool.migrationKeyHex ?: tool.keyHex - // Load NiFiProperties for either scenario; only encrypt if "handling" (see after flow XML) - if (tool.handlingNiFiProperties || tool.handlingFlowXml) { + // Load NiFiProperties for either scenario; only encrypt if "handling" (see after flow JSON) + if (tool.handlingNiFiProperties || tool.handlingFlowJson) { try { tool.niFiProperties = tool.loadNiFiProperties(existingKeyHex) } catch (Exception e) { @@ -1442,14 +1437,14 @@ class ConfigEncryptionTool { tool.authorizers = tool.encryptAuthorizers(tool.authorizers) } - if (tool.handlingFlowXml) { + if (tool.handlingFlowJson) { try { - tool.flowXmlInputStream = tool.loadFlowXml(flowXmlPath) + tool.flowJsonInputStream = tool.loadFlowJson(flowJsonPath) } catch (Exception e) { if (tool.isVerbose) { logger.error("Encountered an error: ", e) } - tool.printUsageAndThrow("Cannot load flow.xml.gz", ExitCode.ERROR_READING_NIFI_PROPERTIES) + tool.printUsageAndThrow("Cannot load flow.json.gz", ExitCode.ERROR_READING_NIFI_PROPERTIES) } } @@ -1477,10 +1472,10 @@ class ConfigEncryptionTool { if (!tool.ignorePropertiesFiles) { tool.writeKeyToBootstrapConf() } - if (tool.handlingFlowXml) { - tool.handleFlowXml(tool.niFiPropertiesAreEncrypted()) + if (tool.handlingFlowJson) { + tool.handleFlowJson(tool.niFiPropertiesAreEncrypted()) } - if (tool.handlingNiFiProperties || tool.handlingFlowXml) { + if (tool.handlingNiFiProperties || tool.handlingFlowJson) { tool.writeNiFiProperties() } if (tool.handlingLoginIdentityProviders) { @@ -1503,7 +1498,7 @@ class ConfigEncryptionTool { System.exit(ExitCode.SUCCESS.ordinal()) } - void handleFlowXml(boolean existingNiFiPropertiesAreEncrypted = false) { + void handleFlowJson(boolean existingNiFiPropertiesAreEncrypted = false) { String existingFlowPassword = existingFlowPropertiesPassword ?: getExistingFlowPassword() // If the new password was not provided in the arguments, read from the console. If that is empty, use the same value (essentially a copy no-op) @@ -1519,12 +1514,12 @@ class ConfigEncryptionTool { String newAlgorithm = newFlowAlgorithm ?: existingAlgorithm try { - logger.info("Migrating flow.xml file at ${flowXmlPath}. This could take a while if the flow XML is very large.") - migrateFlowXmlContent(flowXmlInputStream, existingFlowPassword, newFlowPassword, existingAlgorithm, newAlgorithm) + logger.info("Migrating flow.json file at ${flowJsonPath}. This could take a while if the flow JSON is very large.") + migrateFlowJsonContent(flowJsonInputStream, existingFlowPassword, newFlowPassword, existingAlgorithm, newAlgorithm) } catch (Exception e) { logger.error("Encountered an error: ${e.getLocalizedMessage()}") if (e instanceof BadPaddingException) { - logger.error("This error is likely caused by providing the wrong existing flow password. Check that the existing flow password [-p] is the one used to encrypt the provided flow.xml.gz file") + logger.error("This error is likely caused by providing the wrong existing flow password. Check that the existing flow password [-p] is the one used to encrypt the provided flow.json.gz file") } if (isVerbose) { logger.error("Exception: ", e) @@ -1549,7 +1544,7 @@ class ConfigEncryptionTool { // Manually update the protection scheme or it will be lost rawProperties.put(ApplicationPropertiesProtector.getProtectionKey(NiFiProperties.SENSITIVE_PROPS_KEY), spp.getIdentifierKey()) if (isVerbose) { - logger.info("Tool is not configured to encrypt nifi.properties, but the existing nifi.properties is encrypted and flow.xml.gz was migrated, so manually persisting the new encrypted value to nifi.properties") + logger.info("Tool is not configured to encrypt nifi.properties, but the existing nifi.properties is encrypted and flow.json.gz was migrated, so manually persisting the new encrypted value to nifi.properties") } } else { rawProperties.put(NiFiProperties.SENSITIVE_PROPS_KEY, newFlowPassword) diff --git a/nifi-toolkit/nifi-toolkit-encrypt-config/src/test/groovy/org/apache/nifi/properties/ConfigEncryptionToolTest.groovy b/nifi-toolkit/nifi-toolkit-encrypt-config/src/test/groovy/org/apache/nifi/properties/ConfigEncryptionToolTest.groovy index 8153517fc8..89568e9256 100644 --- a/nifi-toolkit/nifi-toolkit-encrypt-config/src/test/groovy/org/apache/nifi/properties/ConfigEncryptionToolTest.groovy +++ b/nifi-toolkit/nifi-toolkit-encrypt-config/src/test/groovy/org/apache/nifi/properties/ConfigEncryptionToolTest.groovy @@ -88,8 +88,6 @@ class ConfigEncryptionToolTest { private final String PASSWORD_PROP_REGEX = "]* name=\".* Password\"" private final String SECRET_PROP_REGEX = "]* name=\".* Secret\"" - private static final String WFXCTR = ConfigEncryptionTool.WRAPPED_FLOW_XML_CIPHER_TEXT_REGEX - @BeforeClass static void setUpOnce() throws Exception { Assume.assumeTrue("Test only runs on *nix", !SystemUtils.IS_OS_WINDOWS) @@ -3290,7 +3288,7 @@ class ConfigEncryptionToolTest { // Assert assert !tool.handlingNiFiProperties assert !tool.handlingLoginIdentityProviders - assert tool.handlingFlowXml + assert tool.handlingFlowJson } @Test @@ -3766,8 +3764,8 @@ class ConfigEncryptionToolTest { void testFindFieldsInStream() { def verifyTool = new ConfigEncryptionTool() verifyTool.isVerbose = true - verifyTool.flowXmlPath = new File("src/test/resources/flow.xml.gz").path - InputStream updatedFlowXmlContent = verifyTool.loadFlowXml(verifyTool.flowXmlPath) + verifyTool.flowJsonPath = new File("src/test/resources/flow.xml.gz").path + InputStream updatedFlowXmlContent = verifyTool.loadFlowXml(verifyTool.flowJsonPath) Set fieldsFound = findFieldsInStream(updatedFlowXmlContent, WFXCTR) assert(fieldsFound.size() > 0) } diff --git a/nifi-toolkit/nifi-toolkit-encrypt-config/src/test/resources/nifi_default.properties b/nifi-toolkit/nifi-toolkit-encrypt-config/src/test/resources/nifi_default.properties index 4278f161a5..8922c4e89c 100644 --- a/nifi-toolkit/nifi-toolkit-encrypt-config/src/test/resources/nifi_default.properties +++ b/nifi-toolkit/nifi-toolkit-encrypt-config/src/test/resources/nifi_default.properties @@ -14,7 +14,7 @@ # limitations under the License. # Core Properties # -nifi.flow.configuration.file=./target/flow.xml.gz +nifi.flow.configuration.file=./target/flow.json.gz nifi.flow.configuration.archive.dir=./target/archive/ nifi.flowcontroller.autoResumeState=true nifi.flowcontroller.graceful.shutdown.period=10 sec diff --git a/nifi-toolkit/nifi-toolkit-encrypt-config/src/test/resources/nifi_with_sensitive_properties_protected_aes.properties b/nifi-toolkit/nifi-toolkit-encrypt-config/src/test/resources/nifi_with_sensitive_properties_protected_aes.properties index 7e56790de8..4b419b427a 100644 --- a/nifi-toolkit/nifi-toolkit-encrypt-config/src/test/resources/nifi_with_sensitive_properties_protected_aes.properties +++ b/nifi-toolkit/nifi-toolkit-encrypt-config/src/test/resources/nifi_with_sensitive_properties_protected_aes.properties @@ -14,7 +14,7 @@ # limitations under the License. # Core Properties # -nifi.flow.configuration.file=./target/flow.xml.gz +nifi.flow.configuration.file=./target/flow.json.gz nifi.flow.configuration.archive.dir=./target/archive/ nifi.flowcontroller.autoResumeState=true nifi.flowcontroller.graceful.shutdown.period=10 sec diff --git a/nifi-toolkit/nifi-toolkit-encrypt-config/src/test/resources/nifi_with_sensitive_properties_protected_aes_128.properties b/nifi-toolkit/nifi-toolkit-encrypt-config/src/test/resources/nifi_with_sensitive_properties_protected_aes_128.properties index bac1f34985..777f0d03f8 100644 --- a/nifi-toolkit/nifi-toolkit-encrypt-config/src/test/resources/nifi_with_sensitive_properties_protected_aes_128.properties +++ b/nifi-toolkit/nifi-toolkit-encrypt-config/src/test/resources/nifi_with_sensitive_properties_protected_aes_128.properties @@ -14,7 +14,7 @@ # limitations under the License. # Core Properties # -nifi.flow.configuration.file=./target/flow.xml.gz +nifi.flow.configuration.file=./target/flow.json.gz nifi.flow.configuration.archive.dir=./target/archive/ nifi.flowcontroller.autoResumeState=true nifi.flowcontroller.graceful.shutdown.period=10 sec diff --git a/nifi-toolkit/nifi-toolkit-encrypt-config/src/test/resources/nifi_with_sensitive_properties_protected_aes_password.properties b/nifi-toolkit/nifi-toolkit-encrypt-config/src/test/resources/nifi_with_sensitive_properties_protected_aes_password.properties index de6ca74cc1..62df5a541a 100644 --- a/nifi-toolkit/nifi-toolkit-encrypt-config/src/test/resources/nifi_with_sensitive_properties_protected_aes_password.properties +++ b/nifi-toolkit/nifi-toolkit-encrypt-config/src/test/resources/nifi_with_sensitive_properties_protected_aes_password.properties @@ -14,7 +14,7 @@ # limitations under the License. # Core Properties # -nifi.flow.configuration.file=./target/flow.xml.gz +nifi.flow.configuration.file=./target/flow.json.gz nifi.flow.configuration.archive.dir=./target/archive/ nifi.flowcontroller.autoResumeState=true nifi.flowcontroller.graceful.shutdown.period=10 sec diff --git a/nifi-toolkit/nifi-toolkit-encrypt-config/src/test/resources/nifi_with_sensitive_properties_protected_aes_password_128.properties b/nifi-toolkit/nifi-toolkit-encrypt-config/src/test/resources/nifi_with_sensitive_properties_protected_aes_password_128.properties index 8655345132..43002ef121 100644 --- a/nifi-toolkit/nifi-toolkit-encrypt-config/src/test/resources/nifi_with_sensitive_properties_protected_aes_password_128.properties +++ b/nifi-toolkit/nifi-toolkit-encrypt-config/src/test/resources/nifi_with_sensitive_properties_protected_aes_password_128.properties @@ -14,7 +14,7 @@ # limitations under the License. # Core Properties # -nifi.flow.configuration.file=./target/flow.xml.gz +nifi.flow.configuration.file=./target/flow.json.gz nifi.flow.configuration.archive.dir=./target/archive/ nifi.flowcontroller.autoResumeState=true nifi.flowcontroller.graceful.shutdown.period=10 sec diff --git a/nifi-toolkit/nifi-toolkit-encrypt-config/src/test/resources/nifi_with_sensitive_properties_unprotected.properties b/nifi-toolkit/nifi-toolkit-encrypt-config/src/test/resources/nifi_with_sensitive_properties_unprotected.properties index 4e62acf55f..962b9b2b7d 100644 --- a/nifi-toolkit/nifi-toolkit-encrypt-config/src/test/resources/nifi_with_sensitive_properties_unprotected.properties +++ b/nifi-toolkit/nifi-toolkit-encrypt-config/src/test/resources/nifi_with_sensitive_properties_unprotected.properties @@ -14,7 +14,7 @@ # limitations under the License. # Core Properties # -nifi.flow.configuration.file=./target/flow.xml.gz +nifi.flow.configuration.file=./target/flow.json.gz nifi.flow.configuration.archive.dir=./target/archive/ nifi.flowcontroller.autoResumeState=true nifi.flowcontroller.graceful.shutdown.period=10 sec diff --git a/nifi-toolkit/nifi-toolkit-encrypt-config/src/test/resources/nifi_with_sensitive_properties_unprotected_and_empty_protection_schemes.properties b/nifi-toolkit/nifi-toolkit-encrypt-config/src/test/resources/nifi_with_sensitive_properties_unprotected_and_empty_protection_schemes.properties index b68f1a77ed..2d1a0e66a5 100644 --- a/nifi-toolkit/nifi-toolkit-encrypt-config/src/test/resources/nifi_with_sensitive_properties_unprotected_and_empty_protection_schemes.properties +++ b/nifi-toolkit/nifi-toolkit-encrypt-config/src/test/resources/nifi_with_sensitive_properties_unprotected_and_empty_protection_schemes.properties @@ -14,7 +14,7 @@ # limitations under the License. # Core Properties # -nifi.flow.configuration.file=./target/flow.xml.gz +nifi.flow.configuration.file=./target/flow.json.gz nifi.flow.configuration.archive.dir=./target/archive/ nifi.flowcontroller.autoResumeState=true nifi.flowcontroller.graceful.shutdown.period=10 sec diff --git a/nifi-toolkit/nifi-toolkit-tls/src/test/resources/localhost/nifi.properties b/nifi-toolkit/nifi-toolkit-tls/src/test/resources/localhost/nifi.properties index 451bb603c1..9d5f6b483c 100644 --- a/nifi-toolkit/nifi-toolkit-tls/src/test/resources/localhost/nifi.properties +++ b/nifi-toolkit/nifi-toolkit-tls/src/test/resources/localhost/nifi.properties @@ -17,7 +17,7 @@ # Core Properties # nifi.fake.property=fake value -nifi.flow.configuration.file=./conf/flow.xml.gz +nifi.flow.configuration.file=./conf/flow.json.gz nifi.flow.configuration.archive.dir=./conf/archive/ nifi.flowcontroller.autoResumeState=true nifi.flowcontroller.graceful.shutdown.period=10 sec