From 3aa02a022e154416cc6f69acc327aeb26f2a89b2 Mon Sep 17 00:00:00 2001 From: Ferenc Kis Date: Tue, 25 Apr 2023 14:25:29 +0200 Subject: [PATCH] NIFI-11514 Flow JSON support and deprecating YAML format. Revised parameter generation. Generic refactors. Signed-off-by: Ferenc Erdei This closes #7344 --- .../apache/nifi/c2/client/api/C2Client.java | 11 +- .../apache/nifi/c2/client/C2ClientConfig.java | 36 + .../nifi/c2/client/http/C2HttpClient.java | 64 +- .../nifi/c2/client/http/C2HttpClientTest.java | 4 +- .../TransferDebugOperationHandler.java | 60 +- .../UpdateAssetOperationHandler.java | 10 +- .../UpdateConfigurationOperationHandler.java | 32 +- .../UpdateConfigurationStrategy.java | 26 +- .../UpdateAssetOperationHandlerTest.java | 6 +- ...dateConfigurationOperationHandlerTest.java | 13 +- minifi/minifi-assembly/README.md | 4 +- minifi/minifi-bootstrap/pom.xml | 8 +- .../nifi/minifi/bootstrap/RunMiNiFi.java | 6 +- .../minifi/bootstrap/command/StartRunner.java | 104 +- .../ConfigurationChangeCoordinator.java | 118 +- .../ConfigurationChangeListener.java | 3 +- .../ConfigurationChangeNotifier.java | 8 +- .../WholeConfigDifferentiator.java | 61 +- .../ingestors/AbstractPullChangeIngestor.java | 13 +- .../ingestors/FileChangeIngestor.java | 241 ++- .../ingestors/PullHttpChangeIngestor.java | 343 ++-- .../ingestors/RestChangeIngestor.java | 89 +- .../bootstrap/service/BootstrapCodec.java | 31 +- .../service/BootstrapFileProvider.java | 8 - .../MiNiFiConfigurationChangeListener.java | 138 +- .../service/MiNiFiExecCommandProvider.java | 204 ++- .../service/MiNiFiPropertiesGenerator.java | 276 +++ .../PeriodicStatusReporterManager.java | 9 +- .../service/UpdateConfigurationService.java | 95 - .../service/UpdatePropertiesService.java | 14 +- .../status/reporters/StatusLogger.java | 40 +- .../bootstrap/util/BootstrapTransformer.java | 102 -- .../bootstrap/util/ConfigTransformer.java | 856 --------- .../util/ConfigTransformerException.java | 32 - .../bootstrap/util/OrderedProperties.java | 3 +- .../bootstrap/util/ParentGroupIdResolver.java | 101 -- .../nifi/minifi/bootstrap/RunMiNiFiTest.java | 129 -- .../WholeConfigDifferentiatorTest.java | 30 +- .../ingestors/FileChangeIngestorTest.java | 44 +- .../PullHttpChangeIngestorCommonTest.java | 84 +- .../PullHttpChangeIngestorSSLTest.java | 16 +- .../ingestors/PullHttpChangeIngestorTest.java | 16 +- .../RestChangeIngestorCommonTest.java | 39 +- .../ingestors/RestChangeIngestorSSLTest.java | 48 +- .../ingestors/RestChangeIngestorTest.java | 6 +- .../bootstrap/service/BootstrapCodecTest.java | 16 - .../MiNiFiPropertiesGeneratorTest.java | 267 +++ .../status/reporters/StatusLoggerTest.java | 54 +- .../bootstrap/util/ConfigTransformerTest.java | 809 --------- .../bootstrap/util/OrderedPropertiesTest.java | 4 +- .../util/ParentGroupIdResolverTest.java | 200 --- minifi/minifi-c2/README.md | 5 +- minifi/minifi-c2/minifi-c2-assembly/README.md | 2 +- minifi/minifi-c2/minifi-c2-assembly/pom.xml | 14 +- .../main/resources/conf/minifi-c2-context.xml | 20 +- .../files/raspi3/config.test.json.v1 | 55 + .../resources/files/raspi3/config.text.yml.v1 | 63 - .../FileSystemConfigurationCacheTest.java | 13 +- .../test/resources/files/config.test.json.v1 | 55 + .../test/resources/files/config.text.yaml.v1 | 63 - .../minifi-c2-docker/dockerhub/Dockerfile | 2 +- .../minifi-c2-docker/dockermaven/Dockerfile | 2 +- .../minifi-c2-integration-tests/pom.xml | 17 +- .../integration/test/AbstractTestSecure.java | 77 +- .../test/AbstractTestUnsecure.java | 52 +- ...FiRestConfigurationProviderSecureTest.java | 118 -- ...RestConfigurationProviderUnsecureTest.java | 39 - .../c2-secure-rest/conf/minifi-c2-context.xml | 60 - .../conf/minifi-c2-context.xml | 59 - .../files/raspi2/config.application.json.v1 | 40 + .../c2/files/raspi2/config.text.yml.v1 | 63 - .../files/raspi3/config.application.json.v1 | 40 + .../files/raspi3/config.application.json.v2 | 40 + .../c2/files/raspi3/config.text.yml.v1 | 63 - .../c2/files/raspi3/config.text.yml.v2 | 63 - ...iFiRestConfigurationProviderSecureTest.yml | 55 - ...iRestConfigurationProviderUnsecureTest.yml | 37 - .../test/resources/mocknifi-secure/server.py | 48 - .../resources/mocknifi-unsecure/server.py | 44 - .../mocknifi/www/nifi-api/flow/templates | 53 - .../download | 203 --- .../download | 203 --- .../download | 202 --- .../DelegatingConfigurationProvider.java | 5 - .../DelegatingConfigurationProviderTest.java | 2 +- .../rest/NiFiRestConfigurationProvider.java | 188 -- .../provider/nifi/rest/TemplatesIterator.java | 116 -- .../NiFiRestConfigurationProviderTest.java | 66 - .../nifi/rest/TemplatesIteratorTest.java | 110 -- .../src/test/resources/noTemplates.json | 1 - .../src/test/resources/oneTemplate.json | 21 - .../src/test/resources/twoTemplates.json | 37 - minifi/minifi-c2/minifi-c2-provider/pom.xml | 1 - .../minifi-commons/minifi-commons-api/pom.xml | 3 +- .../minifi/commons/api/MiNiFiConstants.java | 4 +- .../minifi/commons/api}/MiNiFiProperties.java | 141 +- .../minifi-commons-framework/.gitignore | 38 + .../minifi-commons-framework}/pom.xml | 55 +- .../commons/service/FlowEnrichService.java | 290 +++ .../minifi/commons/util/FlowUpdateUtils.java | 88 + .../service/FlowEnrichServiceTest.java | 233 +++ .../src/test/resources/default_flow.json | 40 + .../org.mockito.plugins.MockMaker | 1 + .../schema/common/BootstrapPropertyKeys.java | 123 -- minifi/minifi-commons/pom.xml | 2 +- minifi/minifi-docker/dockerhub/Dockerfile | 2 +- minifi/minifi-docker/dockerhub/README.md | 8 +- minifi/minifi-docker/dockermaven/Dockerfile | 2 +- minifi/minifi-docker/pom.xml | 2 +- .../src/main/markdown/System_Admin_Guide.md | 10 +- .../markdown/minifi-java-agent-quick-start.md | 58 +- minifi/minifi-integration-tests/pom.xml | 3 +- .../c2/C2ProtocolIntegrationTest.java | 8 +- .../c2/HierarchicalC2IntegrationTest.java | 24 +- ...eYamlTest.java => StandaloneJsonTest.java} | 62 +- .../standalone/test/StandaloneXmlTest.java | 51 - .../src/test/resources/bootstrap.conf | 171 +- .../conf/minifi-c2-context.xml | 2 +- .../edge1/raspi3/config.application.json.v1 | 40 + .../files/edge1/raspi3/config.text.yml.v1 | 63 - .../edge2/raspi2/config.application.json.v1 | 40 + .../files/edge2/raspi2/config.text.yml.v1 | 63 - .../edge3/raspi3/config.application.json.v1 | 40 + .../files/edge3/raspi3/config.text.yml.v1 | 63 - .../hierarchical/minifi-edge1/bootstrap.conf | 174 +- .../hierarchical/minifi-edge1/expected.json | 2 +- .../hierarchical/minifi-edge2/bootstrap.conf | 172 +- .../hierarchical/minifi-edge2/expected.json | 2 +- .../hierarchical/minifi-edge3/bootstrap.conf | 172 +- .../hierarchical/minifi-edge3/expected.json | 2 +- .../conf/minifi-c2-context.xml | 2 +- .../files/raspi3/config.application.json.v1 | 163 ++ .../files/raspi3/config.text.yml.v1 | 129 -- .../files/raspi4/config.application.json.v1 | 163 ++ .../files/raspi4/config.text.yml.v1 | 129 -- .../c2/protocol/minifi-edge1/bootstrap.conf | 180 +- .../c2/protocol/minifi-edge2/bootstrap.conf | 179 +- .../c2/protocol/minifi-edge3/bootstrap.conf | 179 +- ...lone.yml => docker-compose-standalone.yml} | 2 +- .../standalone/CsvToJson/CsvToJson.json | 415 +++++ .../CsvToJson/yml => CsvToJson}/expected.json | 0 .../DecompressionCircularFlow.json | 920 ++++++++++ .../expected.json | 0 .../MiNiFiTailLogAttribute.json | 168 ++ .../expected.json | 0 .../MultipleRelationships.json | 310 ++++ .../expected.json | 0 .../ProcessGroups/ProcessGroups.json | 375 ++++ .../xml => ProcessGroups}/expected.json | 0 ...TextExpressionLanguageCSVReformatting.json | 324 ++++ .../expected.json | 0 .../StressTestFramework.json | 508 ++++++ .../xml => StressTestFramework}/expected.json | 0 .../standalone/v1/CsvToJson/xml/CsvToJson.xml | 400 ----- .../standalone/v1/CsvToJson/xml/expected.json | 6 - .../standalone/v1/CsvToJson/yml/CsvToJson.yml | 178 -- .../xml/DecompressionCircularFlow.xml | 757 -------- .../xml/expected.json | 6 - .../yml/DecompressionCircularFlow.yml | 293 --- .../xml/MiNiFiTailLogAttribute.xml | 134 -- .../MiNiFiTailLogAttribute/xml/expected.json | 6 - .../yml/MiNiFiTailLogAttribute.yml | 98 - ...eTextExpressionLanguageCSVReformatting.xml | 270 --- .../xml/expected.json | 6 - ...eTextExpressionLanguageCSVReformatting.yml | 146 -- .../xml/MultipleRelationships.xml | 340 ---- .../yml/MultipleRelationships.yml | 153 -- .../MultipleRelationships/yml/expected.json | 6 - .../v2/ProcessGroups/xml/ProcessGroups.xml | 370 ---- .../v2/ProcessGroups/yml/ProcessGroups.yml | 163 -- .../v2/ProcessGroups/yml/expected.json | 6 - .../xml/StressTestFramework.xml | 539 ------ .../yml/StressTestFramework.yml | 211 --- .../v2/StressTestFramework/yml/expected.json | 6 - .../minifi-framework-core/pom.xml | 5 + .../nifi/minifi/c2/C2NifiClientService.java | 269 ++- .../DefaultUpdateConfigurationStrategy.java | 226 +++ .../command/TransferDebugCommandHelper.java | 15 +- .../UpdatePropertiesPropertyProvider.java | 6 +- ...efaultUpdateConfigurationStrategyTest.java | 175 ++ .../c2/command/PropertiesPersisterTest.java | 2 +- .../UpdatePropertiesPropertyProviderTest.java | 12 +- .../minifi-framework/minifi-resources/pom.xml | 9 + .../src/main/resources/conf/bootstrap.conf | 213 ++- .../src/main/resources/conf/config.yml | 62 - .../src/main/resources/conf/flow.json.raw | 38 + .../java/org/apache/nifi/minifi/MiNiFi.java | 19 +- .../nifi/minifi/StandardMiNiFiServer.java | 50 +- .../minifi-toolkit-assembly/README.md | 14 +- .../minifi-toolkit-configuration/pom.xml | 6 +- .../toolkit/configuration/ConfigMain.java | 296 +-- .../ConfigTransformException.java} | 19 +- .../configuration/PathInputStreamFactory.java | 1 + .../PathOutputStreamFactory.java | 1 + .../dto/ConfigSchemaFunction.java | 147 -- .../dto/ConnectionSchemaFunction.java | 63 - .../dto/ControllerServiceSchemaFunction.java | 51 - .../dto/FlowControllerSchemaFunction.java | 36 - .../dto/FlowSnippetDTOEnricher.java | 174 -- .../dto/FunnelSchemaFunction.java | 38 - .../configuration/dto/PortSchemaFunction.java | 46 - .../dto/ProcessorSchemaFunction.java | 72 - .../dto/RemotePortSchemaFunction.java | 46 - .../dto/RemoteProcessGroupSchemaFunction.java | 82 - .../json/ComponentPropertyProvider.java | 164 ++ ...gSchemaToVersionedDataFlowTransformer.java | 467 +++++ .../json/TransformYamlCommandFactory.java | 170 ++ .../registry/NiFiRegConfigSchemaFunction.java | 158 -- .../NiFiRegConnectionSchemaFunction.java | 63 - ...iFiRegControllerServiceSchemaFunction.java | 51 - .../NiFiRegFlowControllerSchemaFunction.java | 51 - .../registry/NiFiRegFunnelSchemaFunction.java | 35 - .../registry/NiFiRegPortSchemaFunction.java | 43 - .../NiFiRegProcessorSchemaFunction.java | 68 - .../NiFiRegRemotePortSchemaFunction.java | 45 - ...FiRegRemoteProcessGroupSchemaFunction.java | 76 - .../VersionedProcessGroupEnricher.java | 159 -- .../toolkit/configuration/ConfigMainTest.java | 487 ----- .../configuration/dto/BaseSchemaTester.java | 55 - .../dto/ConnectionSchemaTest.java | 179 -- .../dto/FlowControllerSchemaTest.java | 70 - .../dto/PortSchemaFunctionTest.java | 75 - .../dto/ProcessorSchemaTest.java | 234 --- .../dto/RemoteInputPortSchemaTest.java | 104 -- .../dto/RemoteProcessGroupSchemaTest.java | 176 -- .../src/test/resources/1.5_RPG_Handling.xml | 173 -- .../src/test/resources/1.5_RPG_Handling.yml | 122 -- .../src/test/resources/CsvToJson-v1.yml | 177 -- .../src/test/resources/CsvToJson.xml | 20 - .../src/test/resources/CsvToJson.yml | 208 --- .../DecompressionCircularFlow-v1.yml | 292 --- .../resources/DecompressionCircularFlow.xml | 18 - .../resources/DecompressionCircularFlow.yml | 347 ---- .../InvokeHttpMiNiFiTemplateTest-v1.yml | 261 --- .../InvokeHttpMiNiFiTemplateTest-v2.yml | 296 --- .../InvokeHttpMiNiFiTemplateTest.xml | 650 ------- .../InvokeHttpMiNiFiTemplateTest.yml | 315 ---- .../test/resources/MINIFI-496/dual_rpgs.xml | 1583 ----------------- .../MINIFI-521_1.3_TemplateEncoding.xml | 219 --- .../MINIFI-521_1.3_TemplateEncoding.yml | 134 -- .../test/resources/MultipleRelationships.xml | 339 ---- .../test/resources/MultipleRelationships.yml | 165 -- .../src/test/resources/MultipleUriRPG.xml | 215 --- .../src/test/resources/MultipleUriRPG.yml | 131 -- .../resources/NestedControllerServices.xml | 1089 ------------ .../resources/NestedControllerServices.yml | 301 ---- .../resources/NoTemplateEncodingVersion.xml | 214 --- .../resources/NoTemplateEncodingVersion.yml | 131 -- ...ProcessGroupsAndRemoteProcessGroups-v2.yml | 287 --- .../ProcessGroupsAndRemoteProcessGroups.xml | 650 ------- .../ProcessGroupsAndRemoteProcessGroups.yml | 309 ---- ...xtExpressionLanguageCSVReformatting-v1.yml | 146 -- ...eTextExpressionLanguageCSVReformatting.xml | 20 - ...eTextExpressionLanguageCSVReformatting.yml | 173 -- .../resources/SimpleRPGToLogAttributes.xml | 149 -- .../resources/SimpleRPGToLogAttributes.yml | 121 -- .../test/resources/SimpleTailFileToRPG-v1.yml | 102 -- .../test/resources/SimpleTailFileToRPG-v2.yml | 113 -- .../test/resources/SimpleTailFileToRPG.xml | 214 --- .../test/resources/SimpleTailFileToRPG.yml | 131 -- .../test/resources/StressTestFramework-v1.yml | 114 -- .../test/resources/StressTestFramework.xml | 18 - .../test/resources/StressTestFramework.yml | 138 -- .../resources/StressTestFrameworkFunnel.xml | 540 ------ .../resources/StressTestFrameworkFunnel.yml | 223 --- .../src/test/resources/TemplateWithFunnel.xml | 18 - .../test/resources/TemplateWithInputPort.xml | 18 - .../test/resources/TemplateWithOutputPort.xml | 18 - .../resources/TemplateWithProcessGroup.xml | 19 - .../VersionedFlowSnapshot-Simple.json | 770 -------- .../VersionedFlowSnapshot-Simple.yml | 299 ---- .../test/resources/config-malformed-field.yml | 108 -- .../src/test/resources/config-v1.yml | 110 -- .../src/test/resources/config.yml | 112 -- .../minifi-toolkit-schema}/pom.xml | 8 +- .../ComponentStatusRepositorySchema.java | 8 +- .../minifi/toolkit}/schema/ConfigSchema.java | 32 +- .../toolkit}/schema/ConnectionSchema.java | 6 +- .../schema/ContentRepositorySchema.java | 10 +- .../schema/ControllerServiceSchema.java | 12 +- .../toolkit}/schema/CorePropertiesSchema.java | 10 +- .../toolkit}/schema/FlowControllerSchema.java | 12 +- .../schema/FlowFileRepositorySchema.java | 12 +- .../minifi/toolkit}/schema/FunnelSchema.java | 4 +- .../minifi/toolkit}/schema/PortSchema.java | 4 +- .../toolkit}/schema/ProcessGroupSchema.java | 28 +- .../toolkit}/schema/ProcessorSchema.java | 20 +- .../schema/ProvenanceReportingSchema.java | 22 +- .../schema/ProvenanceRepositorySchema.java | 8 +- .../toolkit}/schema/RemotePortSchema.java | 10 +- .../schema/RemoteProcessGroupSchema.java | 14 +- .../toolkit}/schema/ReportingSchema.java | 18 +- .../schema/SecurityPropertiesSchema.java | 12 +- .../toolkit}/schema/SensitivePropsSchema.java | 8 +- .../minifi/toolkit}/schema/SwapSchema.java | 8 +- .../toolkit}/schema/common/BaseSchema.java | 4 +- .../schema/common/BaseSchemaWithId.java | 4 +- .../common/BaseSchemaWithIdAndName.java | 4 +- .../schema/common/CollectionOverlap.java | 2 +- .../schema/common/CollectionUtil.java | 2 +- .../schema/common/CommonPropertyKeys.java | 2 +- .../schema/common/ConvertableSchema.java | 2 +- .../minifi/toolkit}/schema/common/Schema.java | 2 +- .../toolkit}/schema/common/StringUtil.java | 2 +- .../schema/common/WritableSchema.java | 2 +- .../SchemaInstantiatonException.java | 2 +- .../exception/SchemaLoaderException.java | 2 +- .../serialization/ConfigRepresenter.java | 2 +- .../schema/serialization/SchemaLoader.java | 14 +- .../schema/serialization/SchemaSaver.java | 4 +- .../toolkit}/schema/v1/ConfigSchemaV1.java | 62 +- .../schema/v1/ConnectionSchemaV1.java | 30 +- .../toolkit}/schema/v1/ProcessorSchemaV1.java | 46 +- .../schema/v1/RemoteProcessGroupSchemaV1.java | 22 +- .../toolkit}/schema/v2/ConfigSchemaV2.java | 52 +- .../schema/v2/CorePropertiesSchemaV2.java | 12 +- .../schema/v2/ProcessGroupSchemaV2.java | 42 +- .../schema/v2/RemoteProcessGroupSchemaV2.java | 34 +- .../toolkit}/schema/ConfigSchemaTest.java | 10 +- .../toolkit}/schema/ConnectionSchemaTest.java | 4 +- .../schema/ProcessGroupSchemaTest.java | 4 +- .../schema/RemoteProcessGroupSchemaTest.java | 6 +- .../schema/SecurityPropertiesSchemaTest.java | 2 +- .../serialization/SchemaLoaderTest.java | 12 +- .../schema/v1/ConfigSchemaV1Test.java | 18 +- .../schema/v1/ConnectionSchemaV1Test.java | 12 +- .../schema/v1/ProcessorSchemaV1Test.java | 34 +- .../v2/RemoteProcessGroupSchemaV2Test.java | 10 +- .../src/test/resources/config-minimal-v2.yml | 0 .../src/test/resources/config-minimal-v3.yml | 0 .../src/test/resources/config-minimal.yml | 0 minifi/minifi-toolkit/pom.xml | 1 + minifi/pom.xml | 10 + .../nifi/headless/HeadlessNiFiServer.java | 10 +- 334 files changed, 9097 insertions(+), 26937 deletions(-) rename minifi/minifi-c2/minifi-c2-provider/minifi-c2-provider-nifi-rest/src/test/java/org/apache/nifi/minifi/c2/provider/nifi/rest/TemplatesIteratorExceptionTest.java => c2/c2-client-bundle/c2-client-service/src/main/java/org/apache/nifi/c2/client/service/operation/UpdateConfigurationStrategy.java (63%) create mode 100644 minifi/minifi-bootstrap/src/main/java/org/apache/nifi/minifi/bootstrap/service/MiNiFiPropertiesGenerator.java delete mode 100644 minifi/minifi-bootstrap/src/main/java/org/apache/nifi/minifi/bootstrap/service/UpdateConfigurationService.java delete mode 100644 minifi/minifi-bootstrap/src/main/java/org/apache/nifi/minifi/bootstrap/util/BootstrapTransformer.java delete mode 100644 minifi/minifi-bootstrap/src/main/java/org/apache/nifi/minifi/bootstrap/util/ConfigTransformer.java delete mode 100644 minifi/minifi-bootstrap/src/main/java/org/apache/nifi/minifi/bootstrap/util/ConfigTransformerException.java delete mode 100644 minifi/minifi-bootstrap/src/main/java/org/apache/nifi/minifi/bootstrap/util/ParentGroupIdResolver.java delete mode 100644 minifi/minifi-bootstrap/src/test/java/org/apache/nifi/minifi/bootstrap/RunMiNiFiTest.java rename minifi/minifi-bootstrap/src/test/java/org/apache/nifi/minifi/bootstrap/configuration/ingestors/{common => }/PullHttpChangeIngestorCommonTest.java (70%) rename minifi/minifi-bootstrap/src/test/java/org/apache/nifi/minifi/bootstrap/configuration/ingestors/{common => }/RestChangeIngestorCommonTest.java (84%) create mode 100644 minifi/minifi-bootstrap/src/test/java/org/apache/nifi/minifi/bootstrap/service/MiNiFiPropertiesGeneratorTest.java delete mode 100644 minifi/minifi-bootstrap/src/test/java/org/apache/nifi/minifi/bootstrap/util/ConfigTransformerTest.java delete mode 100644 minifi/minifi-bootstrap/src/test/java/org/apache/nifi/minifi/bootstrap/util/ParentGroupIdResolverTest.java create mode 100644 minifi/minifi-c2/minifi-c2-assembly/src/main/resources/files/raspi3/config.test.json.v1 delete mode 100644 minifi/minifi-c2/minifi-c2-assembly/src/main/resources/files/raspi3/config.text.yml.v1 create mode 100644 minifi/minifi-c2/minifi-c2-cache/minifi-c2-cache-filesystem/src/test/resources/files/config.test.json.v1 delete mode 100644 minifi/minifi-c2/minifi-c2-cache/minifi-c2-cache-filesystem/src/test/resources/files/config.text.yaml.v1 delete mode 100644 minifi/minifi-c2/minifi-c2-integration-tests/src/test/java/org/apache/nifi/minifi/c2/integration/test/NiFiRestConfigurationProviderSecureTest.java delete mode 100644 minifi/minifi-c2/minifi-c2-integration-tests/src/test/java/org/apache/nifi/minifi/c2/integration/test/NiFiRestConfigurationProviderUnsecureTest.java delete mode 100644 minifi/minifi-c2/minifi-c2-integration-tests/src/test/resources/c2-secure-rest/conf/minifi-c2-context.xml delete mode 100644 minifi/minifi-c2/minifi-c2-integration-tests/src/test/resources/c2-unsecure-rest/conf/minifi-c2-context.xml create mode 100644 minifi/minifi-c2/minifi-c2-integration-tests/src/test/resources/c2/files/raspi2/config.application.json.v1 delete mode 100644 minifi/minifi-c2/minifi-c2-integration-tests/src/test/resources/c2/files/raspi2/config.text.yml.v1 create mode 100644 minifi/minifi-c2/minifi-c2-integration-tests/src/test/resources/c2/files/raspi3/config.application.json.v1 create mode 100644 minifi/minifi-c2/minifi-c2-integration-tests/src/test/resources/c2/files/raspi3/config.application.json.v2 delete mode 100644 minifi/minifi-c2/minifi-c2-integration-tests/src/test/resources/c2/files/raspi3/config.text.yml.v1 delete mode 100644 minifi/minifi-c2/minifi-c2-integration-tests/src/test/resources/c2/files/raspi3/config.text.yml.v2 delete mode 100644 minifi/minifi-c2/minifi-c2-integration-tests/src/test/resources/docker-compose-NiFiRestConfigurationProviderSecureTest.yml delete mode 100644 minifi/minifi-c2/minifi-c2-integration-tests/src/test/resources/docker-compose-NiFiRestConfigurationProviderUnsecureTest.yml delete mode 100644 minifi/minifi-c2/minifi-c2-integration-tests/src/test/resources/mocknifi-secure/server.py delete mode 100644 minifi/minifi-c2/minifi-c2-integration-tests/src/test/resources/mocknifi-unsecure/server.py delete mode 100644 minifi/minifi-c2/minifi-c2-integration-tests/src/test/resources/mocknifi/www/nifi-api/flow/templates delete mode 100644 minifi/minifi-c2/minifi-c2-integration-tests/src/test/resources/mocknifi/www/nifi-api/templates/87048385-a6ca-42fe-b2d8-6a563cedd036/download delete mode 100644 minifi/minifi-c2/minifi-c2-integration-tests/src/test/resources/mocknifi/www/nifi-api/templates/dd737a3e-333e-40df-a0bc-d7e28c8e6843/download delete mode 100644 minifi/minifi-c2/minifi-c2-integration-tests/src/test/resources/mocknifi/www/nifi-api/templates/f080ec50-ca32-4b36-8453-5a7145bec4c5/download delete mode 100644 minifi/minifi-c2/minifi-c2-provider/minifi-c2-provider-nifi-rest/src/main/java/org/apache/nifi/minifi/c2/provider/nifi/rest/NiFiRestConfigurationProvider.java delete mode 100644 minifi/minifi-c2/minifi-c2-provider/minifi-c2-provider-nifi-rest/src/main/java/org/apache/nifi/minifi/c2/provider/nifi/rest/TemplatesIterator.java delete mode 100644 minifi/minifi-c2/minifi-c2-provider/minifi-c2-provider-nifi-rest/src/test/java/org/apache/nifi/minifi/c2/provider/nifi/rest/NiFiRestConfigurationProviderTest.java delete mode 100644 minifi/minifi-c2/minifi-c2-provider/minifi-c2-provider-nifi-rest/src/test/java/org/apache/nifi/minifi/c2/provider/nifi/rest/TemplatesIteratorTest.java delete mode 100644 minifi/minifi-c2/minifi-c2-provider/minifi-c2-provider-nifi-rest/src/test/resources/noTemplates.json delete mode 100644 minifi/minifi-c2/minifi-c2-provider/minifi-c2-provider-nifi-rest/src/test/resources/oneTemplate.json delete mode 100644 minifi/minifi-c2/minifi-c2-provider/minifi-c2-provider-nifi-rest/src/test/resources/twoTemplates.json rename minifi/{minifi-nar-bundles/minifi-framework-bundle/minifi-framework/minifi-framework-core/src/main/java/org/apache/nifi/minifi => minifi-commons/minifi-commons-api/src/main/java/org/apache/nifi/minifi/commons/api}/MiNiFiProperties.java (72%) create mode 100644 minifi/minifi-commons/minifi-commons-framework/.gitignore rename minifi/{minifi-c2/minifi-c2-provider/minifi-c2-provider-nifi-rest => minifi-commons/minifi-commons-framework}/pom.xml (52%) create mode 100644 minifi/minifi-commons/minifi-commons-framework/src/main/java/org/apache/nifi/minifi/commons/service/FlowEnrichService.java create mode 100644 minifi/minifi-commons/minifi-commons-framework/src/main/java/org/apache/nifi/minifi/commons/util/FlowUpdateUtils.java create mode 100644 minifi/minifi-commons/minifi-commons-framework/src/test/java/org/apache/nifi/minifi/commons/service/FlowEnrichServiceTest.java create mode 100644 minifi/minifi-commons/minifi-commons-framework/src/test/resources/default_flow.json create mode 100644 minifi/minifi-commons/minifi-commons-framework/src/test/resources/mockito-extensions/org.mockito.plugins.MockMaker delete mode 100644 minifi/minifi-commons/minifi-commons-schema/src/main/java/org/apache/nifi/minifi/commons/schema/common/BootstrapPropertyKeys.java rename minifi/minifi-integration-tests/src/test/java/org/apache/nifi/minifi/integration/standalone/test/{StandaloneYamlTest.java => StandaloneJsonTest.java} (57%) delete mode 100644 minifi/minifi-integration-tests/src/test/java/org/apache/nifi/minifi/integration/standalone/test/StandaloneXmlTest.java create mode 100644 minifi/minifi-integration-tests/src/test/resources/c2/hierarchical/c2-authoritative/files/edge1/raspi3/config.application.json.v1 delete mode 100644 minifi/minifi-integration-tests/src/test/resources/c2/hierarchical/c2-authoritative/files/edge1/raspi3/config.text.yml.v1 create mode 100644 minifi/minifi-integration-tests/src/test/resources/c2/hierarchical/c2-authoritative/files/edge2/raspi2/config.application.json.v1 delete mode 100644 minifi/minifi-integration-tests/src/test/resources/c2/hierarchical/c2-authoritative/files/edge2/raspi2/config.text.yml.v1 create mode 100644 minifi/minifi-integration-tests/src/test/resources/c2/hierarchical/c2-authoritative/files/edge3/raspi3/config.application.json.v1 delete mode 100644 minifi/minifi-integration-tests/src/test/resources/c2/hierarchical/c2-authoritative/files/edge3/raspi3/config.text.yml.v1 create mode 100644 minifi/minifi-integration-tests/src/test/resources/c2/protocol/c2-authoritative/files/raspi3/config.application.json.v1 delete mode 100644 minifi/minifi-integration-tests/src/test/resources/c2/protocol/c2-authoritative/files/raspi3/config.text.yml.v1 create mode 100644 minifi/minifi-integration-tests/src/test/resources/c2/protocol/c2-authoritative/files/raspi4/config.application.json.v1 delete mode 100644 minifi/minifi-integration-tests/src/test/resources/c2/protocol/c2-authoritative/files/raspi4/config.text.yml.v1 rename minifi/minifi-integration-tests/src/test/resources/{docker-compose-v1-standalone.yml => docker-compose-standalone.yml} (98%) create mode 100644 minifi/minifi-integration-tests/src/test/resources/standalone/CsvToJson/CsvToJson.json rename minifi/minifi-integration-tests/src/test/resources/standalone/{v1/CsvToJson/yml => CsvToJson}/expected.json (100%) create mode 100644 minifi/minifi-integration-tests/src/test/resources/standalone/DecompressionCircularFlow/DecompressionCircularFlow.json rename minifi/minifi-integration-tests/src/test/resources/standalone/{v1/DecompressionCircularFlow/yml => DecompressionCircularFlow}/expected.json (100%) create mode 100644 minifi/minifi-integration-tests/src/test/resources/standalone/MiNiFiTailLogAttribute/MiNiFiTailLogAttribute.json rename minifi/minifi-integration-tests/src/test/resources/standalone/{v1/MiNiFiTailLogAttribute/yml => MiNiFiTailLogAttribute}/expected.json (100%) create mode 100644 minifi/minifi-integration-tests/src/test/resources/standalone/MultipleRelationships/MultipleRelationships.json rename minifi/minifi-integration-tests/src/test/resources/standalone/{v2/MultipleRelationships/xml => MultipleRelationships}/expected.json (100%) create mode 100644 minifi/minifi-integration-tests/src/test/resources/standalone/ProcessGroups/ProcessGroups.json rename minifi/minifi-integration-tests/src/test/resources/standalone/{v2/ProcessGroups/xml => ProcessGroups}/expected.json (100%) create mode 100644 minifi/minifi-integration-tests/src/test/resources/standalone/ReplaceTextExpressionLanguageCSVReformatting/ReplaceTextExpressionLanguageCSVReformatting.json rename minifi/minifi-integration-tests/src/test/resources/standalone/{v1/ReplaceTextExpressionLanguageCSVReformatting/yml => ReplaceTextExpressionLanguageCSVReformatting}/expected.json (100%) create mode 100644 minifi/minifi-integration-tests/src/test/resources/standalone/StressTestFramework/StressTestFramework.json rename minifi/minifi-integration-tests/src/test/resources/standalone/{v2/StressTestFramework/xml => StressTestFramework}/expected.json (100%) delete mode 100644 minifi/minifi-integration-tests/src/test/resources/standalone/v1/CsvToJson/xml/CsvToJson.xml delete mode 100644 minifi/minifi-integration-tests/src/test/resources/standalone/v1/CsvToJson/xml/expected.json delete mode 100644 minifi/minifi-integration-tests/src/test/resources/standalone/v1/CsvToJson/yml/CsvToJson.yml delete mode 100644 minifi/minifi-integration-tests/src/test/resources/standalone/v1/DecompressionCircularFlow/xml/DecompressionCircularFlow.xml delete mode 100644 minifi/minifi-integration-tests/src/test/resources/standalone/v1/DecompressionCircularFlow/xml/expected.json delete mode 100644 minifi/minifi-integration-tests/src/test/resources/standalone/v1/DecompressionCircularFlow/yml/DecompressionCircularFlow.yml delete mode 100644 minifi/minifi-integration-tests/src/test/resources/standalone/v1/MiNiFiTailLogAttribute/xml/MiNiFiTailLogAttribute.xml delete mode 100644 minifi/minifi-integration-tests/src/test/resources/standalone/v1/MiNiFiTailLogAttribute/xml/expected.json delete mode 100644 minifi/minifi-integration-tests/src/test/resources/standalone/v1/MiNiFiTailLogAttribute/yml/MiNiFiTailLogAttribute.yml delete mode 100644 minifi/minifi-integration-tests/src/test/resources/standalone/v1/ReplaceTextExpressionLanguageCSVReformatting/xml/ReplaceTextExpressionLanguageCSVReformatting.xml delete mode 100644 minifi/minifi-integration-tests/src/test/resources/standalone/v1/ReplaceTextExpressionLanguageCSVReformatting/xml/expected.json delete mode 100644 minifi/minifi-integration-tests/src/test/resources/standalone/v1/ReplaceTextExpressionLanguageCSVReformatting/yml/ReplaceTextExpressionLanguageCSVReformatting.yml delete mode 100644 minifi/minifi-integration-tests/src/test/resources/standalone/v2/MultipleRelationships/xml/MultipleRelationships.xml delete mode 100644 minifi/minifi-integration-tests/src/test/resources/standalone/v2/MultipleRelationships/yml/MultipleRelationships.yml delete mode 100644 minifi/minifi-integration-tests/src/test/resources/standalone/v2/MultipleRelationships/yml/expected.json delete mode 100644 minifi/minifi-integration-tests/src/test/resources/standalone/v2/ProcessGroups/xml/ProcessGroups.xml delete mode 100644 minifi/minifi-integration-tests/src/test/resources/standalone/v2/ProcessGroups/yml/ProcessGroups.yml delete mode 100644 minifi/minifi-integration-tests/src/test/resources/standalone/v2/ProcessGroups/yml/expected.json delete mode 100644 minifi/minifi-integration-tests/src/test/resources/standalone/v2/StressTestFramework/xml/StressTestFramework.xml delete mode 100644 minifi/minifi-integration-tests/src/test/resources/standalone/v2/StressTestFramework/yml/StressTestFramework.yml delete mode 100644 minifi/minifi-integration-tests/src/test/resources/standalone/v2/StressTestFramework/yml/expected.json create mode 100644 minifi/minifi-nar-bundles/minifi-framework-bundle/minifi-framework/minifi-framework-core/src/main/java/org/apache/nifi/minifi/c2/command/DefaultUpdateConfigurationStrategy.java create mode 100644 minifi/minifi-nar-bundles/minifi-framework-bundle/minifi-framework/minifi-framework-core/src/test/java/org/apache/nifi/minifi/c2/command/DefaultUpdateConfigurationStrategyTest.java delete mode 100644 minifi/minifi-nar-bundles/minifi-framework-bundle/minifi-framework/minifi-resources/src/main/resources/conf/config.yml create mode 100644 minifi/minifi-nar-bundles/minifi-framework-bundle/minifi-framework/minifi-resources/src/main/resources/conf/flow.json.raw rename minifi/{minifi-c2/minifi-c2-provider/minifi-c2-provider-nifi-rest/src/main/java/org/apache/nifi/minifi/c2/provider/nifi/rest/TemplatesIteratorException.java => minifi-toolkit/minifi-toolkit-configuration/src/main/java/org/apache/nifi/minifi/toolkit/configuration/ConfigTransformException.java} (69%) delete mode 100644 minifi/minifi-toolkit/minifi-toolkit-configuration/src/main/java/org/apache/nifi/minifi/toolkit/configuration/dto/ConfigSchemaFunction.java delete mode 100644 minifi/minifi-toolkit/minifi-toolkit-configuration/src/main/java/org/apache/nifi/minifi/toolkit/configuration/dto/ConnectionSchemaFunction.java delete mode 100644 minifi/minifi-toolkit/minifi-toolkit-configuration/src/main/java/org/apache/nifi/minifi/toolkit/configuration/dto/ControllerServiceSchemaFunction.java delete mode 100644 minifi/minifi-toolkit/minifi-toolkit-configuration/src/main/java/org/apache/nifi/minifi/toolkit/configuration/dto/FlowControllerSchemaFunction.java delete mode 100644 minifi/minifi-toolkit/minifi-toolkit-configuration/src/main/java/org/apache/nifi/minifi/toolkit/configuration/dto/FlowSnippetDTOEnricher.java delete mode 100644 minifi/minifi-toolkit/minifi-toolkit-configuration/src/main/java/org/apache/nifi/minifi/toolkit/configuration/dto/FunnelSchemaFunction.java delete mode 100644 minifi/minifi-toolkit/minifi-toolkit-configuration/src/main/java/org/apache/nifi/minifi/toolkit/configuration/dto/PortSchemaFunction.java delete mode 100644 minifi/minifi-toolkit/minifi-toolkit-configuration/src/main/java/org/apache/nifi/minifi/toolkit/configuration/dto/ProcessorSchemaFunction.java delete mode 100644 minifi/minifi-toolkit/minifi-toolkit-configuration/src/main/java/org/apache/nifi/minifi/toolkit/configuration/dto/RemotePortSchemaFunction.java delete mode 100644 minifi/minifi-toolkit/minifi-toolkit-configuration/src/main/java/org/apache/nifi/minifi/toolkit/configuration/dto/RemoteProcessGroupSchemaFunction.java create mode 100644 minifi/minifi-toolkit/minifi-toolkit-configuration/src/main/java/org/apache/nifi/minifi/toolkit/configuration/json/ComponentPropertyProvider.java create mode 100644 minifi/minifi-toolkit/minifi-toolkit-configuration/src/main/java/org/apache/nifi/minifi/toolkit/configuration/json/ConfigSchemaToVersionedDataFlowTransformer.java create mode 100644 minifi/minifi-toolkit/minifi-toolkit-configuration/src/main/java/org/apache/nifi/minifi/toolkit/configuration/json/TransformYamlCommandFactory.java delete mode 100644 minifi/minifi-toolkit/minifi-toolkit-configuration/src/main/java/org/apache/nifi/minifi/toolkit/configuration/registry/NiFiRegConfigSchemaFunction.java delete mode 100644 minifi/minifi-toolkit/minifi-toolkit-configuration/src/main/java/org/apache/nifi/minifi/toolkit/configuration/registry/NiFiRegConnectionSchemaFunction.java delete mode 100644 minifi/minifi-toolkit/minifi-toolkit-configuration/src/main/java/org/apache/nifi/minifi/toolkit/configuration/registry/NiFiRegControllerServiceSchemaFunction.java delete mode 100644 minifi/minifi-toolkit/minifi-toolkit-configuration/src/main/java/org/apache/nifi/minifi/toolkit/configuration/registry/NiFiRegFlowControllerSchemaFunction.java delete mode 100644 minifi/minifi-toolkit/minifi-toolkit-configuration/src/main/java/org/apache/nifi/minifi/toolkit/configuration/registry/NiFiRegFunnelSchemaFunction.java delete mode 100644 minifi/minifi-toolkit/minifi-toolkit-configuration/src/main/java/org/apache/nifi/minifi/toolkit/configuration/registry/NiFiRegPortSchemaFunction.java delete mode 100644 minifi/minifi-toolkit/minifi-toolkit-configuration/src/main/java/org/apache/nifi/minifi/toolkit/configuration/registry/NiFiRegProcessorSchemaFunction.java delete mode 100644 minifi/minifi-toolkit/minifi-toolkit-configuration/src/main/java/org/apache/nifi/minifi/toolkit/configuration/registry/NiFiRegRemotePortSchemaFunction.java delete mode 100644 minifi/minifi-toolkit/minifi-toolkit-configuration/src/main/java/org/apache/nifi/minifi/toolkit/configuration/registry/NiFiRegRemoteProcessGroupSchemaFunction.java delete mode 100644 minifi/minifi-toolkit/minifi-toolkit-configuration/src/main/java/org/apache/nifi/minifi/toolkit/configuration/registry/VersionedProcessGroupEnricher.java delete mode 100644 minifi/minifi-toolkit/minifi-toolkit-configuration/src/test/java/org/apache/nifi/minifi/toolkit/configuration/ConfigMainTest.java delete mode 100644 minifi/minifi-toolkit/minifi-toolkit-configuration/src/test/java/org/apache/nifi/minifi/toolkit/configuration/dto/BaseSchemaTester.java delete mode 100644 minifi/minifi-toolkit/minifi-toolkit-configuration/src/test/java/org/apache/nifi/minifi/toolkit/configuration/dto/ConnectionSchemaTest.java delete mode 100644 minifi/minifi-toolkit/minifi-toolkit-configuration/src/test/java/org/apache/nifi/minifi/toolkit/configuration/dto/FlowControllerSchemaTest.java delete mode 100644 minifi/minifi-toolkit/minifi-toolkit-configuration/src/test/java/org/apache/nifi/minifi/toolkit/configuration/dto/PortSchemaFunctionTest.java delete mode 100644 minifi/minifi-toolkit/minifi-toolkit-configuration/src/test/java/org/apache/nifi/minifi/toolkit/configuration/dto/ProcessorSchemaTest.java delete mode 100644 minifi/minifi-toolkit/minifi-toolkit-configuration/src/test/java/org/apache/nifi/minifi/toolkit/configuration/dto/RemoteInputPortSchemaTest.java delete mode 100644 minifi/minifi-toolkit/minifi-toolkit-configuration/src/test/java/org/apache/nifi/minifi/toolkit/configuration/dto/RemoteProcessGroupSchemaTest.java delete mode 100644 minifi/minifi-toolkit/minifi-toolkit-configuration/src/test/resources/1.5_RPG_Handling.xml delete mode 100644 minifi/minifi-toolkit/minifi-toolkit-configuration/src/test/resources/1.5_RPG_Handling.yml delete mode 100644 minifi/minifi-toolkit/minifi-toolkit-configuration/src/test/resources/CsvToJson-v1.yml delete mode 100644 minifi/minifi-toolkit/minifi-toolkit-configuration/src/test/resources/CsvToJson.xml delete mode 100644 minifi/minifi-toolkit/minifi-toolkit-configuration/src/test/resources/CsvToJson.yml delete mode 100644 minifi/minifi-toolkit/minifi-toolkit-configuration/src/test/resources/DecompressionCircularFlow-v1.yml delete mode 100644 minifi/minifi-toolkit/minifi-toolkit-configuration/src/test/resources/DecompressionCircularFlow.xml delete mode 100644 minifi/minifi-toolkit/minifi-toolkit-configuration/src/test/resources/DecompressionCircularFlow.yml delete mode 100644 minifi/minifi-toolkit/minifi-toolkit-configuration/src/test/resources/InvokeHttpMiNiFiTemplateTest-v1.yml delete mode 100644 minifi/minifi-toolkit/minifi-toolkit-configuration/src/test/resources/InvokeHttpMiNiFiTemplateTest-v2.yml delete mode 100644 minifi/minifi-toolkit/minifi-toolkit-configuration/src/test/resources/InvokeHttpMiNiFiTemplateTest.xml delete mode 100644 minifi/minifi-toolkit/minifi-toolkit-configuration/src/test/resources/InvokeHttpMiNiFiTemplateTest.yml delete mode 100644 minifi/minifi-toolkit/minifi-toolkit-configuration/src/test/resources/MINIFI-496/dual_rpgs.xml delete mode 100644 minifi/minifi-toolkit/minifi-toolkit-configuration/src/test/resources/MINIFI-521_1.3_TemplateEncoding.xml delete mode 100644 minifi/minifi-toolkit/minifi-toolkit-configuration/src/test/resources/MINIFI-521_1.3_TemplateEncoding.yml delete mode 100644 minifi/minifi-toolkit/minifi-toolkit-configuration/src/test/resources/MultipleRelationships.xml delete mode 100644 minifi/minifi-toolkit/minifi-toolkit-configuration/src/test/resources/MultipleRelationships.yml delete mode 100644 minifi/minifi-toolkit/minifi-toolkit-configuration/src/test/resources/MultipleUriRPG.xml delete mode 100644 minifi/minifi-toolkit/minifi-toolkit-configuration/src/test/resources/MultipleUriRPG.yml delete mode 100644 minifi/minifi-toolkit/minifi-toolkit-configuration/src/test/resources/NestedControllerServices.xml delete mode 100644 minifi/minifi-toolkit/minifi-toolkit-configuration/src/test/resources/NestedControllerServices.yml delete mode 100644 minifi/minifi-toolkit/minifi-toolkit-configuration/src/test/resources/NoTemplateEncodingVersion.xml delete mode 100644 minifi/minifi-toolkit/minifi-toolkit-configuration/src/test/resources/NoTemplateEncodingVersion.yml delete mode 100644 minifi/minifi-toolkit/minifi-toolkit-configuration/src/test/resources/ProcessGroupsAndRemoteProcessGroups-v2.yml delete mode 100644 minifi/minifi-toolkit/minifi-toolkit-configuration/src/test/resources/ProcessGroupsAndRemoteProcessGroups.xml delete mode 100644 minifi/minifi-toolkit/minifi-toolkit-configuration/src/test/resources/ProcessGroupsAndRemoteProcessGroups.yml delete mode 100644 minifi/minifi-toolkit/minifi-toolkit-configuration/src/test/resources/ReplaceTextExpressionLanguageCSVReformatting-v1.yml delete mode 100644 minifi/minifi-toolkit/minifi-toolkit-configuration/src/test/resources/ReplaceTextExpressionLanguageCSVReformatting.xml delete mode 100644 minifi/minifi-toolkit/minifi-toolkit-configuration/src/test/resources/ReplaceTextExpressionLanguageCSVReformatting.yml delete mode 100644 minifi/minifi-toolkit/minifi-toolkit-configuration/src/test/resources/SimpleRPGToLogAttributes.xml delete mode 100644 minifi/minifi-toolkit/minifi-toolkit-configuration/src/test/resources/SimpleRPGToLogAttributes.yml delete mode 100644 minifi/minifi-toolkit/minifi-toolkit-configuration/src/test/resources/SimpleTailFileToRPG-v1.yml delete mode 100644 minifi/minifi-toolkit/minifi-toolkit-configuration/src/test/resources/SimpleTailFileToRPG-v2.yml delete mode 100644 minifi/minifi-toolkit/minifi-toolkit-configuration/src/test/resources/SimpleTailFileToRPG.xml delete mode 100644 minifi/minifi-toolkit/minifi-toolkit-configuration/src/test/resources/SimpleTailFileToRPG.yml delete mode 100644 minifi/minifi-toolkit/minifi-toolkit-configuration/src/test/resources/StressTestFramework-v1.yml delete mode 100644 minifi/minifi-toolkit/minifi-toolkit-configuration/src/test/resources/StressTestFramework.xml delete mode 100644 minifi/minifi-toolkit/minifi-toolkit-configuration/src/test/resources/StressTestFramework.yml delete mode 100644 minifi/minifi-toolkit/minifi-toolkit-configuration/src/test/resources/StressTestFrameworkFunnel.xml delete mode 100644 minifi/minifi-toolkit/minifi-toolkit-configuration/src/test/resources/StressTestFrameworkFunnel.yml delete mode 100644 minifi/minifi-toolkit/minifi-toolkit-configuration/src/test/resources/TemplateWithFunnel.xml delete mode 100644 minifi/minifi-toolkit/minifi-toolkit-configuration/src/test/resources/TemplateWithInputPort.xml delete mode 100644 minifi/minifi-toolkit/minifi-toolkit-configuration/src/test/resources/TemplateWithOutputPort.xml delete mode 100644 minifi/minifi-toolkit/minifi-toolkit-configuration/src/test/resources/TemplateWithProcessGroup.xml delete mode 100644 minifi/minifi-toolkit/minifi-toolkit-configuration/src/test/resources/VersionedFlowSnapshot-Simple.json delete mode 100644 minifi/minifi-toolkit/minifi-toolkit-configuration/src/test/resources/VersionedFlowSnapshot-Simple.yml delete mode 100644 minifi/minifi-toolkit/minifi-toolkit-configuration/src/test/resources/config-malformed-field.yml delete mode 100644 minifi/minifi-toolkit/minifi-toolkit-configuration/src/test/resources/config-v1.yml delete mode 100644 minifi/minifi-toolkit/minifi-toolkit-configuration/src/test/resources/config.yml rename minifi/{minifi-commons/minifi-commons-schema => minifi-toolkit/minifi-toolkit-schema}/pom.xml (87%) rename minifi/{minifi-commons/minifi-commons-schema/src/main/java/org/apache/nifi/minifi/commons => minifi-toolkit/minifi-toolkit-schema/src/main/java/org/apache/nifi/minifi/toolkit}/schema/ComponentStatusRepositorySchema.java (90%) rename minifi/{minifi-commons/minifi-commons-schema/src/main/java/org/apache/nifi/minifi/commons => minifi-toolkit/minifi-toolkit-schema/src/main/java/org/apache/nifi/minifi/toolkit}/schema/ConfigSchema.java (93%) rename minifi/{minifi-commons/minifi-commons-schema/src/main/java/org/apache/nifi/minifi/commons => minifi-toolkit/minifi-toolkit-schema/src/main/java/org/apache/nifi/minifi/toolkit}/schema/ConnectionSchema.java (97%) rename minifi/{minifi-commons/minifi-commons-schema/src/main/java/org/apache/nifi/minifi/commons => minifi-toolkit/minifi-toolkit-schema/src/main/java/org/apache/nifi/minifi/toolkit}/schema/ContentRepositorySchema.java (94%) rename minifi/{minifi-commons/minifi-commons-schema/src/main/java/org/apache/nifi/minifi/commons => minifi-toolkit/minifi-toolkit-schema/src/main/java/org/apache/nifi/minifi/toolkit}/schema/ControllerServiceSchema.java (86%) rename minifi/{minifi-commons/minifi-commons-schema/src/main/java/org/apache/nifi/minifi/commons => minifi-toolkit/minifi-toolkit-schema/src/main/java/org/apache/nifi/minifi/toolkit}/schema/CorePropertiesSchema.java (94%) rename minifi/{minifi-commons/minifi-commons-schema/src/main/java/org/apache/nifi/minifi/commons => minifi-toolkit/minifi-toolkit-schema/src/main/java/org/apache/nifi/minifi/toolkit}/schema/FlowControllerSchema.java (81%) rename minifi/{minifi-commons/minifi-commons-schema/src/main/java/org/apache/nifi/minifi/commons => minifi-toolkit/minifi-toolkit-schema/src/main/java/org/apache/nifi/minifi/toolkit}/schema/FlowFileRepositorySchema.java (90%) rename minifi/{minifi-commons/minifi-commons-schema/src/main/java/org/apache/nifi/minifi/commons => minifi-toolkit/minifi-toolkit-schema/src/main/java/org/apache/nifi/minifi/toolkit}/schema/FunnelSchema.java (90%) rename minifi/{minifi-commons/minifi-commons-schema/src/main/java/org/apache/nifi/minifi/commons => minifi-toolkit/minifi-toolkit-schema/src/main/java/org/apache/nifi/minifi/toolkit}/schema/PortSchema.java (90%) rename minifi/{minifi-commons/minifi-commons-schema/src/main/java/org/apache/nifi/minifi/commons => minifi-toolkit/minifi-toolkit-schema/src/main/java/org/apache/nifi/minifi/toolkit}/schema/ProcessGroupSchema.java (89%) rename minifi/{minifi-commons/minifi-commons-schema/src/main/java/org/apache/nifi/minifi/commons => minifi-toolkit/minifi-toolkit-schema/src/main/java/org/apache/nifi/minifi/toolkit}/schema/ProcessorSchema.java (90%) rename minifi/{minifi-commons/minifi-commons-schema/src/main/java/org/apache/nifi/minifi/commons => minifi-toolkit/minifi-toolkit-schema/src/main/java/org/apache/nifi/minifi/toolkit}/schema/ProvenanceReportingSchema.java (89%) rename minifi/{minifi-commons/minifi-commons-schema/src/main/java/org/apache/nifi/minifi/commons => minifi-toolkit/minifi-toolkit-schema/src/main/java/org/apache/nifi/minifi/toolkit}/schema/ProvenanceRepositorySchema.java (95%) rename minifi/{minifi-commons/minifi-commons-schema/src/main/java/org/apache/nifi/minifi/commons => minifi-toolkit/minifi-toolkit-schema/src/main/java/org/apache/nifi/minifi/toolkit}/schema/RemotePortSchema.java (88%) rename minifi/{minifi-commons/minifi-commons-schema/src/main/java/org/apache/nifi/minifi/commons => minifi-toolkit/minifi-toolkit-schema/src/main/java/org/apache/nifi/minifi/toolkit}/schema/RemoteProcessGroupSchema.java (95%) rename minifi/{minifi-commons/minifi-commons-schema/src/main/java/org/apache/nifi/minifi/commons => minifi-toolkit/minifi-toolkit-schema/src/main/java/org/apache/nifi/minifi/toolkit}/schema/ReportingSchema.java (86%) rename minifi/{minifi-commons/minifi-commons-schema/src/main/java/org/apache/nifi/minifi/commons => minifi-toolkit/minifi-toolkit-schema/src/main/java/org/apache/nifi/minifi/toolkit}/schema/SecurityPropertiesSchema.java (95%) rename minifi/{minifi-commons/minifi-commons-schema/src/main/java/org/apache/nifi/minifi/commons => minifi-toolkit/minifi-toolkit-schema/src/main/java/org/apache/nifi/minifi/toolkit}/schema/SensitivePropsSchema.java (89%) rename minifi/{minifi-commons/minifi-commons-schema/src/main/java/org/apache/nifi/minifi/commons => minifi-toolkit/minifi-toolkit-schema/src/main/java/org/apache/nifi/minifi/toolkit}/schema/SwapSchema.java (92%) rename minifi/{minifi-commons/minifi-commons-schema/src/main/java/org/apache/nifi/minifi/commons => minifi-toolkit/minifi-toolkit-schema/src/main/java/org/apache/nifi/minifi/toolkit}/schema/common/BaseSchema.java (98%) rename minifi/{minifi-commons/minifi-commons-schema/src/main/java/org/apache/nifi/minifi/commons => minifi-toolkit/minifi-toolkit-schema/src/main/java/org/apache/nifi/minifi/toolkit}/schema/common/BaseSchemaWithId.java (95%) rename minifi/{minifi-commons/minifi-commons-schema/src/main/java/org/apache/nifi/minifi/commons => minifi-toolkit/minifi-toolkit-schema/src/main/java/org/apache/nifi/minifi/toolkit}/schema/common/BaseSchemaWithIdAndName.java (93%) rename minifi/{minifi-commons/minifi-commons-schema/src/main/java/org/apache/nifi/minifi/commons => minifi-toolkit/minifi-toolkit-schema/src/main/java/org/apache/nifi/minifi/toolkit}/schema/common/CollectionOverlap.java (97%) rename minifi/{minifi-commons/minifi-commons-schema/src/main/java/org/apache/nifi/minifi/commons => minifi-toolkit/minifi-toolkit-schema/src/main/java/org/apache/nifi/minifi/toolkit}/schema/common/CollectionUtil.java (96%) rename minifi/{minifi-commons/minifi-commons-schema/src/main/java/org/apache/nifi/minifi/commons => minifi-toolkit/minifi-toolkit-schema/src/main/java/org/apache/nifi/minifi/toolkit}/schema/common/CommonPropertyKeys.java (98%) rename minifi/{minifi-commons/minifi-commons-schema/src/main/java/org/apache/nifi/minifi/commons => minifi-toolkit/minifi-toolkit-schema/src/main/java/org/apache/nifi/minifi/toolkit}/schema/common/ConvertableSchema.java (96%) rename minifi/{minifi-commons/minifi-commons-schema/src/main/java/org/apache/nifi/minifi/commons => minifi-toolkit/minifi-toolkit-schema/src/main/java/org/apache/nifi/minifi/toolkit}/schema/common/Schema.java (96%) rename minifi/{minifi-commons/minifi-commons-schema/src/main/java/org/apache/nifi/minifi/commons => minifi-toolkit/minifi-toolkit-schema/src/main/java/org/apache/nifi/minifi/toolkit}/schema/common/StringUtil.java (97%) rename minifi/{minifi-commons/minifi-commons-schema/src/main/java/org/apache/nifi/minifi/commons => minifi-toolkit/minifi-toolkit-schema/src/main/java/org/apache/nifi/minifi/toolkit}/schema/common/WritableSchema.java (95%) rename minifi/{minifi-commons/minifi-commons-schema/src/main/java/org/apache/nifi/minifi/commons => minifi-toolkit/minifi-toolkit-schema/src/main/java/org/apache/nifi/minifi/toolkit}/schema/exception/SchemaInstantiatonException.java (95%) rename minifi/{minifi-commons/minifi-commons-schema/src/main/java/org/apache/nifi/minifi/commons => minifi-toolkit/minifi-toolkit-schema/src/main/java/org/apache/nifi/minifi/toolkit}/schema/exception/SchemaLoaderException.java (94%) rename minifi/{minifi-commons/minifi-commons-schema/src/main/java/org/apache/nifi/minifi/commons => minifi-toolkit/minifi-toolkit-schema/src/main/java/org/apache/nifi/minifi/toolkit}/schema/serialization/ConfigRepresenter.java (94%) rename minifi/{minifi-commons/minifi-commons-schema/src/main/java/org/apache/nifi/minifi/commons => minifi-toolkit/minifi-toolkit-schema/src/main/java/org/apache/nifi/minifi/toolkit}/schema/serialization/SchemaLoader.java (90%) rename minifi/{minifi-commons/minifi-commons-schema/src/main/java/org/apache/nifi/minifi/commons => minifi-toolkit/minifi-toolkit-schema/src/main/java/org/apache/nifi/minifi/toolkit}/schema/serialization/SchemaSaver.java (93%) rename minifi/{minifi-commons/minifi-commons-schema/src/main/java/org/apache/nifi/minifi/commons => minifi-toolkit/minifi-toolkit-schema/src/main/java/org/apache/nifi/minifi/toolkit}/schema/v1/ConfigSchemaV1.java (86%) rename minifi/{minifi-commons/minifi-commons-schema/src/main/java/org/apache/nifi/minifi/commons => minifi-toolkit/minifi-toolkit-schema/src/main/java/org/apache/nifi/minifi/toolkit}/schema/v1/ConnectionSchemaV1.java (81%) rename minifi/{minifi-commons/minifi-commons-schema/src/main/java/org/apache/nifi/minifi/commons => minifi-toolkit/minifi-toolkit-schema/src/main/java/org/apache/nifi/minifi/toolkit}/schema/v1/ProcessorSchemaV1.java (75%) rename minifi/{minifi-commons/minifi-commons-schema/src/main/java/org/apache/nifi/minifi/commons => minifi-toolkit/minifi-toolkit-schema/src/main/java/org/apache/nifi/minifi/toolkit}/schema/v1/RemoteProcessGroupSchemaV1.java (84%) rename minifi/{minifi-commons/minifi-commons-schema/src/main/java/org/apache/nifi/minifi/commons => minifi-toolkit/minifi-toolkit-schema/src/main/java/org/apache/nifi/minifi/toolkit}/schema/v2/ConfigSchemaV2.java (85%) rename minifi/{minifi-commons/minifi-commons-schema/src/main/java/org/apache/nifi/minifi/commons => minifi-toolkit/minifi-toolkit-schema/src/main/java/org/apache/nifi/minifi/toolkit}/schema/v2/CorePropertiesSchemaV2.java (90%) rename minifi/{minifi-commons/minifi-commons-schema/src/main/java/org/apache/nifi/minifi/commons => minifi-toolkit/minifi-toolkit-schema/src/main/java/org/apache/nifi/minifi/toolkit}/schema/v2/ProcessGroupSchemaV2.java (84%) rename minifi/{minifi-commons/minifi-commons-schema/src/main/java/org/apache/nifi/minifi/commons => minifi-toolkit/minifi-toolkit-schema/src/main/java/org/apache/nifi/minifi/toolkit}/schema/v2/RemoteProcessGroupSchemaV2.java (79%) rename minifi/{minifi-commons/minifi-commons-schema/src/test/java/org/apache/nifi/minifi/commons => minifi-toolkit/minifi-toolkit-schema/src/test/java/org/apache/nifi/minifi/toolkit}/schema/ConfigSchemaTest.java (96%) rename minifi/{minifi-commons/minifi-commons-schema/src/test/java/org/apache/nifi/minifi/commons => minifi-toolkit/minifi-toolkit-schema/src/test/java/org/apache/nifi/minifi/toolkit}/schema/ConnectionSchemaTest.java (98%) rename minifi/{minifi-commons/minifi-commons-schema/src/test/java/org/apache/nifi/minifi/commons => minifi-toolkit/minifi-toolkit-schema/src/test/java/org/apache/nifi/minifi/toolkit}/schema/ProcessGroupSchemaTest.java (96%) rename minifi/{minifi-commons/minifi-commons-schema/src/test/java/org/apache/nifi/minifi/commons => minifi-toolkit/minifi-toolkit-schema/src/test/java/org/apache/nifi/minifi/toolkit}/schema/RemoteProcessGroupSchemaTest.java (98%) rename minifi/{minifi-commons/minifi-commons-schema/src/test/java/org/apache/nifi/minifi/commons => minifi-toolkit/minifi-toolkit-schema/src/test/java/org/apache/nifi/minifi/toolkit}/schema/SecurityPropertiesSchemaTest.java (98%) rename minifi/{minifi-commons/minifi-commons-schema/src/test/java/org/apache/nifi/minifi/commons => minifi-toolkit/minifi-toolkit-schema/src/test/java/org/apache/nifi/minifi/toolkit}/schema/serialization/SchemaLoaderTest.java (92%) rename minifi/{minifi-commons/minifi-commons-schema/src/test/java/org/apache/nifi/minifi/commons => minifi-toolkit/minifi-toolkit-schema/src/test/java/org/apache/nifi/minifi/toolkit}/schema/v1/ConfigSchemaV1Test.java (89%) rename minifi/{minifi-commons/minifi-commons-schema/src/test/java/org/apache/nifi/minifi/commons => minifi-toolkit/minifi-toolkit-schema/src/test/java/org/apache/nifi/minifi/toolkit}/schema/v1/ConnectionSchemaV1Test.java (95%) rename minifi/{minifi-commons/minifi-commons-schema/src/test/java/org/apache/nifi/minifi/commons => minifi-toolkit/minifi-toolkit-schema/src/test/java/org/apache/nifi/minifi/toolkit}/schema/v1/ProcessorSchemaV1Test.java (89%) rename minifi/{minifi-commons/minifi-commons-schema/src/test/java/org/apache/nifi/minifi/commons => minifi-toolkit/minifi-toolkit-schema/src/test/java/org/apache/nifi/minifi/toolkit}/schema/v2/RemoteProcessGroupSchemaV2Test.java (92%) rename minifi/{minifi-commons/minifi-commons-schema => minifi-toolkit/minifi-toolkit-schema}/src/test/resources/config-minimal-v2.yml (100%) rename minifi/{minifi-commons/minifi-commons-schema => minifi-toolkit/minifi-toolkit-schema}/src/test/resources/config-minimal-v3.yml (100%) rename minifi/{minifi-commons/minifi-commons-schema => minifi-toolkit/minifi-toolkit-schema}/src/test/resources/config-minimal.yml (100%) diff --git a/c2/c2-client-bundle/c2-client-api/src/main/java/org/apache/nifi/c2/client/api/C2Client.java b/c2/c2-client-bundle/c2-client-api/src/main/java/org/apache/nifi/c2/client/api/C2Client.java index 3ef7e362c6..de90b1b0cd 100644 --- a/c2/c2-client-bundle/c2-client-api/src/main/java/org/apache/nifi/c2/client/api/C2Client.java +++ b/c2/c2-client-bundle/c2-client-api/src/main/java/org/apache/nifi/c2/client/api/C2Client.java @@ -14,6 +14,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ + package org.apache.nifi.c2.client.api; import java.util.Optional; @@ -47,7 +48,15 @@ public interface C2Client { * @param callbackUrl url where the content should be downloaded from * @return the actual downloaded content. Will be empty if no content can be downloaded */ - Optional retrieveUpdateContent(String callbackUrl); + Optional retrieveUpdateConfigurationContent(String callbackUrl); + + /** + * Retrieve the asset from the C2 Server + * + * @param callbackUrl url where the asset should be downloaded from + * @return the actual downloaded asset. Will be empty if no content can be downloaded + */ + Optional retrieveUpdateAssetContent(String callbackUrl); /** * Uploads a binary bundle to C2 server diff --git a/c2/c2-client-bundle/c2-client-base/src/main/java/org/apache/nifi/c2/client/C2ClientConfig.java b/c2/c2-client-bundle/c2-client-base/src/main/java/org/apache/nifi/c2/client/C2ClientConfig.java index 71a94a3698..3171280f99 100644 --- a/c2/c2-client-bundle/c2-client-base/src/main/java/org/apache/nifi/c2/client/C2ClientConfig.java +++ b/c2/c2-client-bundle/c2-client-base/src/main/java/org/apache/nifi/c2/client/C2ClientConfig.java @@ -14,8 +14,17 @@ * See the License for the specific language governing permissions and * limitations under the License. */ + package org.apache.nifi.c2.client; +import static java.util.Optional.ofNullable; +import static java.util.stream.Collectors.toUnmodifiableMap; +import static org.apache.commons.lang3.StringUtils.EMPTY; + +import java.util.Arrays; +import java.util.Map; +import org.apache.commons.lang3.StringUtils; + /** * Configuration for a C2 Client. */ @@ -42,6 +51,7 @@ public class C2ClientConfig { private final String truststoreType; private final long callTimeout; private final long readTimeout; + private final Map httpHeaders; private final long connectTimeout; private final int maxIdleConnections; private final long keepAliveDuration; @@ -71,6 +81,7 @@ public class C2ClientConfig { this.truststoreType = builder.truststoreType; this.readTimeout = builder.readTimeout; this.connectTimeout = builder.connectTimeout; + this.httpHeaders = builder.httpHeaders; this.maxIdleConnections = builder.maxIdleConnections; this.keepAliveDuration = builder.keepAliveDuration; this.c2RequestCompression = builder.c2RequestCompression; @@ -165,6 +176,10 @@ public class C2ClientConfig { return connectTimeout; } + public Map getHttpHeaders() { + return httpHeaders; + } + public String getC2RequestCompression() { return c2RequestCompression; } @@ -186,6 +201,9 @@ public class C2ClientConfig { */ public static class Builder { + private static final String HTTP_HEADERS_SEPARATOR = "#"; + private static final String HTTP_HEADER_KEY_VALUE_SEPARATOR = ":"; + private String c2Url; private String c2AckUrl; private String c2RestPathBase; @@ -208,6 +226,7 @@ public class C2ClientConfig { private String truststoreType; private long readTimeout; private long connectTimeout; + private Map httpHeaders; private int maxIdleConnections; private long keepAliveDuration; private String c2RequestCompression; @@ -227,10 +246,12 @@ public class C2ClientConfig { this.c2RestPathBase = c2RestPathBase; return this; } + public Builder c2RestPathHeartbeat(String c2RestPathHeartbeat) { this.c2RestPathHeartbeat = c2RestPathHeartbeat; return this; } + public Builder c2RestPathAcknowledge(String c2RestPathAcknowledge) { this.c2RestPathAcknowledge = c2RestPathAcknowledge; return this; @@ -321,6 +342,21 @@ public class C2ClientConfig { return this; } + public Builder httpHeaders(String httpHeaders) { + this.httpHeaders = ofNullable(httpHeaders) + .filter(StringUtils::isNotBlank) + .map(headers -> headers.split(HTTP_HEADERS_SEPARATOR)) + .stream() + .flatMap(Arrays::stream) + .map(String::trim) + .map(header -> header.split(HTTP_HEADER_KEY_VALUE_SEPARATOR)) + .filter(split -> split.length == 2) + .collect(toUnmodifiableMap( + split -> ofNullable(split[0]).map(String::trim).orElse(EMPTY), + split -> ofNullable(split[1]).map(String::trim).orElse(EMPTY))); + return this; + } + public Builder maxIdleConnections(int maxIdleConnections) { this.maxIdleConnections = maxIdleConnections; return this; diff --git a/c2/c2-client-bundle/c2-client-http/src/main/java/org/apache/nifi/c2/client/http/C2HttpClient.java b/c2/c2-client-bundle/c2-client-http/src/main/java/org/apache/nifi/c2/client/http/C2HttpClient.java index 563a8a6f14..5aad04d114 100644 --- a/c2/c2-client-bundle/c2-client-http/src/main/java/org/apache/nifi/c2/client/http/C2HttpClient.java +++ b/c2/c2-client-bundle/c2-client-http/src/main/java/org/apache/nifi/c2/client/http/C2HttpClient.java @@ -20,8 +20,10 @@ package org.apache.nifi.c2.client.http; import static okhttp3.MultipartBody.FORM; import java.io.IOException; +import java.util.Map; import java.util.Optional; import java.util.concurrent.atomic.AtomicReference; +import okhttp3.Headers; import okhttp3.MediaType; import okhttp3.MultipartBody; import okhttp3.OkHttpClient; @@ -84,33 +86,13 @@ public class C2HttpClient implements C2Client { } @Override - public Optional retrieveUpdateContent(String callbackUrl) { - Optional updateContent = Optional.empty(); + public Optional retrieveUpdateConfigurationContent(String callbackUrl) { + return retrieveContent(callbackUrl, clientConfig.getHttpHeaders()); + } - Request.Builder requestBuilder = new Request.Builder() - .get() - .url(callbackUrl); - Request request = requestBuilder.build(); - - try (Response response = httpClientReference.get().newCall(request).execute()) { - Optional body = Optional.ofNullable(response.body()); - - if (!response.isSuccessful()) { - StringBuilder messageBuilder = new StringBuilder(String.format("Configuration retrieval failed: HTTP %d", response.code())); - body.map(Object::toString).ifPresent(messageBuilder::append); - throw new C2ServerException(messageBuilder.toString()); - } - - if (body.isPresent()) { - updateContent = Optional.of(body.get().bytes()); - } else { - logger.warn("No body returned when pulling a new configuration"); - } - } catch (Exception e) { - logger.warn("Configuration retrieval failed", e); - } - - return updateContent; + @Override + public Optional retrieveUpdateAssetContent(String callbackUrl) { + return retrieveContent(callbackUrl, Map.of()); } @Override @@ -181,4 +163,34 @@ public class C2HttpClient implements C2Client { logger.error("Could not transmit ack to C2 server {}", c2UrlProvider.getAcknowledgeUrl(), e); } } + + private Optional retrieveContent(String callbackUrl, Map httpHeaders) { + Optional content = Optional.empty(); + + Request.Builder requestBuilder = new Request.Builder() + .get() + .headers(Headers.of(httpHeaders)) + .url(callbackUrl); + Request request = requestBuilder.build(); + + try (Response response = httpClientReference.get().newCall(request).execute()) { + Optional body = Optional.ofNullable(response.body()); + + if (!response.isSuccessful()) { + StringBuilder messageBuilder = new StringBuilder(String.format("Update content retrieval failed: HTTP %d", response.code())); + body.map(Object::toString).ifPresent(messageBuilder::append); + throw new C2ServerException(messageBuilder.toString()); + } + + if (body.isPresent()) { + content = Optional.of(body.get().bytes()); + } else { + logger.warn("No body returned when pulling new content"); + } + } catch (Exception e) { + logger.warn("Update content retrieval failed", e); + } + + return content; + } } diff --git a/c2/c2-client-bundle/c2-client-http/src/test/java/org/apache/nifi/c2/client/http/C2HttpClientTest.java b/c2/c2-client-bundle/c2-client-http/src/test/java/org/apache/nifi/c2/client/http/C2HttpClientTest.java index fb0ee3c49d..d8167b5112 100644 --- a/c2/c2-client-bundle/c2-client-http/src/test/java/org/apache/nifi/c2/client/http/C2HttpClientTest.java +++ b/c2/c2-client-bundle/c2-client-http/src/test/java/org/apache/nifi/c2/client/http/C2HttpClientTest.java @@ -124,7 +124,7 @@ public class C2HttpClientTest { mockWebServer.enqueue(new MockResponse().setBody("updateContent").setResponseCode(HTTP_STATUS_BAD_REQUEST)); C2HttpClient c2HttpClient = C2HttpClient.create(c2ClientConfig, serializer); - Optional response = c2HttpClient.retrieveUpdateContent(baseUrl + UPDATE_PATH); + Optional response = c2HttpClient.retrieveUpdateConfigurationContent(baseUrl + UPDATE_PATH); assertFalse(response.isPresent()); @@ -138,7 +138,7 @@ public class C2HttpClientTest { mockWebServer.enqueue(new MockResponse().setBody(content).setResponseCode(HTTP_STATUS_OK)); C2HttpClient c2HttpClient = C2HttpClient.create(c2ClientConfig, serializer); - Optional response = c2HttpClient.retrieveUpdateContent(baseUrl + UPDATE_PATH); + Optional response = c2HttpClient.retrieveUpdateConfigurationContent(baseUrl + UPDATE_PATH); assertTrue(response.isPresent()); assertArrayEquals(content.getBytes(StandardCharsets.UTF_8), response.get()); diff --git a/c2/c2-client-bundle/c2-client-service/src/main/java/org/apache/nifi/c2/client/service/operation/TransferDebugOperationHandler.java b/c2/c2-client-bundle/c2-client-service/src/main/java/org/apache/nifi/c2/client/service/operation/TransferDebugOperationHandler.java index 398f9c241e..98e5703bd9 100644 --- a/c2/c2-client-bundle/c2-client-service/src/main/java/org/apache/nifi/c2/client/service/operation/TransferDebugOperationHandler.java +++ b/c2/c2-client-bundle/c2-client-service/src/main/java/org/apache/nifi/c2/client/service/operation/TransferDebugOperationHandler.java @@ -33,8 +33,9 @@ import static org.apache.nifi.c2.protocol.api.OperationType.TRANSFER; import java.io.ByteArrayOutputStream; import java.io.File; +import java.io.FileInputStream; +import java.io.FileOutputStream; import java.io.IOException; -import java.io.UncheckedIOException; import java.nio.charset.Charset; import java.nio.file.Files; import java.nio.file.Path; @@ -45,9 +46,11 @@ import java.util.Map; import java.util.Optional; import java.util.function.Predicate; import java.util.stream.Stream; +import java.util.zip.GZIPInputStream; import org.apache.commons.compress.archivers.tar.TarArchiveEntry; import org.apache.commons.compress.archivers.tar.TarArchiveOutputStream; import org.apache.commons.compress.compressors.gzip.GzipCompressorOutputStream; +import org.apache.commons.compress.compressors.gzip.GzipUtils; import org.apache.nifi.c2.client.api.C2Client; import org.apache.nifi.c2.protocol.api.C2Operation; import org.apache.nifi.c2.protocol.api.C2OperationAck; @@ -124,11 +127,11 @@ public class TransferDebugOperationHandler implements C2OperationHandler { return operationAck(operation, operationState(NOT_APPLIED, C2_CALLBACK_URL_NOT_FOUND)); } - List contentFilteredFilePaths = null; + List preparedFiles = null; C2OperationState operationState; try { - contentFilteredFilePaths = filterContent(operation.getIdentifier(), bundleFilePaths); - operationState = createDebugBundle(contentFilteredFilePaths) + preparedFiles = prepareFiles(operation.getIdentifier(), bundleFilePaths); + operationState = createDebugBundle(preparedFiles) .map(bundle -> c2Client.uploadBundle(callbackUrl.get(), bundle) .map(errorMessage -> operationState(NOT_APPLIED, errorMessage)) .orElseGet(() -> operationState(FULLY_APPLIED, SUCCESSFUL_UPLOAD))) @@ -137,7 +140,7 @@ public class TransferDebugOperationHandler implements C2OperationHandler { LOG.error("Unexpected error happened", e); operationState = operationState(NOT_APPLIED, UNABLE_TO_CREATE_BUNDLE); } finally { - ofNullable(contentFilteredFilePaths).ifPresent(this::cleanup); + ofNullable(preparedFiles).ifPresent(this::cleanup); } LOG.debug("Returning operation ack for operation {} with state {} and details {}", operation.getIdentifier(), operationState.getState(), operationState.getDetails()); @@ -158,21 +161,40 @@ public class TransferDebugOperationHandler implements C2OperationHandler { return state; } - private List filterContent(String operationId, List bundleFilePaths) { - List contentFilteredFilePaths = new ArrayList<>(); - for (Path path : bundleFilePaths) { - String fileName = path.getFileName().toString(); - try (Stream fileStream = lines(path, Charset.defaultCharset())) { - Path tempDirectory = createTempDirectory(operationId); - Path tempFile = Paths.get(tempDirectory.toAbsolutePath().toString(), fileName); - Files.write(tempFile, (Iterable) fileStream.filter(contentFilter)::iterator); - contentFilteredFilePaths.add(tempFile); - } catch (IOException e) { - LOG.error("Error during filtering file content: " + path.toAbsolutePath(), e); - throw new UncheckedIOException(e); - } + private List prepareFiles(String operationId, List bundleFilePaths) throws IOException { + List preparedFiles = new ArrayList<>(); + for (Path bundleFile : bundleFilePaths) { + Path tempDirectory = createTempDirectory(operationId); + String fileName = bundleFile.getFileName().toString(); + + Path preparedFile = GzipUtils.isCompressedFilename(fileName) + ? handleGzipFile(bundleFile, Paths.get(tempDirectory.toAbsolutePath().toString(), GzipUtils.getUncompressedFilename(fileName))) + : handleUncompressedFile(bundleFile, Paths.get(tempDirectory.toAbsolutePath().toString(), fileName)); + preparedFiles.add(preparedFile); + } + return preparedFiles; + } + + private Path handleGzipFile(Path sourceFile, Path targetFile) throws IOException { + try (GZIPInputStream gzipInputStream = new GZIPInputStream(new FileInputStream(sourceFile.toFile())); + FileOutputStream fileOutputStream = new FileOutputStream(targetFile.toFile())) { + // no content filter is applied here as flow.json.gz has encoded properties + gzipInputStream.transferTo(fileOutputStream); + return targetFile; + } catch (IOException e) { + LOG.error("Error during filtering gzip file content: " + sourceFile.toAbsolutePath(), e); + throw e; + } + } + + private Path handleUncompressedFile(Path sourceFile, Path targetFile) throws IOException { + try (Stream fileStream = lines(sourceFile, Charset.defaultCharset())) { + Files.write(targetFile, (Iterable) fileStream.filter(contentFilter)::iterator); + return targetFile; + } catch (IOException e) { + LOG.error("Error during filtering uncompressed file content: " + sourceFile.toAbsolutePath(), e); + throw e; } - return contentFilteredFilePaths; } private Optional createDebugBundle(List filePaths) { diff --git a/c2/c2-client-bundle/c2-client-service/src/main/java/org/apache/nifi/c2/client/service/operation/UpdateAssetOperationHandler.java b/c2/c2-client-bundle/c2-client-service/src/main/java/org/apache/nifi/c2/client/service/operation/UpdateAssetOperationHandler.java index 5a964ea7fe..eb775c3da0 100644 --- a/c2/c2-client-bundle/c2-client-service/src/main/java/org/apache/nifi/c2/client/service/operation/UpdateAssetOperationHandler.java +++ b/c2/c2-client-bundle/c2-client-service/src/main/java/org/apache/nifi/c2/client/service/operation/UpdateAssetOperationHandler.java @@ -123,11 +123,11 @@ public class UpdateAssetOperationHandler implements C2OperationHandler { LOG.info("Initiating asset update from url {} with name {}, force update is {}", callbackUrl, assetFileName, forceDownload); C2OperationState operationState = assetUpdatePrecondition.test(assetFileName, forceDownload) - ? c2Client.retrieveUpdateContent(callbackUrl.get()) - .map(content -> assetPersistFunction.apply(assetFileName, content) - ? operationState(FULLY_APPLIED, SUCCESSFULLY_UPDATE_ASSET) - : operationState(NOT_APPLIED, FAILED_TO_PERSIST_ASSET_TO_DISK)) - .orElseGet(() -> operationState(NOT_APPLIED, UPDATE_ASSET_RETRIEVAL_RESULTED_IN_EMPTY_CONTENT)) + ? c2Client.retrieveUpdateAssetContent(callbackUrl.get()) + .map(content -> assetPersistFunction.apply(assetFileName, content) + ? operationState(FULLY_APPLIED, SUCCESSFULLY_UPDATE_ASSET) + : operationState(NOT_APPLIED, FAILED_TO_PERSIST_ASSET_TO_DISK)) + .orElseGet(() -> operationState(NOT_APPLIED, UPDATE_ASSET_RETRIEVAL_RESULTED_IN_EMPTY_CONTENT)) : operationState(NO_OPERATION, UPDATE_ASSET_PRECONDITIONS_WERE_NOT_MET); return operationAck(operationId, operationState); diff --git a/c2/c2-client-bundle/c2-client-service/src/main/java/org/apache/nifi/c2/client/service/operation/UpdateConfigurationOperationHandler.java b/c2/c2-client-bundle/c2-client-service/src/main/java/org/apache/nifi/c2/client/service/operation/UpdateConfigurationOperationHandler.java index b58bdc8796..d2f4c7e9e1 100644 --- a/c2/c2-client-bundle/c2-client-service/src/main/java/org/apache/nifi/c2/client/service/operation/UpdateConfigurationOperationHandler.java +++ b/c2/c2-client-bundle/c2-client-service/src/main/java/org/apache/nifi/c2/client/service/operation/UpdateConfigurationOperationHandler.java @@ -29,7 +29,6 @@ import static org.apache.nifi.c2.protocol.api.OperationType.UPDATE; import java.net.URI; import java.util.Map; import java.util.Optional; -import java.util.function.Function; import java.util.regex.Matcher; import java.util.regex.Pattern; import org.apache.nifi.c2.client.api.C2Client; @@ -43,22 +42,26 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; public class UpdateConfigurationOperationHandler implements C2OperationHandler { - private static final Logger logger = LoggerFactory.getLogger(UpdateConfigurationOperationHandler.class); - private static final Pattern FLOW_ID_PATTERN = Pattern.compile("/[^/]+?/[^/]+?/[^/]+?/([^/]+)?/?.*"); - static final String FLOW_ID = "flowId"; - static final String LOCATION = "location"; + public static final String FLOW_URL_KEY = "flowUrl"; public static final String FLOW_RELATIVE_URL_KEY = "relativeFlowUrl"; + static final String FLOW_ID = "flowId"; + static final String LOCATION = "location"; + + private static final Logger logger = LoggerFactory.getLogger(UpdateConfigurationOperationHandler.class); + + private static final Pattern FLOW_ID_PATTERN = Pattern.compile("/[^/]+?/[^/]+?/[^/]+?/([^/]+)?/?.*"); + private final C2Client client; - private final Function updateFlow; + private final UpdateConfigurationStrategy updateConfigurationStrategy; private final FlowIdHolder flowIdHolder; private final OperandPropertiesProvider operandPropertiesProvider; - public UpdateConfigurationOperationHandler(C2Client client, FlowIdHolder flowIdHolder, Function updateFlow, + public UpdateConfigurationOperationHandler(C2Client client, FlowIdHolder flowIdHolder, UpdateConfigurationStrategy updateConfigurationStrategy, OperandPropertiesProvider operandPropertiesProvider) { this.client = client; - this.updateFlow = updateFlow; + this.updateConfigurationStrategy = updateConfigurationStrategy; this.flowIdHolder = flowIdHolder; this.operandPropertiesProvider = operandPropertiesProvider; } @@ -80,7 +83,7 @@ public class UpdateConfigurationOperationHandler implements C2OperationHandler { @Override public boolean requiresRestart() { - return true; + return false; } @Override @@ -108,19 +111,22 @@ public class UpdateConfigurationOperationHandler implements C2OperationHandler { logger.info("Will perform flow update from {} for operation #{}. Previous flow id was {}, replacing with new id {}", callbackUrl, operationId, ofNullable(flowIdHolder.getFlowId()).orElse("not set"), flowId); - flowIdHolder.setFlowId(flowId); - return operationAck(operationId, updateFlow(operationId, callbackUrl.get())); + C2OperationState state = updateFlow(operationId, callbackUrl.get()); + if (state.getState() == FULLY_APPLIED) { + flowIdHolder.setFlowId(flowId); + } + return operationAck(operationId, state); } private C2OperationState updateFlow(String opIdentifier, String callbackUrl) { - Optional updateContent = client.retrieveUpdateContent(callbackUrl); + Optional updateContent = client.retrieveUpdateConfigurationContent(callbackUrl); if (!updateContent.isPresent()) { logger.error("Update content retrieval resulted in empty content so flow update was omitted for operation #{}.", opIdentifier); return operationState(NOT_APPLIED, "Update content retrieval resulted in empty content"); } - if (!updateFlow.apply(updateContent.get())) { + if (!updateConfigurationStrategy.update(updateContent.get())) { logger.error("Update resulted in error for operation #{}.", opIdentifier); return operationState(NOT_APPLIED, "Update resulted in error"); } diff --git a/minifi/minifi-c2/minifi-c2-provider/minifi-c2-provider-nifi-rest/src/test/java/org/apache/nifi/minifi/c2/provider/nifi/rest/TemplatesIteratorExceptionTest.java b/c2/c2-client-bundle/c2-client-service/src/main/java/org/apache/nifi/c2/client/service/operation/UpdateConfigurationStrategy.java similarity index 63% rename from minifi/minifi-c2/minifi-c2-provider/minifi-c2-provider-nifi-rest/src/test/java/org/apache/nifi/minifi/c2/provider/nifi/rest/TemplatesIteratorExceptionTest.java rename to c2/c2-client-bundle/c2-client-service/src/main/java/org/apache/nifi/c2/client/service/operation/UpdateConfigurationStrategy.java index b60b9075c1..e551cb853e 100644 --- a/minifi/minifi-c2/minifi-c2-provider/minifi-c2-provider-nifi-rest/src/test/java/org/apache/nifi/minifi/c2/provider/nifi/rest/TemplatesIteratorExceptionTest.java +++ b/c2/c2-client-bundle/c2-client-service/src/main/java/org/apache/nifi/c2/client/service/operation/UpdateConfigurationStrategy.java @@ -15,19 +15,19 @@ * limitations under the License. */ -package org.apache.nifi.minifi.c2.provider.nifi.rest; +package org.apache.nifi.c2.client.service.operation; -import org.junit.jupiter.api.Test; +/** + * Common interface for classes implementing approach for MiNiFi flow update + */ +@FunctionalInterface +public interface UpdateConfigurationStrategy { -import java.io.IOException; - -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.mockito.Mockito.mock; - -public class TemplatesIteratorExceptionTest { - @Test - public void testCauseConstructor() { - IOException ioException = mock(IOException.class); - assertEquals(ioException, new TemplatesIteratorException(ioException).getCause()); - } + /** + * Updates the MiNiFi agent's flow with the flow passed as parameter + * + * @param flow the MiNiFi flow config JSON represented as a byte array + * @return true if the flow update was true, false otherwise + */ + boolean update(byte[] flow); } diff --git a/c2/c2-client-bundle/c2-client-service/src/test/java/org/apache/nifi/c2/client/service/operation/UpdateAssetOperationHandlerTest.java b/c2/c2-client-bundle/c2-client-service/src/test/java/org/apache/nifi/c2/client/service/operation/UpdateAssetOperationHandlerTest.java index 444bd9ff27..a078452fe8 100644 --- a/c2/c2-client-bundle/c2-client-service/src/test/java/org/apache/nifi/c2/client/service/operation/UpdateAssetOperationHandlerTest.java +++ b/c2/c2-client-bundle/c2-client-service/src/test/java/org/apache/nifi/c2/client/service/operation/UpdateAssetOperationHandlerTest.java @@ -160,7 +160,7 @@ public class UpdateAssetOperationHandlerTest { // given C2Operation operation = operation(ASSET_URL, ASSET_FILE_NAME, FORCE_DOWNLOAD); when(assetUpdatePrecondition.test(ASSET_FILE_NAME, parseBoolean(FORCE_DOWNLOAD))).thenReturn(TRUE); - when(c2Client.retrieveUpdateContent(ASSET_URL)).thenReturn(empty()); + when(c2Client.retrieveUpdateAssetContent(ASSET_URL)).thenReturn(empty()); // when C2OperationAck result = testHandler.handle(operation); @@ -177,7 +177,7 @@ public class UpdateAssetOperationHandlerTest { C2Operation operation = operation(ASSET_URL, ASSET_FILE_NAME, FORCE_DOWNLOAD); when(assetUpdatePrecondition.test(ASSET_FILE_NAME, parseBoolean(FORCE_DOWNLOAD))).thenReturn(TRUE); byte[] mockUpdateContent = new byte[0]; - when(c2Client.retrieveUpdateContent(ASSET_URL)).thenReturn(Optional.of(mockUpdateContent)); + when(c2Client.retrieveUpdateAssetContent(ASSET_URL)).thenReturn(Optional.of(mockUpdateContent)); when(assetPersistFunction.apply(ASSET_FILE_NAME, mockUpdateContent)).thenReturn(FALSE); // when @@ -195,7 +195,7 @@ public class UpdateAssetOperationHandlerTest { C2Operation operation = operation(ASSET_URL, ASSET_FILE_NAME, FORCE_DOWNLOAD); when(assetUpdatePrecondition.test(ASSET_FILE_NAME, parseBoolean(FORCE_DOWNLOAD))).thenReturn(TRUE); byte[] mockUpdateContent = new byte[0]; - when(c2Client.retrieveUpdateContent(ASSET_URL)).thenReturn(Optional.of(mockUpdateContent)); + when(c2Client.retrieveUpdateAssetContent(ASSET_URL)).thenReturn(Optional.of(mockUpdateContent)); when(assetPersistFunction.apply(ASSET_FILE_NAME, mockUpdateContent)).thenReturn(TRUE); // when diff --git a/c2/c2-client-bundle/c2-client-service/src/test/java/org/apache/nifi/c2/client/service/operation/UpdateConfigurationOperationHandlerTest.java b/c2/c2-client-bundle/c2-client-service/src/test/java/org/apache/nifi/c2/client/service/operation/UpdateConfigurationOperationHandlerTest.java index fb23a15169..d12b1be5f5 100644 --- a/c2/c2-client-bundle/c2-client-service/src/test/java/org/apache/nifi/c2/client/service/operation/UpdateConfigurationOperationHandlerTest.java +++ b/c2/c2-client-bundle/c2-client-service/src/test/java/org/apache/nifi/c2/client/service/operation/UpdateConfigurationOperationHandlerTest.java @@ -30,7 +30,6 @@ import java.util.Collections; import java.util.HashMap; import java.util.Map; import java.util.Optional; -import java.util.function.Function; import org.apache.nifi.c2.client.api.C2Client; import org.apache.nifi.c2.client.service.FlowIdHolder; import org.apache.nifi.c2.protocol.api.C2Operation; @@ -82,9 +81,9 @@ public class UpdateConfigurationOperationHandlerTest { @Test void testHandleFlowIdInArg() { - Function successUpdate = x -> true; + UpdateConfigurationStrategy successUpdate = flow -> true; when(flowIdHolder.getFlowId()).thenReturn(FLOW_ID); - when(client.retrieveUpdateContent(any())).thenReturn(Optional.of("content".getBytes())); + when(client.retrieveUpdateConfigurationContent(any())).thenReturn(Optional.of("content".getBytes())); when(client.getCallbackUrl(any(), any())).thenReturn(Optional.of(INCORRECT_LOCATION)); UpdateConfigurationOperationHandler handler = new UpdateConfigurationOperationHandler(client, flowIdHolder, successUpdate, operandPropertiesProvider); C2Operation operation = new C2Operation(); @@ -117,9 +116,9 @@ public class UpdateConfigurationOperationHandlerTest { @Test void testHandleReturnsNotAppliedWithContentApplyIssues() { - Function failedToUpdate = x -> false; + UpdateConfigurationStrategy failedToUpdate = flow -> false; when(flowIdHolder.getFlowId()).thenReturn("previous_flow_id"); - when(client.retrieveUpdateContent(any())).thenReturn(Optional.of("content".getBytes())); + when(client.retrieveUpdateConfigurationContent(any())).thenReturn(Optional.of("content".getBytes())); when(client.getCallbackUrl(any(), any())).thenReturn(Optional.of(CORRECT_LOCATION)); UpdateConfigurationOperationHandler handler = new UpdateConfigurationOperationHandler(client, flowIdHolder, failedToUpdate, operandPropertiesProvider); C2Operation operation = new C2Operation(); @@ -134,10 +133,10 @@ public class UpdateConfigurationOperationHandlerTest { @Test void testHandleReturnsFullyApplied() { - Function successUpdate = x -> true; + UpdateConfigurationStrategy successUpdate = flow -> true; when(flowIdHolder.getFlowId()).thenReturn("previous_flow_id"); when(client.getCallbackUrl(any(), any())).thenReturn(Optional.of(CORRECT_LOCATION)); - when(client.retrieveUpdateContent(any())).thenReturn(Optional.of("content".getBytes())); + when(client.retrieveUpdateConfigurationContent(any())).thenReturn(Optional.of("content".getBytes())); UpdateConfigurationOperationHandler handler = new UpdateConfigurationOperationHandler(client, flowIdHolder, successUpdate, operandPropertiesProvider); C2Operation operation = new C2Operation(); operation.setIdentifier(OPERATION_ID); diff --git a/minifi/minifi-assembly/README.md b/minifi/minifi-assembly/README.md index 17c87ad334..2151fe7dcd 100644 --- a/minifi/minifi-assembly/README.md +++ b/minifi/minifi-assembly/README.md @@ -39,7 +39,7 @@ Specific goals for MiNiFi are comprised of: Perspectives of the role of MiNiFi should be from the perspective of the agent acting immediately at, or directly adjacent to, source sensors, systems, or servers. ## Requirements -* JRE 1.8 +* JRE 17 ## Getting Started @@ -55,7 +55,7 @@ To run MiNiFi: - View the logs located in the logs folder $ tail -F ~/example-minifi-deploy/logs/minifi-app.log -- For help building your first data flow and sending data to a NiFi instance see the System Admin Guide located in the docs folder or making use of the minifi-toolkit, which aids in adapting NiFi templates to MiNiFi YAML configuration file format. +- For help building your first data flow and sending data to a NiFi instance see the System Admin Guide located in the docs folder or making use of the minifi-toolkit, which aids in converting legacy YAML templates to the new JSON configuration file format. ## Getting Help If you have questions, you can reach out to our mailing list: dev@nifi.apache.org diff --git a/minifi/minifi-bootstrap/pom.xml b/minifi/minifi-bootstrap/pom.xml index 16b18282dd..1aa6fd26b0 100644 --- a/minifi/minifi-bootstrap/pom.xml +++ b/minifi/minifi-bootstrap/pom.xml @@ -79,13 +79,13 @@ limitations under the License. org.apache.nifi.minifi - minifi-commons-schema - ${project.version} + minifi-commons-api + 2.0.0-SNAPSHOT org.apache.nifi.minifi - minifi-commons-api - ${project.version} + minifi-commons-framework + 2.0.0-SNAPSHOT org.eclipse.jetty diff --git a/minifi/minifi-bootstrap/src/main/java/org/apache/nifi/minifi/bootstrap/RunMiNiFi.java b/minifi/minifi-bootstrap/src/main/java/org/apache/nifi/minifi/bootstrap/RunMiNiFi.java index dac7febab6..66811ebfc8 100644 --- a/minifi/minifi-bootstrap/src/main/java/org/apache/nifi/minifi/bootstrap/RunMiNiFi.java +++ b/minifi/minifi-bootstrap/src/main/java/org/apache/nifi/minifi/bootstrap/RunMiNiFi.java @@ -46,6 +46,8 @@ import org.apache.nifi.minifi.bootstrap.service.ReloadService; import org.apache.nifi.minifi.bootstrap.util.ProcessUtils; import org.apache.nifi.minifi.bootstrap.util.UnixProcessUtils; import org.apache.nifi.minifi.commons.api.MiNiFiCommandState; +import org.apache.nifi.minifi.commons.service.FlowEnrichService; +import org.apache.nifi.properties.ApplicationProperties; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -71,7 +73,6 @@ public class RunMiNiFi implements ConfigurationFileHolder { public static final Logger DEFAULT_LOGGER = LoggerFactory.getLogger(RunMiNiFi.class); public static final String CONF_DIR_KEY = "conf.dir"; - public static final String MINIFI_CONFIG_FILE_KEY = "nifi.minifi.config"; public static final String STATUS_FILE_PID_KEY = "pid"; public static final int UNINITIALIZED = -1; private static final String STATUS_FILE_PORT_KEY = "port"; @@ -115,7 +116,8 @@ public class RunMiNiFi implements ConfigurationFileHolder { MiNiFiStatusProvider miNiFiStatusProvider = new MiNiFiStatusProvider(miNiFiCommandSender, processUtils); periodicStatusReporterManager = new PeriodicStatusReporterManager(bootstrapProperties, miNiFiStatusProvider, miNiFiCommandSender, miNiFiParameters); - MiNiFiConfigurationChangeListener configurationChangeListener = new MiNiFiConfigurationChangeListener(this, DEFAULT_LOGGER, bootstrapFileProvider); + MiNiFiConfigurationChangeListener configurationChangeListener = new MiNiFiConfigurationChangeListener(this, DEFAULT_LOGGER, bootstrapFileProvider, + new FlowEnrichService(new ApplicationProperties(bootstrapProperties))); configurationChangeCoordinator = new ConfigurationChangeCoordinator(bootstrapFileProvider, this, singleton(configurationChangeListener)); currentPortProvider = new CurrentPortProvider(miNiFiCommandSender, miNiFiParameters, processUtils); diff --git a/minifi/minifi-bootstrap/src/main/java/org/apache/nifi/minifi/bootstrap/command/StartRunner.java b/minifi/minifi-bootstrap/src/main/java/org/apache/nifi/minifi/bootstrap/command/StartRunner.java index b7ac0de411..827f6ef171 100644 --- a/minifi/minifi-bootstrap/src/main/java/org/apache/nifi/minifi/bootstrap/command/StartRunner.java +++ b/minifi/minifi-bootstrap/src/main/java/org/apache/nifi/minifi/bootstrap/command/StartRunner.java @@ -18,27 +18,21 @@ package org.apache.nifi.minifi.bootstrap.command; import static java.nio.file.StandardCopyOption.REPLACE_EXISTING; -import static org.apache.nifi.minifi.commons.api.MiNiFiCommandState.FULLY_APPLIED; -import static org.apache.nifi.minifi.commons.api.MiNiFiCommandState.NOT_APPLIED_WITH_RESTART; import static org.apache.nifi.minifi.bootstrap.RunMiNiFi.CMD_LOGGER; import static org.apache.nifi.minifi.bootstrap.RunMiNiFi.CONF_DIR_KEY; import static org.apache.nifi.minifi.bootstrap.RunMiNiFi.DEFAULT_LOGGER; -import static org.apache.nifi.minifi.bootstrap.RunMiNiFi.MINIFI_CONFIG_FILE_KEY; import static org.apache.nifi.minifi.bootstrap.RunMiNiFi.STATUS_FILE_PID_KEY; import static org.apache.nifi.minifi.bootstrap.RunMiNiFi.UNINITIALIZED; import static org.apache.nifi.minifi.bootstrap.Status.ERROR; import static org.apache.nifi.minifi.bootstrap.Status.OK; -import static org.apache.nifi.minifi.bootstrap.util.ConfigTransformer.asByteArrayInputStream; -import static org.apache.nifi.minifi.bootstrap.util.ConfigTransformer.generateConfigFiles; +import static org.apache.nifi.minifi.commons.api.MiNiFiCommandState.FULLY_APPLIED; +import static org.apache.nifi.minifi.commons.api.MiNiFiCommandState.NOT_APPLIED_WITH_RESTART; +import static org.apache.nifi.minifi.commons.api.MiNiFiConstants.RAW_EXTENSION; import java.io.File; -import java.io.FileInputStream; -import java.io.FileNotFoundException; import java.io.IOException; -import java.io.InputStream; -import java.nio.ByteBuffer; import java.nio.file.Files; -import java.nio.file.Paths; +import java.nio.file.Path; import java.util.List; import java.util.Optional; import java.util.Properties; @@ -46,6 +40,7 @@ import java.util.concurrent.TimeUnit; import java.util.concurrent.locks.Condition; import java.util.concurrent.locks.Lock; import java.util.concurrent.locks.ReentrantLock; +import org.apache.commons.io.FilenameUtils; import org.apache.nifi.minifi.bootstrap.MiNiFiParameters; import org.apache.nifi.minifi.bootstrap.RunMiNiFi; import org.apache.nifi.minifi.bootstrap.ShutdownHook; @@ -56,10 +51,13 @@ import org.apache.nifi.minifi.bootstrap.service.BootstrapFileProvider; import org.apache.nifi.minifi.bootstrap.service.CurrentPortProvider; import org.apache.nifi.minifi.bootstrap.service.MiNiFiExecCommandProvider; import org.apache.nifi.minifi.bootstrap.service.MiNiFiListener; +import org.apache.nifi.minifi.bootstrap.service.MiNiFiPropertiesGenerator; import org.apache.nifi.minifi.bootstrap.service.MiNiFiStdLogHandler; import org.apache.nifi.minifi.bootstrap.service.PeriodicStatusReporterManager; +import org.apache.nifi.minifi.commons.api.MiNiFiProperties; public class StartRunner implements CommandRunner { + private static final int STARTUP_WAIT_SECONDS = 60; private final CurrentPortProvider currentPortProvider; @@ -71,15 +69,18 @@ public class StartRunner implements CommandRunner { private final Lock lock = new ReentrantLock(); private final Condition startupCondition = lock.newCondition(); private final RunMiNiFi runMiNiFi; - private volatile ShutdownHook shutdownHook; private final MiNiFiExecCommandProvider miNiFiExecCommandProvider; private final ConfigurationChangeListener configurationChangeListener; + private final MiNiFiPropertiesGenerator miNiFiPropertiesGenerator; + + private volatile ShutdownHook shutdownHook; private int listenPort; public StartRunner(CurrentPortProvider currentPortProvider, BootstrapFileProvider bootstrapFileProvider, - PeriodicStatusReporterManager periodicStatusReporterManager, MiNiFiStdLogHandler miNiFiStdLogHandler, MiNiFiParameters miNiFiParameters, File bootstrapConfigFile, - RunMiNiFi runMiNiFi, MiNiFiExecCommandProvider miNiFiExecCommandProvider, ConfigurationChangeListener configurationChangeListener) { + PeriodicStatusReporterManager periodicStatusReporterManager, MiNiFiStdLogHandler miNiFiStdLogHandler, + MiNiFiParameters miNiFiParameters, File bootstrapConfigFile, RunMiNiFi runMiNiFi, + MiNiFiExecCommandProvider miNiFiExecCommandProvider, ConfigurationChangeListener configurationChangeListener) { this.currentPortProvider = currentPortProvider; this.bootstrapFileProvider = bootstrapFileProvider; this.periodicStatusReporterManager = periodicStatusReporterManager; @@ -89,10 +90,12 @@ public class StartRunner implements CommandRunner { this.runMiNiFi = runMiNiFi; this.miNiFiExecCommandProvider = miNiFiExecCommandProvider; this.configurationChangeListener = configurationChangeListener; + this.miNiFiPropertiesGenerator = new MiNiFiPropertiesGenerator(); } /** * Starts (and restarts) MiNiFi process during the whole lifecycle of the bootstrap process. + * * @param args the input arguments * @return status code */ @@ -107,7 +110,7 @@ public class StartRunner implements CommandRunner { return OK.getStatusCode(); } - private void start() throws IOException, InterruptedException { + private void start() throws IOException, InterruptedException, ConfigurationChangeException { Integer port = currentPortProvider.getCurrentPort(); if (port != null) { CMD_LOGGER.info("Apache MiNiFi is already running, listening to Bootstrap on port {}", port); @@ -121,10 +124,11 @@ public class StartRunner implements CommandRunner { Properties bootstrapProperties = bootstrapFileProvider.getBootstrapProperties(); String confDir = bootstrapProperties.getProperty(CONF_DIR_KEY); - initConfigFiles(bootstrapProperties, confDir); + + generateMiNiFiProperties(bootstrapProperties, confDir); + regenerateFlowConfiguration(bootstrapProperties.getProperty(MiNiFiProperties.NIFI_MINIFI_FLOW_CONFIG.getKey())); Process process = startMiNiFi(); - try { while (true) { if (process.isAlive()) { @@ -144,7 +148,7 @@ public class StartRunner implements CommandRunner { Thread.sleep(5000L); continue; } - process = restartMiNifi(bootstrapProperties, confDir); + process = restartMiNifi(confDir); // failed to start process if (process == null) { return; @@ -161,18 +165,13 @@ public class StartRunner implements CommandRunner { } } - private Process restartMiNifi(Properties bootstrapProperties, String confDir) throws IOException { + private Process restartMiNifi(String confDir) throws IOException { Process process; boolean previouslyStarted = runMiNiFi.isNiFiStarted(); boolean configChangeSuccessful = true; if (!previouslyStarted) { - File swapConfigFile = bootstrapFileProvider.getConfigYmlSwapFile(); File bootstrapSwapConfigFile = bootstrapFileProvider.getBootstrapConfSwapFile(); - if (swapConfigFile.exists()) { - if (!revertFlowConfig(bootstrapProperties, confDir, swapConfigFile)) { - return null; - } - } else if(bootstrapSwapConfigFile.exists()) { + if (bootstrapSwapConfigFile.exists()) { if (!revertBootstrapConfig(confDir, bootstrapSwapConfigFile)) { return null; } @@ -202,36 +201,14 @@ public class StartRunner implements CommandRunner { return process; } - private boolean revertFlowConfig(Properties bootstrapProperties, String confDir, File swapConfigFile) throws IOException { - DEFAULT_LOGGER.info("Flow Swap file exists, MiNiFi failed trying to change configuration. Reverting to old configuration."); - - try { - ByteBuffer tempConfigFile = generateConfigFiles(Files.newInputStream(swapConfigFile.toPath()), confDir, bootstrapProperties); - runMiNiFi.getConfigFileReference().set(tempConfigFile.asReadOnlyBuffer()); - } catch (ConfigurationChangeException e) { - DEFAULT_LOGGER.error("The flow swap file is malformed, unable to restart from prior state. Will not attempt to restart MiNiFi. Swap File should be cleaned up manually."); - return false; - } - - Files.copy(swapConfigFile.toPath(), Paths.get(bootstrapProperties.getProperty(MINIFI_CONFIG_FILE_KEY)), REPLACE_EXISTING); - - DEFAULT_LOGGER.info("Replacing flow config file with swap file and deleting swap file"); - if (!swapConfigFile.delete()) { - DEFAULT_LOGGER.warn("The flow swap file failed to delete after replacing using it to revert to the old configuration. It should be cleaned up manually."); - } - runMiNiFi.setReloading(false); - return true; - } - private boolean revertBootstrapConfig(String confDir, File bootstrapSwapConfigFile) throws IOException { DEFAULT_LOGGER.info("Bootstrap Swap file exists, MiNiFi failed trying to change configuration. Reverting to old configuration."); Files.copy(bootstrapSwapConfigFile.toPath(), bootstrapConfigFile.toPath(), REPLACE_EXISTING); try { - ByteBuffer tempConfigFile = generateConfigFiles(asByteArrayInputStream(runMiNiFi.getConfigFileReference().get().duplicate()), confDir, bootstrapFileProvider.getBootstrapProperties()); - runMiNiFi.getConfigFileReference().set(tempConfigFile.asReadOnlyBuffer()); + miNiFiPropertiesGenerator.generateMinifiProperties(confDir, bootstrapFileProvider.getBootstrapProperties()); } catch (ConfigurationChangeException e) { - DEFAULT_LOGGER.error("The bootstrap swap file is malformed, unable to restart from prior state. Will not attempt to restart MiNiFi. Swap File should be cleaned up manually."); + DEFAULT_LOGGER.error("Unable to create MiNiFi properties file. Will not attempt to restart MiNiFi. Swap File should be cleaned up manually."); return false; } @@ -262,7 +239,6 @@ public class StartRunner implements CommandRunner { try { Thread.sleep(1000L); if (runMiNiFi.getReloading() && runMiNiFi.isNiFiStarted()) { - deleteSwapFile(bootstrapFileProvider.getConfigYmlSwapFile()); deleteSwapFile(bootstrapFileProvider.getBootstrapConfSwapFile()); runMiNiFi.setReloading(false); } @@ -283,24 +259,26 @@ public class StartRunner implements CommandRunner { } } - private void initConfigFiles(Properties bootstrapProperties, String confDir) throws IOException { - File configFile = new File(bootstrapProperties.getProperty(MINIFI_CONFIG_FILE_KEY)); - try (InputStream inputStream = new FileInputStream(configFile)) { - ByteBuffer tempConfigFile = generateConfigFiles(inputStream, confDir, bootstrapProperties); - runMiNiFi.getConfigFileReference().set(tempConfigFile.asReadOnlyBuffer()); - } catch (FileNotFoundException e) { - String fileNotFoundMessage = "The config file defined in " + MINIFI_CONFIG_FILE_KEY + " does not exists."; - DEFAULT_LOGGER.error(fileNotFoundMessage, e); - throw new StartupFailureException(fileNotFoundMessage); + private void generateMiNiFiProperties(Properties bootstrapProperties, String confDir) { + DEFAULT_LOGGER.debug("Generating minifi.properties from bootstrap.conf"); + try { + miNiFiPropertiesGenerator.generateMinifiProperties(confDir, bootstrapProperties); } catch (ConfigurationChangeException e) { - String malformedConfigFileMessage = "The config file is malformed, unable to start."; - DEFAULT_LOGGER.error(malformedConfigFileMessage, e); - throw new StartupFailureException(malformedConfigFileMessage); + throw new StartupFailureException("Unable to create MiNiFi properties file", e); + } + } + + private void regenerateFlowConfiguration(String flowConfigFileLocation) throws ConfigurationChangeException, IOException { + Path flowConfigFile = Path.of(flowConfigFileLocation).toAbsolutePath(); + String currentFlowConfigFileBaseName = FilenameUtils.getBaseName(flowConfigFile.toString()); + Path rawConfigFile = flowConfigFile.getParent().resolve(currentFlowConfigFileBaseName + RAW_EXTENSION); + if (Files.exists(rawConfigFile)) { + DEFAULT_LOGGER.debug("Regenerating flow configuration {} from raw flow configuration {}", flowConfigFile, rawConfigFile); + configurationChangeListener.handleChange(Files.newInputStream(rawConfigFile)); } } private Process startMiNiFi() throws IOException { - MiNiFiListener listener = new MiNiFiListener(); listenPort = listener.start(runMiNiFi, bootstrapFileProvider, configurationChangeListener); @@ -309,7 +287,7 @@ public class StartRunner implements CommandRunner { return startMiNiFiProcess(getProcessBuilder()); } - private ProcessBuilder getProcessBuilder() throws IOException{ + private ProcessBuilder getProcessBuilder() throws IOException { ProcessBuilder builder = new ProcessBuilder(); File workingDir = getWorkingDir(); diff --git a/minifi/minifi-bootstrap/src/main/java/org/apache/nifi/minifi/bootstrap/configuration/ConfigurationChangeCoordinator.java b/minifi/minifi-bootstrap/src/main/java/org/apache/nifi/minifi/bootstrap/configuration/ConfigurationChangeCoordinator.java index 7f3a0a8dce..77d787bf46 100644 --- a/minifi/minifi-bootstrap/src/main/java/org/apache/nifi/minifi/bootstrap/configuration/ConfigurationChangeCoordinator.java +++ b/minifi/minifi-bootstrap/src/main/java/org/apache/nifi/minifi/bootstrap/configuration/ConfigurationChangeCoordinator.java @@ -14,18 +14,22 @@ * See the License for the specific language governing permissions and * limitations under the License. */ + package org.apache.nifi.minifi.bootstrap.configuration; +import static java.util.Optional.ofNullable; +import static java.util.function.Predicate.not; +import static java.util.stream.Collectors.toList; + import java.io.Closeable; import java.io.IOException; import java.nio.ByteBuffer; -import java.util.ArrayList; import java.util.Collection; import java.util.Collections; import java.util.HashSet; -import java.util.Optional; import java.util.Properties; import java.util.Set; +import java.util.stream.Stream; import org.apache.nifi.minifi.bootstrap.RunMiNiFi; import org.apache.nifi.minifi.bootstrap.configuration.ingestors.interfaces.ChangeIngestor; import org.apache.nifi.minifi.bootstrap.service.BootstrapFileProvider; @@ -35,64 +39,72 @@ import org.slf4j.LoggerFactory; public class ConfigurationChangeCoordinator implements Closeable, ConfigurationChangeNotifier { - public static final String NOTIFIER_PROPERTY_PREFIX = "nifi.minifi.notifier"; - public static final String NOTIFIER_INGESTORS_KEY = NOTIFIER_PROPERTY_PREFIX + ".ingestors"; - private static final Logger LOGGER = LoggerFactory.getLogger(ConfigurationChangeCoordinator.class); + public static final String NOTIFIER_INGESTORS_KEY = "nifi.minifi.notifier.ingestors"; - private final Set configurationChangeListeners; - private final Set changeIngestors = new HashSet<>(); + private static final Logger LOGGER = LoggerFactory.getLogger(ConfigurationChangeCoordinator.class); + private static final String COMMA = ","; private final BootstrapFileProvider bootstrapFileProvider; private final RunMiNiFi runMiNiFi; + private final Set configurationChangeListeners; + private final Set changeIngestors; public ConfigurationChangeCoordinator(BootstrapFileProvider bootstrapFileProvider, RunMiNiFi runMiNiFi, - Set miNiFiConfigurationChangeListeners) { + Set miNiFiConfigurationChangeListeners) { this.bootstrapFileProvider = bootstrapFileProvider; this.runMiNiFi = runMiNiFi; - this.configurationChangeListeners = Optional.ofNullable(miNiFiConfigurationChangeListeners).map(Collections::unmodifiableSet).orElse(Collections.emptySet()); + this.configurationChangeListeners = ofNullable(miNiFiConfigurationChangeListeners).map(Collections::unmodifiableSet).orElse(Collections.emptySet()); + this.changeIngestors = new HashSet<>(); + } + + @Override + public Collection notifyListeners(ByteBuffer newFlowConfig) { + LOGGER.info("Notifying Listeners of a change"); + return configurationChangeListeners.stream() + .map(listener -> notifyListener(newFlowConfig, listener)) + .collect(toList()); + } + + @Override + public void close() { + closeIngestors(); } /** * Begins the associated notification service provided by the given implementation. In most implementations, no action will occur until this method is invoked. */ - public void start() throws IOException{ + public void start() throws IOException { initialize(); changeIngestors.forEach(ChangeIngestor::start); } - /** - * Provides an immutable collection of listeners for the notifier instance - * - * @return a collection of those listeners registered for notifications - */ - public Set getChangeListeners() { - return Collections.unmodifiableSet(configurationChangeListeners); - } - - /** - * Provide the mechanism by which listeners are notified - */ - public Collection notifyListeners(ByteBuffer newConfig) { - LOGGER.info("Notifying Listeners of a change"); - - Collection listenerHandleResults = new ArrayList<>(configurationChangeListeners.size()); - for (final ConfigurationChangeListener listener : getChangeListeners()) { - ListenerHandleResult result; - try { - listener.handleChange(new ByteBufferInputStream(newConfig.duplicate())); - result = new ListenerHandleResult(listener); - } catch (ConfigurationChangeException ex) { - result = new ListenerHandleResult(listener, ex); - } - listenerHandleResults.add(result); - LOGGER.info("Listener notification result: {}", result); + private ListenerHandleResult notifyListener(ByteBuffer newFlowConfig, ConfigurationChangeListener listener) { + try { + listener.handleChange(new ByteBufferInputStream(newFlowConfig.duplicate())); + ListenerHandleResult listenerHandleResult = new ListenerHandleResult(listener); + LOGGER.info("Listener notification result {}", listenerHandleResult); + return listenerHandleResult; + } catch (ConfigurationChangeException ex) { + ListenerHandleResult listenerHandleResult = new ListenerHandleResult(listener, ex); + LOGGER.error("Listener notification result {} with failure {}", listenerHandleResult, ex); + return listenerHandleResult; } - return listenerHandleResults; } + private void initialize() throws IOException { + closeIngestors(); - @Override - public void close() { + Properties bootstrapProperties = bootstrapFileProvider.getBootstrapProperties(); + ofNullable(bootstrapProperties.getProperty(NOTIFIER_INGESTORS_KEY)) + .filter(not(String::isBlank)) + .map(ingestors -> ingestors.split(COMMA)) + .stream() + .flatMap(Stream::of) + .map(String::trim) + .forEach(ingestorClassname -> instantiateIngestor(bootstrapProperties, ingestorClassname)); + } + + private void closeIngestors() { try { for (ChangeIngestor changeIngestor : changeIngestors) { changeIngestor.close(); @@ -103,25 +115,15 @@ public class ConfigurationChangeCoordinator implements Closeable, ConfigurationC } } - private void initialize() throws IOException { - close(); - Properties bootstrapProperties = bootstrapFileProvider.getBootstrapProperties(); - // cleanup previously initialized ingestors - String ingestorsCsv = bootstrapProperties.getProperty(NOTIFIER_INGESTORS_KEY); - - if (ingestorsCsv != null && !ingestorsCsv.isEmpty()) { - for (String ingestorClassname : ingestorsCsv.split(",")) { - ingestorClassname = ingestorClassname.trim(); - try { - Class ingestorClass = Class.forName(ingestorClassname); - ChangeIngestor changeIngestor = (ChangeIngestor) ingestorClass.newInstance(); - changeIngestor.initialize(bootstrapProperties, runMiNiFi, this); - changeIngestors.add(changeIngestor); - LOGGER.info("Initialized ingestor: {}", ingestorClassname); - } catch (Exception e) { - LOGGER.error("Instantiating [{}] ingestor failed", ingestorClassname, e); - } - } + private void instantiateIngestor(Properties bootstrapProperties, String ingestorClassname) { + try { + Class ingestorClass = Class.forName(ingestorClassname); + ChangeIngestor changeIngestor = (ChangeIngestor) ingestorClass.newInstance(); + changeIngestor.initialize(bootstrapProperties, runMiNiFi, this); + changeIngestors.add(changeIngestor); + LOGGER.info("Initialized ingestor: {}", ingestorClassname); + } catch (Exception e) { + LOGGER.error("Instantiating [{}] ingestor failed", ingestorClassname, e); } } } diff --git a/minifi/minifi-bootstrap/src/main/java/org/apache/nifi/minifi/bootstrap/configuration/ConfigurationChangeListener.java b/minifi/minifi-bootstrap/src/main/java/org/apache/nifi/minifi/bootstrap/configuration/ConfigurationChangeListener.java index 642ed4baf4..11d421b045 100644 --- a/minifi/minifi-bootstrap/src/main/java/org/apache/nifi/minifi/bootstrap/configuration/ConfigurationChangeListener.java +++ b/minifi/minifi-bootstrap/src/main/java/org/apache/nifi/minifi/bootstrap/configuration/ConfigurationChangeListener.java @@ -19,8 +19,7 @@ package org.apache.nifi.minifi.bootstrap.configuration; import java.io.InputStream; /** - * Interface for handling events detected and driven by an associated {@link ConfigurationChangeCoordinator} to which the listener - * has registered via {@link ConfigurationChangeCoordinator#registerListener(ConfigurationChangeListener)}. + * Interface for handling events detected and driven by an associated {@link ConfigurationChangeCoordinator} */ public interface ConfigurationChangeListener { diff --git a/minifi/minifi-bootstrap/src/main/java/org/apache/nifi/minifi/bootstrap/configuration/ConfigurationChangeNotifier.java b/minifi/minifi-bootstrap/src/main/java/org/apache/nifi/minifi/bootstrap/configuration/ConfigurationChangeNotifier.java index 2ebced5b9d..5ac860e2aa 100644 --- a/minifi/minifi-bootstrap/src/main/java/org/apache/nifi/minifi/bootstrap/configuration/ConfigurationChangeNotifier.java +++ b/minifi/minifi-bootstrap/src/main/java/org/apache/nifi/minifi/bootstrap/configuration/ConfigurationChangeNotifier.java @@ -19,11 +19,11 @@ package org.apache.nifi.minifi.bootstrap.configuration; import java.nio.ByteBuffer; import java.util.Collection; -import java.util.Set; public interface ConfigurationChangeNotifier { - Set getChangeListeners(); - - Collection notifyListeners(ByteBuffer is); + /** + * Provides the mechanism by which listeners are notified + */ + Collection notifyListeners(ByteBuffer newFlowConfig); } diff --git a/minifi/minifi-bootstrap/src/main/java/org/apache/nifi/minifi/bootstrap/configuration/differentiators/WholeConfigDifferentiator.java b/minifi/minifi-bootstrap/src/main/java/org/apache/nifi/minifi/bootstrap/configuration/differentiators/WholeConfigDifferentiator.java index 5a081d33b6..04486948e3 100644 --- a/minifi/minifi-bootstrap/src/main/java/org/apache/nifi/minifi/bootstrap/configuration/differentiators/WholeConfigDifferentiator.java +++ b/minifi/minifi-bootstrap/src/main/java/org/apache/nifi/minifi/bootstrap/configuration/differentiators/WholeConfigDifferentiator.java @@ -17,10 +17,8 @@ package org.apache.nifi.minifi.bootstrap.configuration.differentiators; -import java.io.DataInputStream; -import java.io.EOFException; -import java.io.IOException; -import java.io.InputStream; +import static java.util.Optional.ofNullable; + import java.nio.ByteBuffer; import java.util.concurrent.atomic.AtomicReference; import org.apache.nifi.minifi.bootstrap.ConfigurationFileHolder; @@ -29,59 +27,30 @@ import org.slf4j.LoggerFactory; public abstract class WholeConfigDifferentiator { + public static final String WHOLE_CONFIG_KEY = "Whole Config"; private final static Logger logger = LoggerFactory.getLogger(WholeConfigDifferentiator.class); - public static final String WHOLE_CONFIG_KEY = "Whole Config"; - - volatile ConfigurationFileHolder configurationFileHolder; - - boolean compareInputStreamToConfigFile(InputStream inputStream) throws IOException { - logger.debug("Checking if change is different"); - AtomicReference currentConfigFileReference = configurationFileHolder.getConfigFileReference(); - ByteBuffer currentConfigFile = currentConfigFileReference.get(); - ByteBuffer byteBuffer = ByteBuffer.allocate(currentConfigFile.limit()); - DataInputStream dataInputStream = new DataInputStream(inputStream); - try { - dataInputStream.readFully(byteBuffer.array()); - } catch (EOFException e) { - logger.debug("New config is shorter than the current. Must be different."); - return true; - } - logger.debug("Read the input"); - - if (dataInputStream.available() != 0) { - return true; - } else { - return byteBuffer.compareTo(currentConfigFile) != 0; - } - } + protected volatile ConfigurationFileHolder configurationFileHolder; public void initialize(ConfigurationFileHolder configurationFileHolder) { this.configurationFileHolder = configurationFileHolder; } - - public static class InputStreamInput extends WholeConfigDifferentiator implements Differentiator { - public boolean isNew(InputStream inputStream) throws IOException { - return compareInputStreamToConfigFile(inputStream); + public static class ByteBufferInputDifferentiator extends WholeConfigDifferentiator implements Differentiator { + public boolean isNew(ByteBuffer newFlowConfig) { + AtomicReference currentFlowConfigReference = configurationFileHolder.getConfigFileReference(); + ByteBuffer currentFlowConfig = currentFlowConfigReference.get(); + boolean compareResult = ofNullable(currentFlowConfig) + .map(newFlowConfig::compareTo) + .map(result -> result != 0) + .orElse(Boolean.TRUE); + logger.debug("New flow is different from existing flow: {}", compareResult); + return compareResult; } } - public static class ByteBufferInput extends WholeConfigDifferentiator implements Differentiator { - public boolean isNew(ByteBuffer inputBuffer) { - AtomicReference currentConfigFileReference = configurationFileHolder.getConfigFileReference(); - ByteBuffer currentConfigFile = currentConfigFileReference.get(); - return inputBuffer.compareTo(currentConfigFile) != 0; - } - } - - - public static Differentiator getInputStreamDifferentiator() { - return new InputStreamInput(); - } - public static Differentiator getByteBufferDifferentiator() { - return new ByteBufferInput(); + return new ByteBufferInputDifferentiator(); } } diff --git a/minifi/minifi-bootstrap/src/main/java/org/apache/nifi/minifi/bootstrap/configuration/ingestors/AbstractPullChangeIngestor.java b/minifi/minifi-bootstrap/src/main/java/org/apache/nifi/minifi/bootstrap/configuration/ingestors/AbstractPullChangeIngestor.java index 1e217a896d..8d3b225df7 100644 --- a/minifi/minifi-bootstrap/src/main/java/org/apache/nifi/minifi/bootstrap/configuration/ingestors/AbstractPullChangeIngestor.java +++ b/minifi/minifi-bootstrap/src/main/java/org/apache/nifi/minifi/bootstrap/configuration/ingestors/AbstractPullChangeIngestor.java @@ -26,25 +26,22 @@ import java.util.concurrent.atomic.AtomicReference; import org.apache.nifi.minifi.bootstrap.ConfigurationFileHolder; import org.apache.nifi.minifi.bootstrap.configuration.ConfigurationChangeNotifier; import org.apache.nifi.minifi.bootstrap.configuration.ingestors.interfaces.ChangeIngestor; -import org.slf4j.Logger; public abstract class AbstractPullChangeIngestor implements Runnable, ChangeIngestor { - // 5 minute default pulling period - protected static final String DEFAULT_POLLING_PERIOD = "300000"; - protected static Logger logger; + protected static final String DEFAULT_POLLING_PERIOD_MILLISECONDS = "300000"; protected final AtomicInteger pollingPeriodMS = new AtomicInteger(); - private final ScheduledThreadPoolExecutor scheduledThreadPoolExecutor = new ScheduledThreadPoolExecutor(1); - protected volatile ConfigurationChangeNotifier configurationChangeNotifier; - protected volatile ConfigurationFileHolder configurationFileHolder; protected final AtomicReference properties = new AtomicReference<>(); + private final ScheduledThreadPoolExecutor scheduledThreadPoolExecutor = new ScheduledThreadPoolExecutor(1); + + protected volatile ConfigurationChangeNotifier configurationChangeNotifier; + @Override public void initialize(Properties properties, ConfigurationFileHolder configurationFileHolder, ConfigurationChangeNotifier configurationChangeNotifier) { this.configurationChangeNotifier = configurationChangeNotifier; this.properties.set(properties); - this.configurationFileHolder = configurationFileHolder; } @Override diff --git a/minifi/minifi-bootstrap/src/main/java/org/apache/nifi/minifi/bootstrap/configuration/ingestors/FileChangeIngestor.java b/minifi/minifi-bootstrap/src/main/java/org/apache/nifi/minifi/bootstrap/configuration/ingestors/FileChangeIngestor.java index 97260aa026..95a49a0d99 100644 --- a/minifi/minifi-bootstrap/src/main/java/org/apache/nifi/minifi/bootstrap/configuration/ingestors/FileChangeIngestor.java +++ b/minifi/minifi-bootstrap/src/main/java/org/apache/nifi/minifi/bootstrap/configuration/ingestors/FileChangeIngestor.java @@ -14,24 +14,29 @@ * See the License for the specific language governing permissions and * limitations under the License. */ + package org.apache.nifi.minifi.bootstrap.configuration.ingestors; +import static java.nio.ByteBuffer.wrap; import static java.nio.file.StandardWatchEventKinds.ENTRY_MODIFY; import static java.util.Collections.emptyList; +import static java.util.Optional.ofNullable; +import static java.util.concurrent.TimeUnit.SECONDS; +import static java.util.function.Predicate.not; +import static org.apache.commons.io.FilenameUtils.getBaseName; +import static org.apache.commons.io.IOUtils.toByteArray; import static org.apache.nifi.minifi.bootstrap.configuration.ConfigurationChangeCoordinator.NOTIFIER_INGESTORS_KEY; import static org.apache.nifi.minifi.bootstrap.configuration.differentiators.WholeConfigDifferentiator.WHOLE_CONFIG_KEY; +import static org.apache.nifi.minifi.commons.api.MiNiFiConstants.RAW_EXTENSION; import java.io.FileInputStream; import java.io.IOException; import java.nio.ByteBuffer; import java.nio.file.FileSystems; import java.nio.file.Path; -import java.nio.file.Paths; import java.nio.file.WatchEvent; import java.nio.file.WatchKey; import java.nio.file.WatchService; -import java.util.Collections; -import java.util.HashMap; import java.util.Map; import java.util.Optional; import java.util.Properties; @@ -39,13 +44,12 @@ import java.util.concurrent.Executors; import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.TimeUnit; import java.util.function.Supplier; -import org.apache.commons.io.IOUtils; import org.apache.nifi.minifi.bootstrap.ConfigurationFileHolder; import org.apache.nifi.minifi.bootstrap.configuration.ConfigurationChangeNotifier; import org.apache.nifi.minifi.bootstrap.configuration.differentiators.Differentiator; import org.apache.nifi.minifi.bootstrap.configuration.differentiators.WholeConfigDifferentiator; import org.apache.nifi.minifi.bootstrap.configuration.ingestors.interfaces.ChangeIngestor; -import org.apache.nifi.minifi.bootstrap.util.ConfigTransformer; +import org.apache.nifi.minifi.commons.api.MiNiFiProperties; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -56,165 +60,160 @@ import org.slf4j.LoggerFactory; */ public class FileChangeIngestor implements Runnable, ChangeIngestor { - private static final Map>> DIFFERENTIATOR_CONSTRUCTOR_MAP; + private static final Map>> DIFFERENTIATOR_CONSTRUCTOR_MAP = Map.of( + WHOLE_CONFIG_KEY, WholeConfigDifferentiator::getByteBufferDifferentiator + ); - static { - HashMap>> tempMap = new HashMap<>(); - tempMap.put(WHOLE_CONFIG_KEY, WholeConfigDifferentiator::getByteBufferDifferentiator); - - DIFFERENTIATOR_CONSTRUCTOR_MAP = Collections.unmodifiableMap(tempMap); - } - - - protected static final int DEFAULT_POLLING_PERIOD_INTERVAL = 15; - protected static final TimeUnit DEFAULT_POLLING_PERIOD_UNIT = TimeUnit.SECONDS; + static final String CONFIG_FILE_BASE_KEY = NOTIFIER_INGESTORS_KEY + ".file"; + static final String CONFIG_FILE_PATH_KEY = CONFIG_FILE_BASE_KEY + ".config.path"; + static final String POLLING_PERIOD_INTERVAL_KEY = CONFIG_FILE_BASE_KEY + ".polling.period.seconds"; + static final int DEFAULT_POLLING_PERIOD_INTERVAL = 15; private final static Logger logger = LoggerFactory.getLogger(FileChangeIngestor.class); - private static final String CONFIG_FILE_BASE_KEY = NOTIFIER_INGESTORS_KEY + ".file"; - protected static final String CONFIG_FILE_PATH_KEY = CONFIG_FILE_BASE_KEY + ".config.path"; - protected static final String POLLING_PERIOD_INTERVAL_KEY = CONFIG_FILE_BASE_KEY + ".polling.period.seconds"; - public static final String DIFFERENTIATOR_KEY = CONFIG_FILE_BASE_KEY + ".differentiator"; + private static final TimeUnit DEFAULT_POLLING_PERIOD_UNIT = SECONDS; + private static final String DIFFERENTIATOR_KEY = CONFIG_FILE_BASE_KEY + ".differentiator"; + + private volatile Differentiator differentiator; + private volatile ConfigurationChangeNotifier configurationChangeNotifier; + + private ScheduledExecutorService executorService; private Path configFilePath; private WatchService watchService; private long pollingSeconds; - private volatile Differentiator differentiator; - private volatile ConfigurationChangeNotifier configurationChangeNotifier; - private volatile ConfigurationFileHolder configurationFileHolder; - private volatile Properties properties; - private ScheduledExecutorService executorService; - protected static WatchService initializeWatcher(Path filePath) { + @Override + public void initialize(Properties properties, ConfigurationFileHolder configurationFileHolder, ConfigurationChangeNotifier configurationChangeNotifier) { + Path configFile = ofNullable(properties.getProperty(CONFIG_FILE_PATH_KEY)) + .filter(not(String::isBlank)) + .map(Path::of) + .map(Path::toAbsolutePath) + .orElseThrow(() -> new IllegalArgumentException("Property, " + CONFIG_FILE_PATH_KEY + ", for the path of the config file must be specified")); try { - final WatchService fsWatcher = FileSystems.getDefault().newWatchService(); - final Path watchDirectory = filePath.getParent(); - watchDirectory.register(fsWatcher, ENTRY_MODIFY); + this.configurationChangeNotifier = configurationChangeNotifier; + this.configFilePath = configFile; + this.pollingSeconds = ofNullable(properties.getProperty(POLLING_PERIOD_INTERVAL_KEY, Long.toString(DEFAULT_POLLING_PERIOD_INTERVAL))) + .map(Long::parseLong) + .filter(duration -> duration > 0) + .map(duration -> SECONDS.convert(duration, DEFAULT_POLLING_PERIOD_UNIT)) + .orElseThrow(() -> new IllegalArgumentException("Cannot specify a polling period with duration <=0")); + this.watchService = initializeWatcher(configFile); + this.differentiator = ofNullable(properties.getProperty(DIFFERENTIATOR_KEY)) + .filter(not(String::isBlank)) + .map(differentiator -> ofNullable(DIFFERENTIATOR_CONSTRUCTOR_MAP.get(differentiator)) + .map(Supplier::get) + .orElseThrow(unableToFindDifferentiatorExceptionSupplier(differentiator))) + .orElseGet(WholeConfigDifferentiator::getByteBufferDifferentiator); + this.differentiator.initialize(configurationFileHolder); + } catch (Exception e) { + throw new IllegalStateException("Could not successfully initialize file change notifier", e); + } - return fsWatcher; - } catch (IOException ioe) { - throw new IllegalStateException("Unable to initialize a file system watcher for the path " + filePath, ioe); + checkConfigFileLocationCorrectness(properties, configFile); + } + + @Override + public void start() { + executorService = Executors.newScheduledThreadPool(1, runnable -> { + Thread notifierThread = Executors.defaultThreadFactory().newThread(runnable); + notifierThread.setName("File Change Notifier Thread"); + notifierThread.setDaemon(true); + return notifierThread; + }); + executorService.scheduleWithFixedDelay(this, 0, pollingSeconds, DEFAULT_POLLING_PERIOD_UNIT); + } + + @Override + public void run() { + logger.debug("Checking for a change in {}", configFilePath); + if (targetFileChanged()) { + logger.debug("Target file changed, checking if it's different than current flow"); + try (FileInputStream flowCandidateInputStream = new FileInputStream(configFilePath.toFile())) { + ByteBuffer newFlowConfig = wrap(toByteArray(flowCandidateInputStream)); + if (differentiator.isNew(newFlowConfig)) { + logger.debug("Current flow and new flow is different, notifying listener"); + configurationChangeNotifier.notifyListeners(newFlowConfig); + logger.debug("Listeners have been notified"); + } + } catch (Exception e) { + logger.error("Could not successfully notify listeners.", e); + } + } else { + logger.debug("No change detected in {}", configFilePath); } } - protected boolean targetChanged() { - boolean targetChanged; + @Override + public void close() { + if (executorService != null) { + executorService.shutdownNow(); + } + } - Optional watchKey = Optional.ofNullable(watchService.poll()); - - targetChanged = watchKey + boolean targetFileChanged() { + logger.trace("Attempting to acquire watch key"); + Optional watchKey = ofNullable(watchService.poll()); + logger.trace("Watch key acquire with value {}", watchKey); + boolean targetChanged = watchKey .map(WatchKey::pollEvents) .orElse(emptyList()) .stream() .anyMatch(watchEvent -> ENTRY_MODIFY == watchEvent.kind() && ((WatchEvent) watchEvent).context().equals(configFilePath.getName(configFilePath.getNameCount() - 1))); - + logger.debug("Target file changed: {}", targetChanged); // After completing inspection, reset for detection of subsequent change events watchKey.map(WatchKey::reset) .filter(valid -> !valid) .ifPresent(valid -> { + logger.error("Unable to reinitialize file system watcher."); throw new IllegalStateException("Unable to reinitialize file system watcher."); }); - + logger.trace("Watch key has been reset successfully"); return targetChanged; } - @Override - public void run() { - logger.debug("Checking for a change"); - if (targetChanged()) { - logger.debug("Target changed, checking if it's different than current flow."); - try (FileInputStream configFile = new FileInputStream(configFilePath.toFile())) { - ByteBuffer readOnlyNewConfig = - ConfigTransformer.overrideNonFlowSectionsFromOriginalSchema( - IOUtils.toByteArray(configFile), configurationFileHolder.getConfigFileReference().get().duplicate(), properties); - - if (differentiator.isNew(readOnlyNewConfig)) { - logger.debug("New change, notifying listener"); - configurationChangeNotifier.notifyListeners(readOnlyNewConfig); - logger.debug("Listeners notified"); - } - } catch (Exception e) { - logger.error("Could not successfully notify listeners.", e); - } - } - } - - @Override - public void initialize(Properties properties, ConfigurationFileHolder configurationFileHolder, ConfigurationChangeNotifier configurationChangeNotifier) { - this.properties = properties; - this.configurationFileHolder = configurationFileHolder; - final String rawPath = properties.getProperty(CONFIG_FILE_PATH_KEY); - final String rawPollingDuration = properties.getProperty(POLLING_PERIOD_INTERVAL_KEY, Long.toString(DEFAULT_POLLING_PERIOD_INTERVAL)); - - if (rawPath == null || rawPath.isEmpty()) { - throw new IllegalArgumentException("Property, " + CONFIG_FILE_PATH_KEY + ", for the path of the config file must be specified."); - } - + private WatchService initializeWatcher(Path filePath) { try { - setConfigFilePath(Paths.get(rawPath)); - setPollingPeriod(Long.parseLong(rawPollingDuration), DEFAULT_POLLING_PERIOD_UNIT); - setWatchService(initializeWatcher(configFilePath)); - } catch (Exception e) { - throw new IllegalStateException("Could not successfully initialize file change notifier.", e); + WatchService fileSystemWatcher = FileSystems.getDefault().newWatchService(); + Path watchDirectory = filePath.getParent(); + watchDirectory.register(fileSystemWatcher, ENTRY_MODIFY); + logger.trace("Watch service registered for {}", watchDirectory); + return fileSystemWatcher; + } catch (IOException ioe) { + throw new IllegalStateException("Unable to initialize a file system watcher for the path " + filePath, ioe); } - - this.configurationChangeNotifier = configurationChangeNotifier; - - final String differentiatorName = properties.getProperty(DIFFERENTIATOR_KEY); - - if (differentiatorName != null && !differentiatorName.isEmpty()) { - Supplier> differentiatorSupplier = DIFFERENTIATOR_CONSTRUCTOR_MAP.get(differentiatorName); - if (differentiatorSupplier == null) { - throw new IllegalArgumentException("Property, " + DIFFERENTIATOR_KEY + ", has value " + differentiatorName + " which does not " + - "correspond to any in the PullHttpChangeIngestor Map:" + DIFFERENTIATOR_CONSTRUCTOR_MAP.keySet()); - } - differentiator = differentiatorSupplier.get(); - } else { - differentiator = WholeConfigDifferentiator.getByteBufferDifferentiator(); - } - differentiator.initialize(configurationFileHolder); } - protected void setConfigFilePath(Path configFilePath) { + private Supplier unableToFindDifferentiatorExceptionSupplier(String differentiator) { + return () -> new IllegalArgumentException("Property, " + DIFFERENTIATOR_KEY + ", has value " + differentiator + + " which does not correspond to any in the FileChangeIngestor Map:" + DIFFERENTIATOR_CONSTRUCTOR_MAP.keySet()); + } + + private void checkConfigFileLocationCorrectness(Properties properties, Path configFile) { + Path flowConfigFile = Path.of(properties.getProperty(MiNiFiProperties.NIFI_MINIFI_FLOW_CONFIG.getKey())).toAbsolutePath(); + Path rawFlowConfigFile = flowConfigFile.getParent().resolve(getBaseName(flowConfigFile.toString()) + RAW_EXTENSION); + if (flowConfigFile.equals(configFile) || rawFlowConfigFile.equals(configFile)) { + throw new IllegalStateException("File ingestor config file (" + CONFIG_FILE_PATH_KEY + + ") must point to a different file than MiNiFi flow config file and raw flow config file"); + } + } + + // Methods exposed only for enable testing + void setConfigFilePath(Path configFilePath) { this.configFilePath = configFilePath; } - protected void setWatchService(WatchService watchService) { + void setWatchService(WatchService watchService) { this.watchService = watchService; } - protected void setConfigurationChangeNotifier(ConfigurationChangeNotifier configurationChangeNotifier) { + void setConfigurationChangeNotifier(ConfigurationChangeNotifier configurationChangeNotifier) { this.configurationChangeNotifier = configurationChangeNotifier; } - protected void setDifferentiator(Differentiator differentiator) { + void setDifferentiator(Differentiator differentiator) { this.differentiator = differentiator; } - - protected void setPollingPeriod(long duration, TimeUnit unit) { - if (duration < 0) { - throw new IllegalArgumentException("Cannot specify a polling period with duration <=0"); - } - this.pollingSeconds = TimeUnit.SECONDS.convert(duration, unit); - } - - @Override - public void start() { - executorService = Executors.newScheduledThreadPool(1, r -> { - Thread t = Executors.defaultThreadFactory().newThread(r); - t.setName("File Change Notifier Thread"); - t.setDaemon(true); - return t; - }); - this.executorService.scheduleWithFixedDelay(this, 0, pollingSeconds, DEFAULT_POLLING_PERIOD_UNIT); - } - - @Override - public void close() { - if (this.executorService != null) { - this.executorService.shutdownNow(); - } - } } diff --git a/minifi/minifi-bootstrap/src/main/java/org/apache/nifi/minifi/bootstrap/configuration/ingestors/PullHttpChangeIngestor.java b/minifi/minifi-bootstrap/src/main/java/org/apache/nifi/minifi/bootstrap/configuration/ingestors/PullHttpChangeIngestor.java index f321f19b1f..0b1238f1f4 100644 --- a/minifi/minifi-bootstrap/src/main/java/org/apache/nifi/minifi/bootstrap/configuration/ingestors/PullHttpChangeIngestor.java +++ b/minifi/minifi-bootstrap/src/main/java/org/apache/nifi/minifi/bootstrap/configuration/ingestors/PullHttpChangeIngestor.java @@ -17,8 +17,21 @@ package org.apache.nifi.minifi.bootstrap.configuration.ingestors; +import static java.lang.Boolean.FALSE; +import static java.lang.Boolean.parseBoolean; +import static java.lang.Long.parseLong; +import static java.nio.ByteBuffer.wrap; +import static java.util.Optional.ofNullable; +import static java.util.concurrent.TimeUnit.MILLISECONDS; +import static java.util.function.Predicate.not; +import static java.util.stream.Collectors.joining; +import static java.util.stream.Collectors.toMap; +import static org.apache.commons.lang3.StringUtils.EMPTY; +import static org.apache.commons.lang3.StringUtils.isNotBlank; import static org.apache.nifi.minifi.bootstrap.configuration.ConfigurationChangeCoordinator.NOTIFIER_INGESTORS_KEY; import static org.apache.nifi.minifi.bootstrap.configuration.differentiators.WholeConfigDifferentiator.WHOLE_CONFIG_KEY; +import static org.eclipse.jetty.http.HttpScheme.HTTP; +import static org.eclipse.jetty.http.HttpScheme.HTTPS; import java.io.FileInputStream; import java.io.IOException; @@ -26,49 +39,35 @@ import java.net.InetSocketAddress; import java.net.Proxy; import java.nio.ByteBuffer; import java.security.KeyStore; -import java.util.Collections; -import java.util.HashMap; +import java.util.Arrays; import java.util.Map; import java.util.Properties; -import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicReference; import java.util.function.Supplier; -import javax.net.ssl.SSLContext; +import java.util.stream.Stream; import javax.net.ssl.SSLSocketFactory; import javax.net.ssl.X509TrustManager; +import okhttp3.Authenticator; import okhttp3.Credentials; import okhttp3.HttpUrl; import okhttp3.OkHttpClient; import okhttp3.Request; import okhttp3.Response; import okhttp3.ResponseBody; +import org.apache.commons.lang3.StringUtils; import org.apache.nifi.minifi.bootstrap.ConfigurationFileHolder; import org.apache.nifi.minifi.bootstrap.configuration.ConfigurationChangeNotifier; import org.apache.nifi.minifi.bootstrap.configuration.differentiators.Differentiator; import org.apache.nifi.minifi.bootstrap.configuration.differentiators.WholeConfigDifferentiator; -import org.apache.nifi.minifi.bootstrap.util.ConfigTransformer; -import org.apache.nifi.minifi.commons.schema.common.StringUtil; import org.apache.nifi.security.ssl.StandardKeyStoreBuilder; import org.apache.nifi.security.ssl.StandardSslContextBuilder; import org.apache.nifi.security.ssl.StandardTrustManagerBuilder; +import org.slf4j.Logger; import org.slf4j.LoggerFactory; public class PullHttpChangeIngestor extends AbstractPullChangeIngestor { - private static final int NOT_MODIFIED_STATUS_CODE = 304; - private static final Map>> DIFFERENTIATOR_CONSTRUCTOR_MAP; - - static { - HashMap>> tempMap = new HashMap<>(); - tempMap.put(WHOLE_CONFIG_KEY, WholeConfigDifferentiator::getByteBufferDifferentiator); - - DIFFERENTIATOR_CONSTRUCTOR_MAP = Collections.unmodifiableMap(tempMap); - } - - private static final String DEFAULT_CONNECT_TIMEOUT_MS = "5000"; - private static final String DEFAULT_READ_TIMEOUT_MS = "15000"; - public static final String PULL_HTTP_BASE_KEY = NOTIFIER_INGESTORS_KEY + ".pull.http"; public static final String PULL_HTTP_POLLING_PERIOD_KEY = PULL_HTTP_BASE_KEY + ".period.ms"; public static final String PORT_KEY = PULL_HTTP_BASE_KEY + ".port"; @@ -90,261 +89,233 @@ public class PullHttpChangeIngestor extends AbstractPullChangeIngestor { public static final String DIFFERENTIATOR_KEY = PULL_HTTP_BASE_KEY + ".differentiator"; public static final String USE_ETAG_KEY = PULL_HTTP_BASE_KEY + ".use.etag"; public static final String OVERRIDE_SECURITY = PULL_HTTP_BASE_KEY + ".override.security"; + public static final String HTTP_HEADERS = PULL_HTTP_BASE_KEY + ".headers"; + + private static final Logger logger = LoggerFactory.getLogger(PullHttpChangeIngestor.class); + + private static final Map>> DIFFERENTIATOR_CONSTRUCTOR_MAP = Map.of( + WHOLE_CONFIG_KEY, WholeConfigDifferentiator::getByteBufferDifferentiator + ); + private static final int NOT_MODIFIED_STATUS_CODE = 304; + private static final String DEFAULT_CONNECT_TIMEOUT_MS = "5000"; + private static final String DEFAULT_READ_TIMEOUT_MS = "15000"; + private static final String DOUBLE_QUOTES = "\""; + private static final String ETAG_HEADER = "ETag"; + private static final String PROXY_AUTHORIZATION_HEADER = "Proxy-Authorization"; + private static final String DEFAULT_PATH = "/"; + private static final int BAD_REQUEST_STATUS_CODE = 400; + private static final String IF_NONE_MATCH_HEADER_KEY = "If-None-Match"; + private static final String HTTP_HEADERS_SEPARATOR = ","; + private static final String HTTP_HEADER_KEY_VALUE_SEPARATOR = ":"; private final AtomicReference httpClientReference = new AtomicReference<>(); private final AtomicReference portReference = new AtomicReference<>(); private final AtomicReference hostReference = new AtomicReference<>(); private final AtomicReference pathReference = new AtomicReference<>(); private final AtomicReference queryReference = new AtomicReference<>(); + private final AtomicReference> httpHeadersReference = new AtomicReference<>(); + private volatile Differentiator differentiator; private volatile String connectionScheme; private volatile String lastEtag = ""; private volatile boolean useEtag = false; - public PullHttpChangeIngestor() { - logger = LoggerFactory.getLogger(PullHttpChangeIngestor.class); - } - @Override public void initialize(Properties properties, ConfigurationFileHolder configurationFileHolder, ConfigurationChangeNotifier configurationChangeNotifier) { super.initialize(properties, configurationFileHolder, configurationChangeNotifier); - pollingPeriodMS.set(Integer.parseInt(properties.getProperty(PULL_HTTP_POLLING_PERIOD_KEY, DEFAULT_POLLING_PERIOD))); + pollingPeriodMS.set(Integer.parseInt(properties.getProperty(PULL_HTTP_POLLING_PERIOD_KEY, DEFAULT_POLLING_PERIOD_MILLISECONDS))); if (pollingPeriodMS.get() < 1) { - throw new IllegalArgumentException("Property, " + PULL_HTTP_POLLING_PERIOD_KEY + ", for the polling period ms must be set with a positive integer."); + throw new IllegalArgumentException("Property, " + PULL_HTTP_POLLING_PERIOD_KEY + ", for the polling period ms must be set with a positive integer"); } - final String host = properties.getProperty(HOST_KEY); - if (host == null || host.isEmpty()) { - throw new IllegalArgumentException("Property, " + HOST_KEY + ", for the hostname to pull configurations from must be specified."); - } + String host = ofNullable(properties.getProperty(HOST_KEY)) + .filter(StringUtils::isNotBlank) + .orElseThrow(() -> new IllegalArgumentException("Property, " + HOST_KEY + ", for the hostname to pull configurations from must be specified")); + String path = properties.getProperty(PATH_KEY, DEFAULT_PATH); + String query = properties.getProperty(QUERY_KEY, EMPTY); + Map httpHeaders = ofNullable(properties.getProperty(HTTP_HEADERS)) + .filter(StringUtils::isNotBlank) + .map(headers -> headers.split(HTTP_HEADERS_SEPARATOR)) + .stream() + .flatMap(Arrays::stream) + .map(String::trim) + .map(header -> header.split(HTTP_HEADER_KEY_VALUE_SEPARATOR)) + .filter(split -> split.length == 2) + .collect(toMap(split -> ofNullable(split[0]).map(String::trim).orElse(EMPTY), split -> ofNullable(split[1]).map(String::trim).orElse(EMPTY))); + logger.debug("Configured HTTP headers: {}", httpHeaders); - final String path = properties.getProperty(PATH_KEY, "/"); - final String query = properties.getProperty(QUERY_KEY, ""); - - final String portString = (String) properties.get(PORT_KEY); - final Integer port; - if (portString == null) { - throw new IllegalArgumentException("Property, " + PORT_KEY + ", for the hostname to pull configurations from must be specified."); - } else { - port = Integer.parseInt(portString); - } - - portReference.set(port); + ofNullable(properties.get(PORT_KEY)) + .map(String.class::cast) + .map(Integer::parseInt) + .ifPresentOrElse( + portReference::set, + () -> { + throw new IllegalArgumentException("Property, " + PORT_KEY + ", for the hostname to pull configurations from must be specified"); + }); hostReference.set(host); pathReference.set(path); queryReference.set(query); - - final String useEtagString = (String) properties.getOrDefault(USE_ETAG_KEY, "false"); - if ("true".equalsIgnoreCase(useEtagString) || "false".equalsIgnoreCase(useEtagString)) { - useEtag = Boolean.parseBoolean(useEtagString); - } else { - throw new IllegalArgumentException("Property, " + USE_ETAG_KEY + ", to specify whether to use the ETag header, must either be a value boolean value (\"true\" or \"false\") or left to " + - "the default value of \"false\". It is set to \"" + useEtagString + "\"."); - } + httpHeadersReference.set(httpHeaders); + useEtag = parseBoolean((String) properties.getOrDefault(USE_ETAG_KEY, FALSE.toString())); httpClientReference.set(null); - final OkHttpClient.Builder okHttpClientBuilder = new OkHttpClient.Builder(); + OkHttpClient.Builder okHttpClientBuilder = new OkHttpClient.Builder() + .connectTimeout(parseLong(properties.getProperty(CONNECT_TIMEOUT_KEY, DEFAULT_CONNECT_TIMEOUT_MS)), MILLISECONDS) + .readTimeout(parseLong(properties.getProperty(READ_TIMEOUT_KEY, DEFAULT_READ_TIMEOUT_MS)), MILLISECONDS) + .followRedirects(true); - // Set timeouts - okHttpClientBuilder.connectTimeout(Long.parseLong(properties.getProperty(CONNECT_TIMEOUT_KEY, DEFAULT_CONNECT_TIMEOUT_MS)), TimeUnit.MILLISECONDS); - okHttpClientBuilder.readTimeout(Long.parseLong(properties.getProperty(READ_TIMEOUT_KEY, DEFAULT_READ_TIMEOUT_MS)), TimeUnit.MILLISECONDS); + String proxyHost = properties.getProperty(PROXY_HOST_KEY); + if (isNotBlank(proxyHost)) { + ofNullable(properties.getProperty(PROXY_PORT_KEY)) + .filter(StringUtils::isNotBlank) + .map(Integer::parseInt) + .map(port -> new InetSocketAddress(proxyHost, port)) + .map(inetSocketAddress -> new Proxy(Proxy.Type.HTTP, inetSocketAddress)) + .ifPresentOrElse( + okHttpClientBuilder::proxy, + () -> { + throw new IllegalArgumentException("Proxy port required if proxy specified"); + }); - // Set whether to follow redirects - okHttpClientBuilder.followRedirects(true); - - String proxyHost = properties.getProperty(PROXY_HOST_KEY, ""); - if (!proxyHost.isEmpty()) { - String proxyPort = properties.getProperty(PROXY_PORT_KEY); - if (proxyPort == null || proxyPort.isEmpty()) { - throw new IllegalArgumentException("Proxy port required if proxy specified."); - } - okHttpClientBuilder.proxy(new Proxy(Proxy.Type.HTTP, new InetSocketAddress(proxyHost, Integer.parseInt(proxyPort)))); - String proxyUsername = properties.getProperty(PROXY_USERNAME); - if (proxyUsername != null) { - String proxyPassword = properties.getProperty(PROXY_PASSWORD); - if (proxyPassword == null) { - throw new IllegalArgumentException("Must specify proxy password with proxy username."); - } - okHttpClientBuilder.proxyAuthenticator((route, response) -> response.request().newBuilder().addHeader("Proxy-Authorization", Credentials.basic(proxyUsername, proxyPassword)).build()); - } + ofNullable(properties.getProperty(PROXY_USERNAME)) + .filter(StringUtils::isNotBlank) + .ifPresent(proxyUserName -> + ofNullable(properties.getProperty(PROXY_PASSWORD)) + .map(proxyPassword -> Credentials.basic(proxyUserName, proxyPassword)) + .map(credentials -> (Authenticator) (route, response) -> response.request().newBuilder().addHeader(PROXY_AUTHORIZATION_HEADER, credentials).build()) + .ifPresentOrElse( + okHttpClientBuilder::proxyAuthenticator, + () -> { + throw new IllegalArgumentException("Must specify proxy password with proxy username"); + })); } - // check if the ssl path is set and add the factory if so if (properties.containsKey(KEYSTORE_LOCATION_KEY)) { - try { - setSslSocketFactory(okHttpClientBuilder, properties); - connectionScheme = "https"; - } catch (Exception e) { - throw new IllegalStateException(e); - } + connectionScheme = HTTPS.toString(); + setSslSocketFactory(okHttpClientBuilder, properties); } else { - connectionScheme = "http"; + connectionScheme = HTTP.toString(); } httpClientReference.set(okHttpClientBuilder.build()); - final String differentiatorName = properties.getProperty(DIFFERENTIATOR_KEY); - - if (differentiatorName != null && !differentiatorName.isEmpty()) { - Supplier> differentiatorSupplier = DIFFERENTIATOR_CONSTRUCTOR_MAP.get(differentiatorName); - if (differentiatorSupplier == null) { - throw new IllegalArgumentException("Property, " + DIFFERENTIATOR_KEY + ", has value " + differentiatorName + " which does not " + - "correspond to any in the PullHttpChangeIngestor Map:" + DIFFERENTIATOR_CONSTRUCTOR_MAP.keySet()); - } - differentiator = differentiatorSupplier.get(); - } else { - differentiator = WholeConfigDifferentiator.getByteBufferDifferentiator(); - } + differentiator = ofNullable(properties.getProperty(DIFFERENTIATOR_KEY)) + .filter(not(String::isBlank)) + .map(differentiator -> ofNullable(DIFFERENTIATOR_CONSTRUCTOR_MAP.get(differentiator)) + .map(Supplier::get) + .orElseThrow(unableToFindDifferentiatorExceptionSupplier(differentiator))) + .orElseGet(WholeConfigDifferentiator::getByteBufferDifferentiator); differentiator.initialize(configurationFileHolder); } - @Override public void run() { logger.debug("Attempting to pull new config"); HttpUrl.Builder builder = new HttpUrl.Builder() - .host(hostReference.get()) - .port(portReference.get()) - .encodedPath(pathReference.get()); - final String query = queryReference.get(); - if (!StringUtil.isNullOrEmpty(query)) { - builder = builder.encodedQuery(query); - } - final HttpUrl url = builder - .scheme(connectionScheme) - .build(); - - final Request.Builder requestBuilder = new Request.Builder() - .get() - .url(url); + .host(hostReference.get()) + .port(portReference.get()) + .encodedPath(pathReference.get()); + ofNullable(queryReference.get()) + .filter(StringUtils::isNotBlank) + .ifPresent(builder::encodedQuery); + HttpUrl url = builder.scheme(connectionScheme).build(); + Request.Builder requestBuilder = new Request.Builder().get().url(url); if (useEtag) { - requestBuilder.addHeader("If-None-Match", lastEtag); + requestBuilder.addHeader(IF_NONE_MATCH_HEADER_KEY, lastEtag); } + httpHeadersReference.get().forEach(requestBuilder::addHeader); - final Request request = requestBuilder.build(); + Request request = requestBuilder.build(); + logger.debug("Sending request: {}", request); - ResponseBody body = null; try (Response response = httpClientReference.get().newCall(request).execute()) { - logger.debug("Response received: {}", response.toString()); - + logger.debug("Response received: {}", response); int code = response.code(); - if (code == NOT_MODIFIED_STATUS_CODE) { return; } - - if (code >= 400) { + if (code >= BAD_REQUEST_STATUS_CODE) { throw new IOException("Got response code " + code + " while trying to pull configuration: " + response.body().string()); } - body = response.body(); - + ResponseBody body = response.body(); if (body == null) { logger.warn("No body returned when pulling a new configuration"); return; } - // checking if some parts of the configuration must be preserved - ByteBuffer readOnlyNewConfig = - ConfigTransformer.overrideNonFlowSectionsFromOriginalSchema(body.bytes(), configurationFileHolder.getConfigFileReference().get().duplicate(), properties.get()); - - if (differentiator.isNew(readOnlyNewConfig)) { + ByteBuffer newFlowConfig = wrap(body.bytes()).duplicate(); + if (differentiator.isNew(newFlowConfig)) { logger.debug("New change received, notifying listener"); - configurationChangeNotifier.notifyListeners(readOnlyNewConfig); + configurationChangeNotifier.notifyListeners(newFlowConfig); logger.debug("Listeners notified"); } else { - logger.debug("Pulled config same as currently running."); + logger.debug("Pulled config same as currently running"); } if (useEtag) { - lastEtag = (new StringBuilder("\"")) - .append(response.header("ETag").trim()) - .append("\"").toString(); + lastEtag = Stream.of(DOUBLE_QUOTES, response.header(ETAG_HEADER).trim(), DOUBLE_QUOTES).collect(joining()); } } catch (Exception e) { logger.warn("Hit an exception while trying to pull", e); } } - private void setSslSocketFactory(OkHttpClient.Builder okHttpClientBuilder, Properties properties) throws Exception { - final String keystoreLocation = properties.getProperty(KEYSTORE_LOCATION_KEY); - final String keystorePass = properties.getProperty(KEYSTORE_PASSWORD_KEY); - final String keystoreType = properties.getProperty(KEYSTORE_TYPE_KEY); - assertKeystorePropertiesSet(keystoreLocation, keystorePass, keystoreType); + private void setSslSocketFactory(OkHttpClient.Builder okHttpClientBuilder, Properties properties) { + String keystorePass = properties.getProperty(KEYSTORE_PASSWORD_KEY); + KeyStore keyStore = buildKeyStore(properties, KEYSTORE_LOCATION_KEY, KEYSTORE_PASSWORD_KEY, KEYSTORE_TYPE_KEY); + KeyStore truststore = buildKeyStore(properties, TRUSTSTORE_LOCATION_KEY, TRUSTSTORE_PASSWORD_KEY, TRUSTSTORE_TYPE_KEY); - final KeyStore keyStore; - try (final FileInputStream keyStoreStream = new FileInputStream(keystoreLocation)) { - keyStore = new StandardKeyStoreBuilder() - .type(keystoreType) - .inputStream(keyStoreStream) - .password(keystorePass.toCharArray()) - .build(); - } - - final String truststoreLocation = properties.getProperty(TRUSTSTORE_LOCATION_KEY); - final String truststorePass = properties.getProperty(TRUSTSTORE_PASSWORD_KEY); - final String truststoreType = properties.getProperty(TRUSTSTORE_TYPE_KEY); - assertTruststorePropertiesSet(truststoreLocation, truststorePass, truststoreType); - - final KeyStore truststore; - try (final FileInputStream trustStoreStream = new FileInputStream(truststoreLocation)) { - truststore = new StandardKeyStoreBuilder() - .type(truststoreType) - .inputStream(trustStoreStream) - .password(truststorePass.toCharArray()) - .build(); - } - - final X509TrustManager trustManager = new StandardTrustManagerBuilder().trustStore(truststore).build(); - final SSLContext sslContext = new StandardSslContextBuilder() - .keyStore(keyStore) - .keyPassword(keystorePass.toCharArray()) - .trustStore(truststore) - .build(); - final SSLSocketFactory sslSocketFactory = sslContext.getSocketFactory(); + SSLSocketFactory sslSocketFactory = new StandardSslContextBuilder() + .keyStore(keyStore) + .keyPassword(keystorePass.toCharArray()) + .trustStore(truststore) + .build() + .getSocketFactory(); + X509TrustManager trustManager = new StandardTrustManagerBuilder().trustStore(truststore).build(); okHttpClientBuilder.sslSocketFactory(sslSocketFactory, trustManager); } - private void assertKeystorePropertiesSet(String location, String password, String type) { - if (location == null || location.isEmpty()) { - throw new IllegalArgumentException(KEYSTORE_LOCATION_KEY + " is null or is empty"); - } + private KeyStore buildKeyStore(Properties properties, String locationKey, String passKey, String typeKey) { + String keystoreLocation = ofNullable(properties.getProperty(locationKey)) + .filter(StringUtils::isNotBlank) + .orElseThrow(() -> new IllegalArgumentException(locationKey + " is null or empty")); + String keystorePass = ofNullable(properties.getProperty(passKey)) + .filter(StringUtils::isNotBlank) + .orElseThrow(() -> new IllegalArgumentException(passKey + " is null or empty")); + String keystoreType = ofNullable(properties.getProperty(typeKey)) + .filter(StringUtils::isNotBlank) + .orElseThrow(() -> new IllegalArgumentException(typeKey + " is null or empty")); - if (password == null || password.isEmpty()) { - throw new IllegalArgumentException(KEYSTORE_LOCATION_KEY + " is set but " + KEYSTORE_PASSWORD_KEY + " is not (or is empty). If the location is set, the password must also be."); - } - - if (type == null || type.isEmpty()) { - throw new IllegalArgumentException(KEYSTORE_LOCATION_KEY + " is set but " + KEYSTORE_TYPE_KEY + " is not (or is empty). If the location is set, the type must also be."); + try (FileInputStream keyStoreStream = new FileInputStream(keystoreLocation)) { + return new StandardKeyStoreBuilder() + .type(keystoreType) + .inputStream(keyStoreStream) + .password(keystorePass.toCharArray()) + .build(); + } catch (IOException e) { + throw new IllegalStateException("Unable to create keyStore", e); } } - private void assertTruststorePropertiesSet(String location, String password, String type) { - if (location == null || location.isEmpty()) { - throw new IllegalArgumentException(TRUSTSTORE_LOCATION_KEY + " is not set or is empty"); - } - - if (password == null || password.isEmpty()) { - throw new IllegalArgumentException(TRUSTSTORE_LOCATION_KEY + " is set but " + TRUSTSTORE_PASSWORD_KEY + " is not (or is empty). If the location is set, the password must also be."); - } - - if (type == null || type.isEmpty()) { - throw new IllegalArgumentException(TRUSTSTORE_LOCATION_KEY + " is set but " + TRUSTSTORE_TYPE_KEY + " is not (or is empty). If the location is set, the type must also be."); - } + private Supplier unableToFindDifferentiatorExceptionSupplier(String differentiator) { + return () -> new IllegalArgumentException("Property, " + DIFFERENTIATOR_KEY + ", has value " + differentiator + + " which does not correspond to any in the FileChangeIngestor Map:" + DIFFERENTIATOR_CONSTRUCTOR_MAP.keySet()); } - protected void setDifferentiator(Differentiator differentiator) { + // Methods exposed only for enable testing + void setDifferentiator(Differentiator differentiator) { this.differentiator = differentiator; } - public void setLastEtag(String lastEtag) { + void setLastEtag(String lastEtag) { this.lastEtag = lastEtag; } - public void setUseEtag(boolean useEtag) { + void setUseEtag(boolean useEtag) { this.useEtag = useEtag; } } diff --git a/minifi/minifi-bootstrap/src/main/java/org/apache/nifi/minifi/bootstrap/configuration/ingestors/RestChangeIngestor.java b/minifi/minifi-bootstrap/src/main/java/org/apache/nifi/minifi/bootstrap/configuration/ingestors/RestChangeIngestor.java index d9695b81ad..a33520e318 100644 --- a/minifi/minifi-bootstrap/src/main/java/org/apache/nifi/minifi/bootstrap/configuration/ingestors/RestChangeIngestor.java +++ b/minifi/minifi-bootstrap/src/main/java/org/apache/nifi/minifi/bootstrap/configuration/ingestors/RestChangeIngestor.java @@ -17,6 +17,10 @@ package org.apache.nifi.minifi.bootstrap.configuration.ingestors; +import static java.nio.ByteBuffer.wrap; +import static java.util.Optional.ofNullable; +import static java.util.function.Predicate.not; +import static org.apache.commons.io.IOUtils.toByteArray; import static org.apache.nifi.minifi.bootstrap.configuration.ConfigurationChangeCoordinator.NOTIFIER_INGESTORS_KEY; import static org.apache.nifi.minifi.bootstrap.configuration.differentiators.WholeConfigDifferentiator.WHOLE_CONFIG_KEY; @@ -29,15 +33,12 @@ import java.nio.ByteBuffer; import java.security.KeyStore; import java.security.Security; import java.util.Collection; -import java.util.Collections; -import java.util.HashMap; import java.util.Map; import java.util.Properties; import java.util.function.Supplier; import javax.net.ssl.SSLContext; import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletResponse; -import org.apache.commons.io.IOUtils; import org.apache.nifi.jetty.configuration.connector.StandardServerConnectorFactory; import org.apache.nifi.minifi.bootstrap.ConfigurationFileHolder; import org.apache.nifi.minifi.bootstrap.configuration.ConfigurationChangeNotifier; @@ -45,7 +46,6 @@ import org.apache.nifi.minifi.bootstrap.configuration.ListenerHandleResult; import org.apache.nifi.minifi.bootstrap.configuration.differentiators.Differentiator; import org.apache.nifi.minifi.bootstrap.configuration.differentiators.WholeConfigDifferentiator; import org.apache.nifi.minifi.bootstrap.configuration.ingestors.interfaces.ChangeIngestor; -import org.apache.nifi.minifi.bootstrap.util.ConfigTransformer; import org.apache.nifi.security.ssl.StandardKeyStoreBuilder; import org.apache.nifi.security.ssl.StandardSslContextBuilder; import org.apache.nifi.security.util.TlsPlatform; @@ -62,25 +62,17 @@ import org.slf4j.LoggerFactory; public class RestChangeIngestor implements ChangeIngestor { - private static final Map>> DIFFERENTIATOR_CONSTRUCTOR_MAP; - static { - HashMap>> tempMap = new HashMap<>(); - tempMap.put(WHOLE_CONFIG_KEY, WholeConfigDifferentiator::getByteBufferDifferentiator); - - DIFFERENTIATOR_CONSTRUCTOR_MAP = Collections.unmodifiableMap(tempMap); Security.addProvider(new BouncyCastleProvider()); } - public static final String GET_TEXT = "This is a config change listener for an Apache NiFi - MiNiFi instance.\n" + - "Use this rest server to upload a conf.yml to configure the MiNiFi instance.\n" + - "Send a POST http request to '/' to upload the file."; - public static final String OTHER_TEXT = "This is not a support HTTP operation. Please use GET to get more information or POST to upload a new config.yml file.\n"; + "Use this rest server to upload a flow.json to configure the MiNiFi instance.\n" + + "Send a POST http request to '/' to upload the file."; + public static final String OTHER_TEXT = "This is not a supported HTTP operation. Please use GET to get more information or POST to upload a new flow.json file.\n"; public static final String POST = "POST"; public static final String GET = "GET"; - private final static Logger logger = LoggerFactory.getLogger(RestChangeIngestor.class); - private static final String RECEIVE_HTTP_BASE_KEY = NOTIFIER_INGESTORS_KEY + ".receive.http"; + public static final String RECEIVE_HTTP_BASE_KEY = NOTIFIER_INGESTORS_KEY + ".receive.http"; public static final String PORT_KEY = RECEIVE_HTTP_BASE_KEY + ".port"; public static final String HOST_KEY = RECEIVE_HTTP_BASE_KEY + ".host"; public static final String TRUSTSTORE_LOCATION_KEY = RECEIVE_HTTP_BASE_KEY + ".truststore.location"; @@ -91,12 +83,17 @@ public class RestChangeIngestor implements ChangeIngestor { public static final String KEYSTORE_TYPE_KEY = RECEIVE_HTTP_BASE_KEY + ".keystore.type"; public static final String NEED_CLIENT_AUTH_KEY = RECEIVE_HTTP_BASE_KEY + ".need.client.auth"; public static final String DIFFERENTIATOR_KEY = RECEIVE_HTTP_BASE_KEY + ".differentiator"; + + private final static Logger logger = LoggerFactory.getLogger(RestChangeIngestor.class); + + private static final Map>> DIFFERENTIATOR_CONSTRUCTOR_MAP = Map.of( + WHOLE_CONFIG_KEY, WholeConfigDifferentiator::getByteBufferDifferentiator + ); + private final Server jetty; private volatile Differentiator differentiator; private volatile ConfigurationChangeNotifier configurationChangeNotifier; - private volatile ConfigurationFileHolder configurationFileHolder; - private volatile Properties properties; public RestChangeIngestor() { QueuedThreadPool queuedThreadPool = new QueuedThreadPool(); @@ -106,36 +103,23 @@ public class RestChangeIngestor implements ChangeIngestor { @Override public void initialize(Properties properties, ConfigurationFileHolder configurationFileHolder, ConfigurationChangeNotifier configurationChangeNotifier) { - this.configurationFileHolder = configurationFileHolder; - this.properties = properties; - logger.info("Initializing"); - String differentiatorName = properties.getProperty(DIFFERENTIATOR_KEY); + logger.info("Initializing RestChangeIngestor"); + this.differentiator = ofNullable(properties.getProperty(DIFFERENTIATOR_KEY)) + .filter(not(String::isBlank)) + .map(differentiator -> ofNullable(DIFFERENTIATOR_CONSTRUCTOR_MAP.get(differentiator)) + .map(Supplier::get) + .orElseThrow(unableToFindDifferentiatorExceptionSupplier(differentiator))) + .orElseGet(WholeConfigDifferentiator::getByteBufferDifferentiator); + this.differentiator.initialize(configurationFileHolder); - if (differentiatorName != null && !differentiatorName.isEmpty()) { - Supplier> differentiatorSupplier = DIFFERENTIATOR_CONSTRUCTOR_MAP.get(differentiatorName); - if (differentiatorSupplier == null) { - throw new IllegalArgumentException("Property, " + DIFFERENTIATOR_KEY + ", has value " + differentiatorName + " which does not " + - "correspond to any in the PullHttpChangeIngestor Map:" + DIFFERENTIATOR_CONSTRUCTOR_MAP.keySet()); - } - differentiator = differentiatorSupplier.get(); - } else { - differentiator = WholeConfigDifferentiator.getByteBufferDifferentiator(); - } - differentiator.initialize(configurationFileHolder); - - // create the secure connector if keystore location is specified - if (properties.getProperty(KEYSTORE_LOCATION_KEY) != null) { - createSecureConnector(properties); - } else { - // create the unsecure connector otherwise - createConnector(properties); - } + ofNullable(properties.getProperty(KEYSTORE_LOCATION_KEY)) + .ifPresentOrElse(keyStoreLocation -> createSecureConnector(properties), () -> createConnector(properties)); this.configurationChangeNotifier = configurationChangeNotifier; HandlerCollection handlerCollection = new HandlerCollection(true); handlerCollection.addHandler(new JettyHandler()); - jetty.setHandler(handlerCollection); + this.jetty.setHandler(handlerCollection); } @Override @@ -231,7 +215,13 @@ public class RestChangeIngestor implements ChangeIngestor { logger.info("HTTPS Connector added for Host [{}] and Port [{}]", https.getHost(), https.getPort()); } - protected void setDifferentiator(Differentiator differentiator) { + private Supplier unableToFindDifferentiatorExceptionSupplier(String differentiator) { + return () -> new IllegalArgumentException("Property, " + DIFFERENTIATOR_KEY + ", has value " + differentiator + + " which does not correspond to any in the FileChangeIngestor Map:" + DIFFERENTIATOR_CONSTRUCTOR_MAP.keySet()); + } + + // Method exposed only for enable testing + void setDifferentiator(Differentiator differentiator) { this.differentiator = differentiator; } @@ -239,7 +229,7 @@ public class RestChangeIngestor implements ChangeIngestor { @Override public void handle(String target, Request baseRequest, HttpServletRequest request, HttpServletResponse response) - throws IOException { + throws IOException { logRequest(request); @@ -249,13 +239,9 @@ public class RestChangeIngestor implements ChangeIngestor { int statusCode; String responseText; try { - ByteBuffer readOnlyNewConfig = - ConfigTransformer.overrideNonFlowSectionsFromOriginalSchema( - IOUtils.toByteArray(request.getInputStream()), configurationFileHolder.getConfigFileReference().get().duplicate(), properties); - - if (differentiator.isNew(readOnlyNewConfig)) { - - Collection listenerHandleResults = configurationChangeNotifier.notifyListeners(readOnlyNewConfig); + ByteBuffer newFlowConfig = wrap(toByteArray(request.getInputStream())).duplicate(); + if (differentiator.isNew(newFlowConfig)) { + Collection listenerHandleResults = configurationChangeNotifier.notifyListeners(newFlowConfig); statusCode = 200; for (ListenerHandleResult result : listenerHandleResults) { @@ -312,6 +298,5 @@ public class RestChangeIngestor implements ChangeIngestor { logger.info("request content type = " + request.getContentType()); logger.info("<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<"); } - } } diff --git a/minifi/minifi-bootstrap/src/main/java/org/apache/nifi/minifi/bootstrap/service/BootstrapCodec.java b/minifi/minifi-bootstrap/src/main/java/org/apache/nifi/minifi/bootstrap/service/BootstrapCodec.java index 9d95f941f0..46a0bbdf6c 100644 --- a/minifi/minifi-bootstrap/src/main/java/org/apache/nifi/minifi/bootstrap/service/BootstrapCodec.java +++ b/minifi/minifi-bootstrap/src/main/java/org/apache/nifi/minifi/bootstrap/service/BootstrapCodec.java @@ -17,6 +17,8 @@ package org.apache.nifi.minifi.bootstrap.service; +import static org.apache.commons.lang3.StringUtils.SPACE; + import java.io.BufferedReader; import java.io.BufferedWriter; import java.io.IOException; @@ -40,12 +42,10 @@ public class BootstrapCodec { private final RunMiNiFi runner; private final Logger logger = LoggerFactory.getLogger(BootstrapCodec.class); private final UpdatePropertiesService updatePropertiesService; - private final UpdateConfigurationService updateConfigurationService; public BootstrapCodec(RunMiNiFi runner, BootstrapFileProvider bootstrapFileProvider, ConfigurationChangeListener configurationChangeListener) { this.runner = runner; this.updatePropertiesService = new UpdatePropertiesService(runner, logger, bootstrapFileProvider); - this.updateConfigurationService = new UpdateConfigurationService(runner, configurationChangeListener, bootstrapFileProvider); } public void communicate(InputStream in, OutputStream out) throws IOException { @@ -53,18 +53,13 @@ public class BootstrapCodec { BufferedWriter writer = new BufferedWriter(new OutputStreamWriter(out)); String line = reader.readLine(); - String[] splits = Optional.ofNullable(line).map(l -> l.split(" ")).orElse(new String[0]); - if (splits.length == 0) { - throw new IOException("Received invalid command from MiNiFi: " + line); - } + String[] splits = Optional.ofNullable(line) + .map(l -> l.split(SPACE)) + .filter(items -> items.length > 0) + .orElseThrow(() -> new IOException("Received invalid command from MiNiFi: " + line)); BootstrapCommand cmd = BootstrapCommand.safeValueOf(splits[0]); - String[] args; - if (splits.length == 1) { - args = new String[0]; - } else { - args = Arrays.copyOfRange(splits, 1, splits.length); - } + String[] args = splits.length == 1 ? new String[0] : Arrays.copyOfRange(splits, 1, splits.length); try { processRequest(cmd, args, writer); @@ -90,21 +85,11 @@ public class BootstrapCodec { case UPDATE_PROPERTIES: handlePropertiesUpdateCommand(writer); break; - case UPDATE_CONFIGURATION: - handleUpdateConfigurationCommand(writer); - break; default: throw new InvalidCommandException("Unknown command: " + cmd); } } - private void handleUpdateConfigurationCommand(BufferedWriter writer) throws IOException { - logger.debug("Received 'UPDATE_CONFIGURATION' command from MINIFI"); - writeOk(writer); - runner.setCommandInProgress(true); - updateConfigurationService.handleUpdate().ifPresent(runner::sendAcknowledgeToMiNiFi); - } - private void handlePropertiesUpdateCommand(BufferedWriter writer) throws IOException { logger.debug("Received 'UPDATE_PROPERTIES' command from MINIFI"); writeOk(writer); @@ -170,7 +155,7 @@ public class BootstrapCodec { } private enum BootstrapCommand { - PORT, STARTED, SHUTDOWN, RELOAD, UPDATE_PROPERTIES, UPDATE_CONFIGURATION, UNKNOWN; + PORT, STARTED, SHUTDOWN, RELOAD, UPDATE_PROPERTIES, UNKNOWN; public static BootstrapCommand safeValueOf(String value) { try { diff --git a/minifi/minifi-bootstrap/src/main/java/org/apache/nifi/minifi/bootstrap/service/BootstrapFileProvider.java b/minifi/minifi-bootstrap/src/main/java/org/apache/nifi/minifi/bootstrap/service/BootstrapFileProvider.java index ee94602aea..aaef9afbc7 100644 --- a/minifi/minifi-bootstrap/src/main/java/org/apache/nifi/minifi/bootstrap/service/BootstrapFileProvider.java +++ b/minifi/minifi-bootstrap/src/main/java/org/apache/nifi/minifi/bootstrap/service/BootstrapFileProvider.java @@ -94,14 +94,6 @@ public class BootstrapFileProvider { return reloadFile; } - public File getConfigYmlSwapFile() { - File confDir = bootstrapConfigFile.getParentFile(); - File swapFile = new File(confDir, "swap.yml"); - - LOGGER.debug("Swap File: {}", swapFile); - return swapFile; - } - public File getBootstrapConfSwapFile() { File confDir = bootstrapConfigFile.getParentFile(); File swapFile = new File(confDir, "bootstrap-swap.conf"); diff --git a/minifi/minifi-bootstrap/src/main/java/org/apache/nifi/minifi/bootstrap/service/MiNiFiConfigurationChangeListener.java b/minifi/minifi-bootstrap/src/main/java/org/apache/nifi/minifi/bootstrap/service/MiNiFiConfigurationChangeListener.java index 7800d4b28a..f043f6d5a1 100644 --- a/minifi/minifi-bootstrap/src/main/java/org/apache/nifi/minifi/bootstrap/service/MiNiFiConfigurationChangeListener.java +++ b/minifi/minifi-bootstrap/src/main/java/org/apache/nifi/minifi/bootstrap/service/MiNiFiConfigurationChangeListener.java @@ -14,82 +14,48 @@ * See the License for the specific language governing permissions and * limitations under the License. */ + package org.apache.nifi.minifi.bootstrap.service; -import static java.nio.file.StandardCopyOption.REPLACE_EXISTING; -import static org.apache.nifi.minifi.bootstrap.RunMiNiFi.CONF_DIR_KEY; -import static org.apache.nifi.minifi.bootstrap.RunMiNiFi.MINIFI_CONFIG_FILE_KEY; -import static org.apache.nifi.minifi.bootstrap.util.ConfigTransformer.generateConfigFiles; +import static java.nio.ByteBuffer.wrap; +import static java.nio.charset.StandardCharsets.UTF_8; +import static org.apache.commons.io.IOUtils.closeQuietly; +import static org.apache.commons.io.IOUtils.toByteArray; +import static org.apache.nifi.minifi.commons.api.MiNiFiConstants.BACKUP_EXTENSION; +import static org.apache.nifi.minifi.commons.api.MiNiFiConstants.RAW_EXTENSION; +import static org.apache.nifi.minifi.commons.util.FlowUpdateUtils.backup; +import static org.apache.nifi.minifi.commons.util.FlowUpdateUtils.persist; +import static org.apache.nifi.minifi.commons.util.FlowUpdateUtils.removeIfExists; +import static org.apache.nifi.minifi.commons.util.FlowUpdateUtils.revert; -import java.io.File; -import java.io.FileInputStream; import java.io.IOException; import java.io.InputStream; import java.nio.ByteBuffer; -import java.nio.file.Files; +import java.nio.file.Path; import java.util.Properties; import java.util.concurrent.locks.ReentrantLock; -import org.apache.commons.io.IOUtils; +import org.apache.commons.io.FilenameUtils; import org.apache.nifi.minifi.bootstrap.RunMiNiFi; import org.apache.nifi.minifi.bootstrap.configuration.ConfigurationChangeException; import org.apache.nifi.minifi.bootstrap.configuration.ConfigurationChangeListener; +import org.apache.nifi.minifi.commons.api.MiNiFiProperties; +import org.apache.nifi.minifi.commons.service.FlowEnrichService; import org.slf4j.Logger; public class MiNiFiConfigurationChangeListener implements ConfigurationChangeListener { + private static final ReentrantLock handlingLock = new ReentrantLock(); + private final RunMiNiFi runner; private final Logger logger; private final BootstrapFileProvider bootstrapFileProvider; + private final FlowEnrichService flowEnrichService; - private static final ReentrantLock handlingLock = new ReentrantLock(); - - public MiNiFiConfigurationChangeListener(RunMiNiFi runner, Logger logger, BootstrapFileProvider bootstrapFileProvider) { + public MiNiFiConfigurationChangeListener(RunMiNiFi runner, Logger logger, BootstrapFileProvider bootstrapFileProvider, FlowEnrichService flowEnrichService) { this.runner = runner; this.logger = logger; this.bootstrapFileProvider = bootstrapFileProvider; - } - - @Override - public void handleChange(InputStream configInputStream) throws ConfigurationChangeException { - logger.info("Received notification of a change"); - - if (!handlingLock.tryLock()) { - throw new ConfigurationChangeException("Instance is already handling another change"); - } - // Store the incoming stream as a byte array to be shared among components that need it - try { - Properties bootstrapProperties = bootstrapFileProvider.getBootstrapProperties(); - File configFile = new File(bootstrapProperties.getProperty(MINIFI_CONFIG_FILE_KEY)); - - File swapConfigFile = bootstrapFileProvider.getConfigYmlSwapFile(); - logger.info("Persisting old configuration to {}", swapConfigFile.getAbsolutePath()); - - try (FileInputStream configFileInputStream = new FileInputStream(configFile)) { - Files.copy(configFileInputStream, swapConfigFile.toPath(), REPLACE_EXISTING); - } - - // write out new config to file - Files.copy(configInputStream, configFile.toPath(), REPLACE_EXISTING); - - // Create an input stream to feed to the config transformer - try (FileInputStream newConfigIs = new FileInputStream(configFile)) { - try { - String confDir = bootstrapProperties.getProperty(CONF_DIR_KEY); - transformConfigurationFiles(confDir, newConfigIs, configFile, swapConfigFile); - } catch (Exception e) { - logger.debug("Transformation of new config file failed after swap file was created, deleting it."); - if (!swapConfigFile.delete()) { - logger.warn("The swap file failed to delete after a failed handling of a change. It should be cleaned up manually."); - } - throw e; - } - } - } catch (Exception e) { - throw new ConfigurationChangeException("Unable to perform reload of received configuration change", e); - } finally { - IOUtils.closeQuietly(configInputStream); - handlingLock.unlock(); - } + this.flowEnrichService = flowEnrichService; } @Override @@ -97,33 +63,53 @@ public class MiNiFiConfigurationChangeListener implements ConfigurationChangeLis return "MiNiFiConfigurationChangeListener"; } - private void transformConfigurationFiles(String confDir, FileInputStream newConfigIs, File configFile, File swapConfigFile) throws Exception { + @Override + public void handleChange(InputStream flowConfigInputStream) throws ConfigurationChangeException { + logger.info("Received notification of a change"); + + if (!handlingLock.tryLock()) { + throw new ConfigurationChangeException("Instance is already handling another change"); + } + + Path currentFlowConfigFile = null; + Path backupFlowConfigFile = null; + Path currentRawFlowConfigFile = null; + Path backupRawFlowConfigFile = null; try { - logger.info("Performing transformation for input and saving outputs to {}", confDir); - ByteBuffer tempConfigFile = generateConfigFiles(newConfigIs, confDir, bootstrapFileProvider.getBootstrapProperties()); - runner.getConfigFileReference().set(tempConfigFile.asReadOnlyBuffer()); - reloadNewConfiguration(swapConfigFile, confDir); + Properties bootstrapProperties = bootstrapFileProvider.getBootstrapProperties(); + + currentFlowConfigFile = Path.of(bootstrapProperties.getProperty(MiNiFiProperties.NIFI_MINIFI_FLOW_CONFIG.getKey())).toAbsolutePath(); + backupFlowConfigFile = Path.of(currentFlowConfigFile + BACKUP_EXTENSION); + String currentFlowConfigFileBaseName = FilenameUtils.getBaseName(currentFlowConfigFile.toString()); + currentRawFlowConfigFile = currentFlowConfigFile.getParent().resolve(currentFlowConfigFileBaseName + RAW_EXTENSION); + backupRawFlowConfigFile = currentFlowConfigFile.getParent().resolve(currentFlowConfigFileBaseName + RAW_EXTENSION + BACKUP_EXTENSION); + + backup(currentFlowConfigFile, backupFlowConfigFile); + backup(currentRawFlowConfigFile, backupRawFlowConfigFile); + + byte[] rawFlow = toByteArray(flowConfigInputStream); + byte[] enrichedFlow = flowEnrichService.enrichFlow(rawFlow); + persist(enrichedFlow, currentFlowConfigFile, true); + restartInstance(); + persist(rawFlow, currentRawFlowConfigFile, false); + setActiveFlowReference(wrap(rawFlow)); + logger.info("MiNiFi has finished reloading successfully and applied the new flow configuration"); } catch (Exception e) { - logger.debug("Transformation of new config file failed after replacing original with the swap file, reverting."); - try (FileInputStream swapConfigFileStream = new FileInputStream(swapConfigFile)) { - Files.copy(swapConfigFileStream, configFile.toPath(), REPLACE_EXISTING); - } - throw e; + logger.error("Configuration update failed. Reverting to previous flow", e); + revert(backupFlowConfigFile, currentFlowConfigFile); + revert(backupRawFlowConfigFile, currentRawFlowConfigFile); + throw new ConfigurationChangeException("Unable to perform reload of received configuration change", e); + } finally { + removeIfExists(backupFlowConfigFile); + removeIfExists(backupRawFlowConfigFile); + closeQuietly(flowConfigInputStream); + handlingLock.unlock(); } } - private void reloadNewConfiguration(File swapConfigFile, String confDir) throws Exception { - try { - logger.info("Reloading instance with new configuration"); - restartInstance(); - } catch (Exception e) { - logger.debug("Transformation of new config file failed after transformation into Flow.xml and nifi.properties, reverting."); - try (FileInputStream swapConfigFileStream = new FileInputStream(swapConfigFile)) { - ByteBuffer resetConfigFile = generateConfigFiles(swapConfigFileStream, confDir, bootstrapFileProvider.getBootstrapProperties()); - runner.getConfigFileReference().set(resetConfigFile.asReadOnlyBuffer()); - } - throw e; - } + private void setActiveFlowReference(ByteBuffer flowConfig) { + logger.debug("Setting active flow reference {} with content:\n{}", flowConfig, new String(flowConfig.array(), UTF_8)); + runner.getConfigFileReference().set(flowConfig); } private void restartInstance() throws IOException { diff --git a/minifi/minifi-bootstrap/src/main/java/org/apache/nifi/minifi/bootstrap/service/MiNiFiExecCommandProvider.java b/minifi/minifi-bootstrap/src/main/java/org/apache/nifi/minifi/bootstrap/service/MiNiFiExecCommandProvider.java index 1dd14686b1..8f2b0fce08 100644 --- a/minifi/minifi-bootstrap/src/main/java/org/apache/nifi/minifi/bootstrap/service/MiNiFiExecCommandProvider.java +++ b/minifi/minifi-bootstrap/src/main/java/org/apache/nifi/minifi/bootstrap/service/MiNiFiExecCommandProvider.java @@ -17,15 +17,19 @@ package org.apache.nifi.minifi.bootstrap.service; +import static java.lang.String.join; +import static java.util.Optional.ofNullable; +import static java.util.stream.Collectors.joining; import static org.apache.nifi.minifi.bootstrap.RunMiNiFi.CONF_DIR_KEY; +import static org.apache.nifi.util.NiFiProperties.PROPERTIES_FILE_PATH; import java.io.File; import java.io.IOException; -import java.util.ArrayList; +import java.nio.file.Path; import java.util.List; -import java.util.Map.Entry; import java.util.Optional; import java.util.Properties; +import java.util.stream.Stream; public class MiNiFiExecCommandProvider { @@ -40,12 +44,28 @@ public class MiNiFiExecCommandProvider { public static final String DEFAULT_BOOTSTRAP_LOG_FILE_NAME = "minifi-bootstrap"; public static final String DEFAULT_LOG_FILE_EXTENSION = "log"; + private static final String PROPERTIES_FILE_KEY = "props.file"; + private static final String LIB_DIR_KEY = "lib.dir"; + private static final String JAVA_COMMAND_KEY = "java"; + private static final String JAVA_ARG_KEY_PREFIX = "java.arg"; private static final String DEFAULT_JAVA_CMD = "java"; private static final String DEFAULT_LIB_DIR = "./lib"; private static final String DEFAULT_CONF_DIR = "./conf"; - private static final String DEFAULT_CONFIG_FILE = DEFAULT_CONF_DIR + "/bootstrap.conf"; + private static final String DEFAULT_MINIFI_PROPERTIES_FILE = "minifi.properties"; + private static final String WINDOWS_FILE_EXTENSION = ".exe"; + private static final String LINUX_FILE_EXTENSION = ""; + private static final String JAR_FILE_EXTENSION = ".jar"; + + private static final String JAVA_HOME_ENVIRONMENT_VARIABLE = "JAVA_HOME"; + private static final String MINIFI_CLASS_NAME = "MiNiFi"; + private static final String MINIFI_FULLY_QUALIFIED_CLASS_NAME = "org.apache.nifi.minifi." + MINIFI_CLASS_NAME; + private static final String SYSTEM_PROPERTY_TEMPLATE = "-D%s=%s"; + private static final String NIFI_BOOTSTRAP_LISTEN_PORT = "nifi.bootstrap.listen.port"; + private static final String APP = "app"; + private static final String CLASSPATH = "-classpath"; + private static final String BIN_DIRECTORY = "bin"; private final BootstrapFileProvider bootstrapFileProvider; @@ -53,6 +73,16 @@ public class MiNiFiExecCommandProvider { this.bootstrapFileProvider = bootstrapFileProvider; } + public static String getMiNiFiPropertiesPath(Properties props, File confDir) { + return ofNullable(props.getProperty(PROPERTIES_FILE_KEY)) + .orElseGet(() -> ofNullable(confDir) + .filter(File::exists) + .map(File::getAbsolutePath) + .map(parent -> Path.of(parent, DEFAULT_MINIFI_PROPERTIES_FILE).toAbsolutePath().toString()) + .orElseGet(() -> Path.of(DEFAULT_CONF_DIR, DEFAULT_MINIFI_PROPERTIES_FILE).toAbsolutePath().toString())) + .trim(); + } + /** * Returns the process arguments required for the bootstrap to start the MiNiFi process. * @@ -62,116 +92,82 @@ public class MiNiFiExecCommandProvider { * @throws IOException throws IOException if any of the configuration file read fails */ public List getMiNiFiExecCommand(int listenPort, File workingDir) throws IOException { - Properties props = bootstrapFileProvider.getBootstrapProperties(); - File confDir = getFile(props.getProperty(CONF_DIR_KEY, DEFAULT_CONF_DIR).trim(), workingDir); - File libDir = getFile(props.getProperty("lib.dir", DEFAULT_LIB_DIR).trim(), workingDir); + Properties bootstrapProperties = bootstrapFileProvider.getBootstrapProperties(); + + File confDir = getFile(bootstrapProperties.getProperty(CONF_DIR_KEY, DEFAULT_CONF_DIR).trim(), workingDir); + File libDir = getFile(bootstrapProperties.getProperty(LIB_DIR_KEY, DEFAULT_LIB_DIR).trim(), workingDir); + String minifiLogDir = System.getProperty(LOG_DIR, DEFAULT_LOG_DIR).trim(); String minifiAppLogFileName = System.getProperty(APP_LOG_FILE_NAME, DEFAULT_APP_LOG_FILE_NAME).trim(); String minifiAppLogFileExtension = System.getProperty(APP_LOG_FILE_EXTENSION, DEFAULT_LOG_FILE_EXTENSION).trim(); String minifiBootstrapLogFileName = System.getProperty(BOOTSTRAP_LOG_FILE_NAME, DEFAULT_BOOTSTRAP_LOG_FILE_NAME).trim(); String minifiBootstrapLogFileExtension = System.getProperty(BOOTSTRAP_LOG_FILE_EXTENSION, DEFAULT_LOG_FILE_EXTENSION).trim(); - List cmd = new ArrayList<>(); - cmd.add(getJavaCommand(props)); - cmd.add("-classpath"); - cmd.add(buildClassPath(props, confDir, libDir)); - cmd.addAll(getJavaAdditionalArgs(props)); - cmd.add("-Dnifi.properties.file.path=" + getMiNiFiPropsFileName(props, confDir)); - cmd.add("-Dnifi.bootstrap.listen.port=" + listenPort); - cmd.add("-Dapp=MiNiFi"); - cmd.add("-D" + LOG_DIR + "=" + minifiLogDir); - cmd.add("-D" + APP_LOG_FILE_NAME + "=" + minifiAppLogFileName); - cmd.add("-D" + APP_LOG_FILE_EXTENSION + "=" + minifiAppLogFileExtension); - cmd.add("-D" + BOOTSTRAP_LOG_FILE_NAME + "=" + minifiBootstrapLogFileName); - cmd.add("-D" + BOOTSTRAP_LOG_FILE_EXTENSION + "=" + minifiBootstrapLogFileExtension); - cmd.add("org.apache.nifi.minifi.MiNiFi"); + List javaCommand = List.of(getJavaCommand(bootstrapProperties), CLASSPATH, buildClassPath(confDir, libDir)); + List javaAdditionalArgs = getJavaAdditionalArgs(bootstrapProperties); + List systemProperties = List.of( + systemProperty(PROPERTIES_FILE_PATH, getMiNiFiPropertiesPath(bootstrapProperties, confDir)), + systemProperty(NIFI_BOOTSTRAP_LISTEN_PORT, listenPort), + systemProperty(APP, MINIFI_CLASS_NAME), + systemProperty(LOG_DIR, minifiLogDir), + systemProperty(APP_LOG_FILE_NAME, minifiAppLogFileName), + systemProperty(APP_LOG_FILE_EXTENSION, minifiAppLogFileExtension), + systemProperty(BOOTSTRAP_LOG_FILE_NAME, minifiBootstrapLogFileName), + systemProperty(BOOTSTRAP_LOG_FILE_EXTENSION, minifiBootstrapLogFileExtension) + ); - return cmd; - } - - private String getJavaCommand(Properties props) { - String javaCmd = props.getProperty("java"); - if (javaCmd == null) { - javaCmd = DEFAULT_JAVA_CMD; - } - if (javaCmd.equals(DEFAULT_JAVA_CMD)) { - Optional.ofNullable(System.getenv("JAVA_HOME")) - .map(javaHome -> getJavaCommandBasedOnExtension(javaHome, WINDOWS_FILE_EXTENSION) - .orElseGet(() -> getJavaCommandBasedOnExtension(javaHome, "").orElse(DEFAULT_JAVA_CMD))); - } - return javaCmd; - } - - private Optional getJavaCommandBasedOnExtension(String javaHome, String extension) { - String javaCmd = null; - File javaFile = new File(javaHome + File.separatorChar + "bin" + File.separatorChar + "java" + extension); - if (javaFile.exists() && javaFile.canExecute()) { - javaCmd = javaFile.getAbsolutePath(); - } - return Optional.ofNullable(javaCmd); - } - - private String buildClassPath(Properties props, File confDir, File libDir) { - - File[] libFiles = libDir.listFiles((dir, filename) -> filename.toLowerCase().endsWith(".jar")); - if (libFiles == null || libFiles.length == 0) { - throw new RuntimeException("Could not find lib directory at " + libDir.getAbsolutePath()); - } - - File[] confFiles = confDir.listFiles(); - if (confFiles == null || confFiles.length == 0) { - throw new RuntimeException("Could not find conf directory at " + confDir.getAbsolutePath()); - } - - List cpFiles = new ArrayList<>(confFiles.length + libFiles.length); - cpFiles.add(confDir.getAbsolutePath()); - for (File file : libFiles) { - cpFiles.add(file.getAbsolutePath()); - } - - StringBuilder classPathBuilder = new StringBuilder(); - for (int i = 0; i < cpFiles.size(); i++) { - String filename = cpFiles.get(i); - classPathBuilder.append(filename); - if (i < cpFiles.size() - 1) { - classPathBuilder.append(File.pathSeparatorChar); - } - } - - return classPathBuilder.toString(); - } - - private List getJavaAdditionalArgs(Properties props) { - List javaAdditionalArgs = new ArrayList<>(); - for (Entry entry : props.entrySet()) { - String key = (String) entry.getKey(); - String value = (String) entry.getValue(); - - if (key.startsWith("java.arg")) { - javaAdditionalArgs.add(value); - } - } - return javaAdditionalArgs; - } - - private String getMiNiFiPropsFileName(Properties props, File confDir) { - String minifiPropsFilename = props.getProperty("props.file"); - if (minifiPropsFilename == null) { - if (confDir.exists()) { - minifiPropsFilename = new File(confDir, "nifi.properties").getAbsolutePath(); - } else { - minifiPropsFilename = DEFAULT_CONFIG_FILE; - } - } - - return minifiPropsFilename.trim(); + return List.of(javaCommand, javaAdditionalArgs, systemProperties, List.of(MINIFI_FULLY_QUALIFIED_CLASS_NAME)) + .stream() + .flatMap(List::stream) + .toList(); } private File getFile(String filename, File workingDir) { File file = new File(filename); - if (!file.isAbsolute()) { - file = new File(workingDir, filename); - } - return file; + return file.isAbsolute() ? file : new File(workingDir, filename).getAbsoluteFile(); } -} + + private String getJavaCommand(Properties bootstrapProperties) { + String javaCommand = bootstrapProperties.getProperty(JAVA_COMMAND_KEY, DEFAULT_JAVA_CMD); + return javaCommand.equals(DEFAULT_JAVA_CMD) + ? ofNullable(System.getenv(JAVA_HOME_ENVIRONMENT_VARIABLE)) + .flatMap(javaHome -> + getJavaCommandBasedOnExtension(javaHome, javaCommand, WINDOWS_FILE_EXTENSION) + .or(() -> getJavaCommandBasedOnExtension(javaHome, javaCommand, LINUX_FILE_EXTENSION))) + .orElse(DEFAULT_JAVA_CMD) + : javaCommand; + } + + private Optional getJavaCommandBasedOnExtension(String javaHome, String javaCommand, String extension) { + return Optional.of(new File(join(File.separator, javaHome, BIN_DIRECTORY, javaCommand + extension))) + .filter(File::exists) + .filter(File::canExecute) + .map(File::getAbsolutePath); + } + + private String buildClassPath(File confDir, File libDir) { + File[] libFiles = ofNullable(libDir.listFiles((dir, filename) -> filename.toLowerCase().endsWith(JAR_FILE_EXTENSION))) + .filter(files -> files.length > 0) + .orElseThrow(() -> new RuntimeException("Could not find lib directory at " + libDir.getAbsolutePath())); + + ofNullable(confDir.listFiles()) + .filter(files -> files.length > 0) + .orElseThrow(() -> new RuntimeException("Could not find conf directory at " + confDir.getAbsolutePath())); + + return Stream.concat(Stream.of(confDir), Stream.of(libFiles)) + .map(File::getAbsolutePath) + .collect(joining(File.pathSeparator)); + } + + private List getJavaAdditionalArgs(Properties props) { + return props.entrySet() + .stream() + .filter(entry -> ((String) entry.getKey()).startsWith(JAVA_ARG_KEY_PREFIX)) + .map(entry -> (String) entry.getValue()) + .toList(); + } + + private String systemProperty(String key, Object value) { + return String.format(SYSTEM_PROPERTY_TEMPLATE, key, value); + } +} \ No newline at end of file diff --git a/minifi/minifi-bootstrap/src/main/java/org/apache/nifi/minifi/bootstrap/service/MiNiFiPropertiesGenerator.java b/minifi/minifi-bootstrap/src/main/java/org/apache/nifi/minifi/bootstrap/service/MiNiFiPropertiesGenerator.java new file mode 100644 index 0000000000..b8de952cb1 --- /dev/null +++ b/minifi/minifi-bootstrap/src/main/java/org/apache/nifi/minifi/bootstrap/service/MiNiFiPropertiesGenerator.java @@ -0,0 +1,276 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.nifi.minifi.bootstrap.service; + +import static java.lang.String.join; +import static java.lang.System.getProperty; +import static java.util.Optional.ofNullable; +import static org.apache.commons.lang3.StringUtils.EMPTY; +import static org.apache.commons.lang3.StringUtils.isNotBlank; +import static org.apache.nifi.minifi.bootstrap.service.BootstrapFileProvider.getBootstrapConfFile; +import static org.apache.nifi.minifi.bootstrap.service.MiNiFiExecCommandProvider.APP_LOG_FILE_EXTENSION; +import static org.apache.nifi.minifi.bootstrap.service.MiNiFiExecCommandProvider.APP_LOG_FILE_NAME; +import static org.apache.nifi.minifi.bootstrap.service.MiNiFiExecCommandProvider.BOOTSTRAP_LOG_FILE_EXTENSION; +import static org.apache.nifi.minifi.bootstrap.service.MiNiFiExecCommandProvider.BOOTSTRAP_LOG_FILE_NAME; +import static org.apache.nifi.minifi.bootstrap.service.MiNiFiExecCommandProvider.DEFAULT_APP_LOG_FILE_NAME; +import static org.apache.nifi.minifi.bootstrap.service.MiNiFiExecCommandProvider.DEFAULT_BOOTSTRAP_LOG_FILE_NAME; +import static org.apache.nifi.minifi.bootstrap.service.MiNiFiExecCommandProvider.DEFAULT_LOG_DIR; +import static org.apache.nifi.minifi.bootstrap.service.MiNiFiExecCommandProvider.DEFAULT_LOG_FILE_EXTENSION; +import static org.apache.nifi.minifi.bootstrap.service.MiNiFiExecCommandProvider.LOG_DIR; +import static org.apache.nifi.minifi.bootstrap.service.MiNiFiExecCommandProvider.getMiNiFiPropertiesPath; +import static org.apache.nifi.minifi.commons.api.MiNiFiProperties.MINIFI_APP_LOG_FILE; +import static org.apache.nifi.minifi.commons.api.MiNiFiProperties.MINIFI_BOOTSTRAP_FILE_PATH; +import static org.apache.nifi.minifi.commons.api.MiNiFiProperties.MINIFI_BOOTSTRAP_LOG_FILE; +import static org.apache.nifi.minifi.commons.api.MiNiFiProperties.MINIFI_LOG_DIRECTORY; + +import java.io.File; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.nio.file.Files; +import java.nio.file.Path; +import java.security.SecureRandom; +import java.util.Base64; +import java.util.Comparator; +import java.util.List; +import java.util.Map; +import java.util.Properties; +import java.util.Set; +import java.util.function.Predicate; +import org.apache.commons.io.output.ByteArrayOutputStream; +import org.apache.commons.lang3.StringUtils; +import org.apache.commons.lang3.tuple.Pair; +import org.apache.commons.lang3.tuple.Triple; +import org.apache.nifi.minifi.bootstrap.configuration.ConfigurationChangeException; +import org.apache.nifi.minifi.bootstrap.util.OrderedProperties; +import org.apache.nifi.minifi.commons.api.MiNiFiProperties; +import org.apache.nifi.util.NiFiProperties; + +public class MiNiFiPropertiesGenerator { + + public static final String PROPERTIES_FILE_APACHE_2_0_LICENSE = + " Licensed to the Apache Software Foundation (ASF) under one or more\n" + + "# contributor license agreements. See the NOTICE file distributed with\n" + + "# this work for additional information regarding copyright ownership.\n" + + "# The ASF licenses this file to You under the Apache License, Version 2.0\n" + + "# (the \"License\"); you may not use this file except in compliance with\n" + + "# the License. You may obtain a copy of the License at\n" + + "#\n" + + "# http://www.apache.org/licenses/LICENSE-2.0\n" + + "#\n" + + "# Unless required by applicable law or agreed to in writing, software\n" + + "# distributed under the License is distributed on an \"AS IS\" BASIS,\n" + + "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n" + + "# See the License for the specific language governing permissions and\n" + + "# limitations under the License.\n" + + "\n"; + + static final List> NIFI_PROPERTIES_WITH_DEFAULT_VALUES_AND_COMMENTS = List.of( + Triple.of(NiFiProperties.FLOW_CONFIGURATION_ARCHIVE_ENABLED, "false", EMPTY), + Triple.of(NiFiProperties.FLOW_CONFIGURATION_ARCHIVE_DIR, "./conf/archive/", EMPTY), + Triple.of(NiFiProperties.AUTO_RESUME_STATE, "true", EMPTY), + Triple.of(NiFiProperties.FLOW_CONTROLLER_GRACEFUL_SHUTDOWN_PERIOD, "10 sec", EMPTY), + Triple.of(NiFiProperties.WRITE_DELAY_INTERVAL, "500 ms", EMPTY), + Triple.of(NiFiProperties.ADMINISTRATIVE_YIELD_DURATION, "30 sec", EMPTY), + Triple.of(NiFiProperties.VARIABLE_REGISTRY_PROPERTIES, EMPTY, EMPTY), + Triple.of(NiFiProperties.BORED_YIELD_DURATION, "10 millis", + "# If a component has no work to do (is \"bored\"), how long should we wait before checking again for work"), + Triple.of(NiFiProperties.LOGIN_IDENTITY_PROVIDER_CONFIGURATION_FILE, "./conf/login-identity-providers.xml", EMPTY), + Triple.of(NiFiProperties.TEMPLATE_DIRECTORY, "./conf/templates", EMPTY), + Triple.of(NiFiProperties.UI_BANNER_TEXT, EMPTY, EMPTY), + Triple.of(NiFiProperties.UI_AUTO_REFRESH_INTERVAL, "30 sec", EMPTY), + Triple.of(NiFiProperties.NAR_LIBRARY_DIRECTORY, "./lib", EMPTY), + Triple.of(NiFiProperties.NAR_WORKING_DIRECTORY, "./work/nar/", EMPTY), + Triple.of(NiFiProperties.NAR_LIBRARY_AUTOLOAD_DIRECTORY, "./extensions", EMPTY), + Triple.of(NiFiProperties.COMPONENT_DOCS_DIRECTORY, "./work/docs/components", EMPTY), + Triple.of(NiFiProperties.STATE_MANAGEMENT_CONFIG_FILE, "./conf/state-management.xml", "# State Management"), + Triple.of(NiFiProperties.STATE_MANAGEMENT_LOCAL_PROVIDER_ID, "local-provider", "# The ID of the local state provider"), + Triple.of(NiFiProperties.REPOSITORY_DATABASE_DIRECTORY, "./database_repository", "# H2 Settings"), + Triple.of(NiFiProperties.H2_URL_APPEND, ";LOCK_TIMEOUT=25000;WRITE_DELAY=0;AUTO_SERVER=FALSE", EMPTY), + Triple.of(NiFiProperties.FLOWFILE_REPOSITORY_IMPLEMENTATION, "org.apache.nifi.controller.repository.WriteAheadFlowFileRepository", "# FlowFile Repository"), + Triple.of(NiFiProperties.FLOWFILE_REPOSITORY_DIRECTORY, "./flowfile_repository", EMPTY), + Triple.of(NiFiProperties.FLOWFILE_REPOSITORY_CHECKPOINT_INTERVAL, "20 secs", EMPTY), + Triple.of(NiFiProperties.FLOWFILE_REPOSITORY_ALWAYS_SYNC, "false", EMPTY), + Triple.of(NiFiProperties.FLOWFILE_SWAP_MANAGER_IMPLEMENTATION, "org.apache.nifi.controller.FileSystemSwapManager", EMPTY), + Triple.of(NiFiProperties.QUEUE_SWAP_THRESHOLD, "20000", EMPTY), + Triple.of(NiFiProperties.CONTENT_REPOSITORY_IMPLEMENTATION, "org.apache.nifi.controller.repository.FileSystemRepository", "# Content Repository"), + Triple.of(NiFiProperties.MAX_APPENDABLE_CLAIM_SIZE, "50 KB", EMPTY), + Triple.of(NiFiProperties.CONTENT_ARCHIVE_MAX_RETENTION_PERIOD, "7 days", EMPTY), + Triple.of(NiFiProperties.CONTENT_ARCHIVE_MAX_USAGE_PERCENTAGE, "50%", EMPTY), + Triple.of(NiFiProperties.CONTENT_ARCHIVE_ENABLED, "false", EMPTY), + Triple.of(NiFiProperties.REPOSITORY_CONTENT_PREFIX + ".default", "./content_repository", EMPTY), + Triple.of(NiFiProperties.PROVENANCE_REPO_IMPLEMENTATION_CLASS, "org.apache.nifi.provenance.NoOpProvenanceRepository", + "# Provenance Repository Properties"), + Triple.of(NiFiProperties.PROVENANCE_ROLLOVER_TIME, EMPTY, EMPTY), + Triple.of(NiFiProperties.PROVENANCE_INDEX_SHARD_SIZE, EMPTY, EMPTY), + Triple.of(NiFiProperties.PROVENANCE_MAX_STORAGE_SIZE, EMPTY, EMPTY), + Triple.of(NiFiProperties.PROVENANCE_MAX_STORAGE_TIME, EMPTY, EMPTY), + Triple.of(NiFiProperties.COMPONENT_STATUS_REPOSITORY_IMPLEMENTATION, "org.apache.nifi.controller.status.history.VolatileComponentStatusRepository", + "# Component Status Repository"), + Triple.of(NiFiProperties.COMPONENT_STATUS_SNAPSHOT_FREQUENCY, "1 min", EMPTY), + Triple.of(NiFiProperties.WEB_HTTP_HOST, EMPTY, EMPTY), + Triple.of(NiFiProperties.WEB_HTTP_PORT, EMPTY, EMPTY), + Triple.of(NiFiProperties.WEB_HTTPS_HOST, EMPTY, EMPTY), + Triple.of(NiFiProperties.WEB_HTTPS_PORT, EMPTY, EMPTY), + Triple.of(NiFiProperties.WEB_WORKING_DIR, "./work/jetty", EMPTY), + Triple.of(NiFiProperties.WEB_THREADS, "200", EMPTY), + Triple.of(NiFiProperties.SECURITY_KEYSTORE, EMPTY, EMPTY), + Triple.of(NiFiProperties.SECURITY_KEYSTORE_TYPE, EMPTY, EMPTY), + Triple.of(NiFiProperties.SECURITY_KEYSTORE_PASSWD, EMPTY, EMPTY), + Triple.of(NiFiProperties.SECURITY_KEY_PASSWD, EMPTY, EMPTY), + Triple.of(NiFiProperties.SECURITY_TRUSTSTORE, EMPTY, EMPTY), + Triple.of(NiFiProperties.SECURITY_TRUSTSTORE_TYPE, EMPTY, EMPTY), + Triple.of(NiFiProperties.SECURITY_TRUSTSTORE_PASSWD, EMPTY, EMPTY), + Triple.of(NiFiProperties.SECURITY_OCSP_RESPONDER_URL, EMPTY, EMPTY), + Triple.of(NiFiProperties.SECURITY_OCSP_RESPONDER_CERTIFICATE, EMPTY, EMPTY), + Triple.of(NiFiProperties.CLUSTER_IS_NODE, "false", EMPTY), + Triple.of(NiFiProperties.FLOW_CONFIGURATION_FILE, "./conf/flow.xml.gz", EMPTY) + ); + + static final Map MINIFI_TO_NIFI_PROPERTY_MAPPING = Map.of( + MiNiFiProperties.NIFI_MINIFI_FLOW_CONFIG.getKey(), NiFiProperties.FLOW_CONFIGURATION_JSON_FILE, + MiNiFiProperties.NIFI_MINIFI_SECURITY_KEYSTORE.getKey(), NiFiProperties.SECURITY_KEYSTORE, + MiNiFiProperties.NIFI_MINIFI_SECURITY_KEYSTORE_TYPE.getKey(), NiFiProperties.SECURITY_KEYSTORE_TYPE, + MiNiFiProperties.NIFI_MINIFI_SECURITY_KEYSTORE_PASSWD.getKey(), NiFiProperties.SECURITY_KEYSTORE_PASSWD, + MiNiFiProperties.NIFI_MINIFI_SECURITY_KEY_PASSWD.getKey(), NiFiProperties.SECURITY_KEY_PASSWD, + MiNiFiProperties.NIFI_MINIFI_SECURITY_TRUSTSTORE.getKey(), NiFiProperties.SECURITY_TRUSTSTORE, + MiNiFiProperties.NIFI_MINIFI_SECURITY_TRUSTSTORE_TYPE.getKey(), NiFiProperties.SECURITY_TRUSTSTORE_TYPE, + MiNiFiProperties.NIFI_MINIFI_SECURITY_TRUSTSTORE_PASSWD.getKey(), NiFiProperties.SECURITY_TRUSTSTORE_PASSWD + ); + + static final String DEFAULT_SENSITIVE_PROPERTIES_ENCODING_ALGORITHM = "NIFI_PBKDF2_AES_GCM_256"; + + private static final Base64.Encoder KEY_ENCODER = Base64.getEncoder().withoutPadding(); + private static final int SENSITIVE_PROPERTIES_KEY_LENGTH = 24; + + private static final String C2_PROPERTY_PREFIX = "c2."; + private static final String NIFI_PREFIX = "nifi."; + + public static final String FILE_EXTENSION_DELIMITER = "."; + + public void generateMinifiProperties(String configDirectory, Properties bootstrapProperties) throws ConfigurationChangeException { + String minifiPropertiesFileName = Path.of(getMiNiFiPropertiesPath(bootstrapProperties, new File(configDirectory))).getFileName().toString(); + Path minifiPropertiesFile = Path.of(configDirectory, minifiPropertiesFileName); + + Map existingSensitivePropertiesConfiguration = extractSensitivePropertiesConfiguration(minifiPropertiesFile); + OrderedProperties minifiProperties = prepareMinifiProperties(bootstrapProperties, existingSensitivePropertiesConfiguration); + + persistMinifiProperties(minifiPropertiesFile, minifiProperties); + } + + private Map extractSensitivePropertiesConfiguration(Path minifiPropertiesFile) throws ConfigurationChangeException { + if (!Files.exists(minifiPropertiesFile)) { + return Map.of(); + } + + Properties minifiProperties = new Properties(); + try (InputStream inputStream = Files.newInputStream(minifiPropertiesFile)) { + minifiProperties.load(inputStream); + } catch (IOException e) { + throw new ConfigurationChangeException("Unable to load MiNiFi properties from " + minifiPropertiesFile, e); + } + + return Map.of( + NiFiProperties.SENSITIVE_PROPS_KEY, minifiProperties.getProperty(NiFiProperties.SENSITIVE_PROPS_KEY), + NiFiProperties.SENSITIVE_PROPS_ALGORITHM, minifiProperties.getProperty(NiFiProperties.SENSITIVE_PROPS_ALGORITHM) + ); + } + + private OrderedProperties prepareMinifiProperties(Properties bootstrapProperties, Map existingSensitivePropertiesConfiguration) { + OrderedProperties minifiProperties = new OrderedProperties(); + + NIFI_PROPERTIES_WITH_DEFAULT_VALUES_AND_COMMENTS + .forEach(triple -> minifiProperties.setProperty(triple.getLeft(), triple.getMiddle(), triple.getRight())); + + getNonBlankPropertiesWithPredicate(bootstrapProperties, entry -> MINIFI_TO_NIFI_PROPERTY_MAPPING.containsKey(entry.getKey())) + .forEach(entry -> minifiProperties.setProperty(MINIFI_TO_NIFI_PROPERTY_MAPPING.get(entry.getKey()), entry.getValue())); + + getSensitiveProperties(bootstrapProperties, existingSensitivePropertiesConfiguration) + .forEach(entry -> minifiProperties.setProperty(entry.getKey(), entry.getValue())); + + getNonBlankPropertiesWithPredicate(bootstrapProperties, entry -> ((String) entry.getKey()).startsWith(C2_PROPERTY_PREFIX)) + .forEach(entry -> minifiProperties.setProperty(entry.getKey(), entry.getValue())); + + getNonBlankPropertiesWithPredicate(bootstrapProperties, entry -> ((String) entry.getKey()).startsWith(NIFI_PREFIX)) + .forEach(entry -> minifiProperties.setProperty(entry.getKey(), entry.getValue())); + + bootstrapFileAndLogProperties() + .forEach(entry -> minifiProperties.setProperty(entry.getKey(), entry.getValue())); + + return minifiProperties; + } + + private List> getNonBlankPropertiesWithPredicate(Properties bootstrapProperties, Predicate predicate) { + return ofNullable(bootstrapProperties) + .map(Properties::entrySet) + .orElseGet(Set::of) + .stream() + .filter(predicate) + .filter(entry -> isNotBlank((String) entry.getValue())) + .map(entry -> Pair.of((String) entry.getKey(), (String) entry.getValue())) + .sorted((o1, o2) -> Comparator.naturalOrder().compare(o1.getKey(), o2.getKey())) + .toList(); + } + + private List> getSensitiveProperties(Properties bootstrapProperties, Map existingSensitivePropertiesConfiguration) { + return existingSensitivePropertiesConfiguration.isEmpty() + ? List.of( + Pair.of(NiFiProperties.SENSITIVE_PROPS_KEY, + ofNullable(bootstrapProperties.getProperty(MiNiFiProperties.NIFI_MINIFI_SENSITIVE_PROPS_KEY.getKey())) + .filter(StringUtils::isNotBlank) + .orElseGet(this::generateSensitivePropertiesKey)), + Pair.of(NiFiProperties.SENSITIVE_PROPS_ALGORITHM, + ofNullable(bootstrapProperties.getProperty(MiNiFiProperties.NIFI_MINIFI_SENSITIVE_PROPS_ALGORITHM.getKey())) + .filter(StringUtils::isNotBlank) + .orElse(DEFAULT_SENSITIVE_PROPERTIES_ENCODING_ALGORITHM))) + : existingSensitivePropertiesConfiguration.entrySet() + .stream() + .map(entry -> Pair.of(entry.getKey(), entry.getValue())) + .toList(); + } + + private String generateSensitivePropertiesKey() { + SecureRandom secureRandom = new SecureRandom(); + byte[] sensitivePropertiesKeyBinary = new byte[SENSITIVE_PROPERTIES_KEY_LENGTH]; + secureRandom.nextBytes(sensitivePropertiesKeyBinary); + return KEY_ENCODER.encodeToString(sensitivePropertiesKeyBinary); + } + + private List> bootstrapFileAndLogProperties() { + return List.of( + Pair.of(MINIFI_BOOTSTRAP_FILE_PATH, getBootstrapConfFile().getAbsolutePath()), + Pair.of(MINIFI_LOG_DIRECTORY, getProperty(LOG_DIR, DEFAULT_LOG_DIR).trim()), + Pair.of(MINIFI_APP_LOG_FILE, join(FILE_EXTENSION_DELIMITER, + getProperty(APP_LOG_FILE_NAME, DEFAULT_APP_LOG_FILE_NAME).trim(), + getProperty(APP_LOG_FILE_EXTENSION, DEFAULT_LOG_FILE_EXTENSION).trim())), + Pair.of(MINIFI_BOOTSTRAP_LOG_FILE, join(FILE_EXTENSION_DELIMITER, + getProperty(BOOTSTRAP_LOG_FILE_NAME, DEFAULT_BOOTSTRAP_LOG_FILE_NAME).trim(), + getProperty(BOOTSTRAP_LOG_FILE_EXTENSION, DEFAULT_LOG_FILE_EXTENSION).trim())) + ); + } + + private void persistMinifiProperties(Path minifiPropertiesFile, OrderedProperties minifiProperties) throws ConfigurationChangeException { + try (ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream(); + FileOutputStream fileOutputStream = new FileOutputStream(minifiPropertiesFile.toString())) { + minifiProperties.store(byteArrayOutputStream, PROPERTIES_FILE_APACHE_2_0_LICENSE); + byteArrayOutputStream.writeTo(fileOutputStream); + } catch (Exception e) { + throw new ConfigurationChangeException("Failed to write MiNiFi properties to " + minifiPropertiesFile, e); + } + } +} diff --git a/minifi/minifi-bootstrap/src/main/java/org/apache/nifi/minifi/bootstrap/service/PeriodicStatusReporterManager.java b/minifi/minifi-bootstrap/src/main/java/org/apache/nifi/minifi/bootstrap/service/PeriodicStatusReporterManager.java index 9086ca8045..7ad382b8ea 100644 --- a/minifi/minifi-bootstrap/src/main/java/org/apache/nifi/minifi/bootstrap/service/PeriodicStatusReporterManager.java +++ b/minifi/minifi-bootstrap/src/main/java/org/apache/nifi/minifi/bootstrap/service/PeriodicStatusReporterManager.java @@ -14,9 +14,10 @@ * See the License for the specific language governing permissions and * limitations under the License. */ + package org.apache.nifi.minifi.bootstrap.service; -import static org.apache.nifi.minifi.commons.schema.common.BootstrapPropertyKeys.STATUS_REPORTER_COMPONENTS_KEY; +import static org.apache.nifi.minifi.commons.api.MiNiFiProperties.NIFI_MINIFI_STATUS_REPORTER_COMPONENTS; import java.util.Collections; import java.util.HashSet; @@ -44,7 +45,7 @@ public class PeriodicStatusReporterManager implements QueryableStatusAggregator private Set periodicStatusReporters = Collections.emptySet(); public PeriodicStatusReporterManager(Properties bootstrapProperties, MiNiFiStatusProvider miNiFiStatusProvider, MiNiFiCommandSender miNiFiCommandSender, - MiNiFiParameters miNiFiParameters) { + MiNiFiParameters miNiFiParameters) { this.bootstrapProperties = bootstrapProperties; this.miNiFiStatusProvider = miNiFiStatusProvider; this.miNiFiCommandSender = miNiFiCommandSender; @@ -54,7 +55,7 @@ public class PeriodicStatusReporterManager implements QueryableStatusAggregator public void startPeriodicNotifiers() { periodicStatusReporters = initializePeriodicNotifiers(); - for (PeriodicStatusReporter periodicStatusReporter: periodicStatusReporters) { + for (PeriodicStatusReporter periodicStatusReporter : periodicStatusReporters) { periodicStatusReporter.start(); LOGGER.debug("Started {} notifier", periodicStatusReporter.getClass().getCanonicalName()); } @@ -96,7 +97,7 @@ public class PeriodicStatusReporterManager implements QueryableStatusAggregator LOGGER.debug("Initiating bootstrap periodic status reporters..."); Set statusReporters = new HashSet<>(); - String reportersCsv = bootstrapProperties.getProperty(STATUS_REPORTER_COMPONENTS_KEY); + String reportersCsv = bootstrapProperties.getProperty(NIFI_MINIFI_STATUS_REPORTER_COMPONENTS.getKey()); if (reportersCsv != null && !reportersCsv.isEmpty()) { for (String reporterClassname : reportersCsv.split(",")) { diff --git a/minifi/minifi-bootstrap/src/main/java/org/apache/nifi/minifi/bootstrap/service/UpdateConfigurationService.java b/minifi/minifi-bootstrap/src/main/java/org/apache/nifi/minifi/bootstrap/service/UpdateConfigurationService.java deleted file mode 100644 index a2fc043a8f..0000000000 --- a/minifi/minifi-bootstrap/src/main/java/org/apache/nifi/minifi/bootstrap/service/UpdateConfigurationService.java +++ /dev/null @@ -1,95 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.nifi.minifi.bootstrap.service; - -import static java.util.Optional.ofNullable; -import static org.apache.nifi.minifi.commons.api.MiNiFiConstants.CONFIG_UPDATED_FILE_NAME; - -import java.io.File; -import java.io.FileInputStream; -import java.io.IOException; -import java.nio.ByteBuffer; -import java.nio.file.Path; -import java.util.Optional; -import org.apache.commons.io.IOUtils; -import org.apache.nifi.minifi.bootstrap.RunMiNiFi; -import org.apache.nifi.minifi.bootstrap.configuration.ConfigurationChangeListener; -import org.apache.nifi.minifi.bootstrap.configuration.differentiators.Differentiator; -import org.apache.nifi.minifi.bootstrap.configuration.differentiators.WholeConfigDifferentiator; -import org.apache.nifi.minifi.bootstrap.util.ByteBufferInputStream; -import org.apache.nifi.minifi.bootstrap.util.ConfigTransformer; -import org.apache.nifi.minifi.commons.api.MiNiFiCommandState; -import org.apache.nifi.util.NiFiProperties; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -public class UpdateConfigurationService { - - private static final Logger logger = LoggerFactory.getLogger(UpdateConfigurationService.class); - private static final String FALLBACK_CONFIG_FILE_DIR = "./conf/"; - - private final Differentiator differentiator; - private final RunMiNiFi runMiNiFi; - private final ConfigurationChangeListener miNiFiConfigurationChangeListener; - private final BootstrapFileProvider bootstrapFileProvider; - - public UpdateConfigurationService(RunMiNiFi runMiNiFi, ConfigurationChangeListener miNiFiConfigurationChangeListener, BootstrapFileProvider bootstrapFileProvider) { - this.differentiator = WholeConfigDifferentiator.getByteBufferDifferentiator(); - this.differentiator.initialize(runMiNiFi); - this.runMiNiFi = runMiNiFi; - this.miNiFiConfigurationChangeListener = miNiFiConfigurationChangeListener; - this.bootstrapFileProvider = bootstrapFileProvider; - } - - public Optional handleUpdate() { - logger.info("Handling configuration update"); - MiNiFiCommandState commandState = null; - try (FileInputStream configFile = new FileInputStream(getConfigFilePath().toFile())) { - ByteBuffer readOnlyNewConfig = ConfigTransformer.overrideNonFlowSectionsFromOriginalSchema( - IOUtils.toByteArray(configFile), runMiNiFi.getConfigFileReference().get().duplicate(), bootstrapFileProvider.getBootstrapProperties()); - if (differentiator.isNew(readOnlyNewConfig)) { - miNiFiConfigurationChangeListener.handleChange(new ByteBufferInputStream(readOnlyNewConfig.duplicate())); - } else { - logger.info("The given configuration does not contain any update. No operation required"); - commandState = MiNiFiCommandState.NO_OPERATION; - } - } catch (Exception e) { - commandState = MiNiFiCommandState.NOT_APPLIED_WITHOUT_RESTART; - logger.error("Could not handle configuration update", e); - } - return Optional.ofNullable(commandState); - } - - private Path getConfigFilePath() { - return ofNullable(safeGetPropertiesFilePath()) - .map(File::new) - .map(File::getParent) - .map(parentDir -> new File(parentDir + CONFIG_UPDATED_FILE_NAME)) - .orElse(new File(FALLBACK_CONFIG_FILE_DIR + CONFIG_UPDATED_FILE_NAME)).toPath(); - } - - private String safeGetPropertiesFilePath() { - String propertyFile = null; - try { - propertyFile = bootstrapFileProvider.getBootstrapProperties().getProperty(NiFiProperties.PROPERTIES_FILE_PATH, null); - } catch (IOException e) { - logger.error("Failed to get properties file path"); - } - return propertyFile; - } -} diff --git a/minifi/minifi-bootstrap/src/main/java/org/apache/nifi/minifi/bootstrap/service/UpdatePropertiesService.java b/minifi/minifi-bootstrap/src/main/java/org/apache/nifi/minifi/bootstrap/service/UpdatePropertiesService.java index 4b3de7e89a..36e68f7124 100644 --- a/minifi/minifi-bootstrap/src/main/java/org/apache/nifi/minifi/bootstrap/service/UpdatePropertiesService.java +++ b/minifi/minifi-bootstrap/src/main/java/org/apache/nifi/minifi/bootstrap/service/UpdatePropertiesService.java @@ -19,13 +19,10 @@ package org.apache.nifi.minifi.bootstrap.service; import static java.nio.file.StandardCopyOption.REPLACE_EXISTING; import static org.apache.nifi.minifi.bootstrap.RunMiNiFi.CONF_DIR_KEY; -import static org.apache.nifi.minifi.bootstrap.util.ConfigTransformer.asByteArrayInputStream; -import static org.apache.nifi.minifi.bootstrap.util.ConfigTransformer.generateConfigFiles; import java.io.File; import java.io.FileInputStream; import java.io.IOException; -import java.nio.ByteBuffer; import java.nio.file.Files; import java.util.Optional; import java.util.Properties; @@ -38,11 +35,13 @@ public class UpdatePropertiesService { private final RunMiNiFi runner; private final Logger logger; private final BootstrapFileProvider bootstrapFileProvider; + private final MiNiFiPropertiesGenerator miNiFiPropertiesGenerator; public UpdatePropertiesService(RunMiNiFi runner, Logger logger, BootstrapFileProvider bootstrapFileProvider) { this.runner = runner; this.logger = logger; this.bootstrapFileProvider = bootstrapFileProvider; + this.miNiFiPropertiesGenerator = new MiNiFiPropertiesGenerator(); } public Optional handleUpdate() { @@ -72,9 +71,7 @@ public class UpdatePropertiesService { throws IOException, ConfigurationChangeException { Optional commandState = Optional.empty(); try { - ByteBuffer byteBuffer = generateConfigFiles(asByteArrayInputStream(runner.getConfigFileReference().get().duplicate()), - bootstrapProperties.getProperty(CONF_DIR_KEY), bootstrapProperties); - runner.getConfigFileReference().set(byteBuffer.asReadOnlyBuffer()); + miNiFiPropertiesGenerator.generateMinifiProperties(bootstrapProperties.getProperty(CONF_DIR_KEY), bootstrapProperties); restartInstance(); } catch (Exception e) { commandState = Optional.of(MiNiFiCommandState.NOT_APPLIED_WITHOUT_RESTART); @@ -84,10 +81,7 @@ public class UpdatePropertiesService { } // read reverted properties bootstrapProperties = bootstrapFileProvider.getBootstrapProperties(); - - ByteBuffer byteBuffer = generateConfigFiles( - asByteArrayInputStream(runner.getConfigFileReference().get().duplicate()), bootstrapProperties.getProperty(CONF_DIR_KEY), bootstrapProperties); - runner.getConfigFileReference().set(byteBuffer.asReadOnlyBuffer()); + miNiFiPropertiesGenerator.generateMinifiProperties(bootstrapProperties.getProperty(CONF_DIR_KEY), bootstrapProperties); logger.debug("Transformation of new config file failed after swap file was created, deleting it."); if (!bootstrapSwapConfigFile.delete()) { diff --git a/minifi/minifi-bootstrap/src/main/java/org/apache/nifi/minifi/bootstrap/status/reporters/StatusLogger.java b/minifi/minifi-bootstrap/src/main/java/org/apache/nifi/minifi/bootstrap/status/reporters/StatusLogger.java index 5de8754a1b..2d6f7fa5bb 100644 --- a/minifi/minifi-bootstrap/src/main/java/org/apache/nifi/minifi/bootstrap/status/reporters/StatusLogger.java +++ b/minifi/minifi-bootstrap/src/main/java/org/apache/nifi/minifi/bootstrap/status/reporters/StatusLogger.java @@ -17,6 +17,12 @@ package org.apache.nifi.minifi.bootstrap.status.reporters; +import static org.apache.nifi.minifi.commons.api.MiNiFiProperties.NIFI_MINIFI_STATUS_REPORTER_LOG_LEVEL; +import static org.apache.nifi.minifi.commons.api.MiNiFiProperties.NIFI_MINIFI_STATUS_REPORTER_LOG_PERIOD; +import static org.apache.nifi.minifi.commons.api.MiNiFiProperties.NIFI_MINIFI_STATUS_REPORTER_LOG_QUERY; + +import java.io.IOException; +import java.util.Properties; import org.apache.nifi.logging.LogLevel; import org.apache.nifi.minifi.bootstrap.QueryableStatusAggregator; import org.apache.nifi.minifi.bootstrap.status.PeriodicStatusReporter; @@ -24,61 +30,49 @@ import org.apache.nifi.minifi.commons.status.FlowStatusReport; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.io.IOException; -import java.util.Properties; - -import static org.apache.nifi.minifi.commons.schema.common.BootstrapPropertyKeys.STATUS_REPORTER_PROPERTY_PREFIX; - public class StatusLogger extends PeriodicStatusReporter { - private volatile QueryableStatusAggregator queryableStatusAggregator; private volatile LogLevel logLevel; private volatile String statusQuery; protected static Logger logger = LoggerFactory.getLogger(StatusLogger.class); - - public static final String LOGGER_STATUS_REPORTER_PROPERTY_PREFIX = STATUS_REPORTER_PROPERTY_PREFIX + ".log"; - public static final String REPORT_PERIOD_KEY = LOGGER_STATUS_REPORTER_PROPERTY_PREFIX + ".period"; - public static final String LOGGING_LEVEL_KEY = LOGGER_STATUS_REPORTER_PROPERTY_PREFIX + ".level"; - public static final String QUERY_KEY = LOGGER_STATUS_REPORTER_PROPERTY_PREFIX + ".query"; - static final String ENCOUNTERED_IO_EXCEPTION = "Encountered an IO Exception while attempting to query the flow status."; @Override - public void initialize(Properties properties, QueryableStatusAggregator queryableStatusAggregator) { + public void initialize(Properties properties, QueryableStatusAggregator queryableStatusAggregator) { this.queryableStatusAggregator = queryableStatusAggregator; - String periodString = properties.getProperty(REPORT_PERIOD_KEY); + String periodString = properties.getProperty(NIFI_MINIFI_STATUS_REPORTER_LOG_PERIOD.getKey()); if (periodString == null) { - throw new IllegalStateException(REPORT_PERIOD_KEY + " is null but it is required. Please configure it."); + throw new IllegalStateException(NIFI_MINIFI_STATUS_REPORTER_LOG_PERIOD.getKey() + " is null but it is required. Please configure it."); } try { setPeriod(Integer.parseInt(periodString)); } catch (NumberFormatException e) { - throw new IllegalStateException(REPORT_PERIOD_KEY + " is not a valid number.", e); + throw new IllegalStateException(NIFI_MINIFI_STATUS_REPORTER_LOG_PERIOD.getKey() + " is not a valid number.", e); } - String loglevelString = properties.getProperty(LOGGING_LEVEL_KEY); + String loglevelString = properties.getProperty(NIFI_MINIFI_STATUS_REPORTER_LOG_LEVEL.getKey()); if (loglevelString == null) { - throw new IllegalStateException(LOGGING_LEVEL_KEY + " is null but it is required. Please configure it."); + throw new IllegalStateException(NIFI_MINIFI_STATUS_REPORTER_LOG_LEVEL.getKey() + " is null but it is required. Please configure it."); } try { logLevel = LogLevel.valueOf(loglevelString.toUpperCase()); } catch (IllegalArgumentException e) { - throw new IllegalStateException("Value set for " + LOGGING_LEVEL_KEY + " is not a valid log level."); + throw new IllegalStateException("Value set for " + NIFI_MINIFI_STATUS_REPORTER_LOG_LEVEL.getKey() + " is not a valid log level."); } - if (LogLevel.FATAL.equals(logLevel)){ - throw new IllegalStateException("Cannot log status at the FATAL level. Please configure " + LOGGING_LEVEL_KEY + " to another value."); + if (LogLevel.FATAL.equals(logLevel)) { + throw new IllegalStateException("Cannot log status at the FATAL level. Please configure " + NIFI_MINIFI_STATUS_REPORTER_LOG_LEVEL.getKey() + " to another value."); } - statusQuery = properties.getProperty(QUERY_KEY); + statusQuery = properties.getProperty(NIFI_MINIFI_STATUS_REPORTER_LOG_QUERY.getKey()); if (statusQuery == null) { - throw new IllegalStateException(QUERY_KEY + " is null but it is required. Please configure it."); + throw new IllegalStateException(NIFI_MINIFI_STATUS_REPORTER_LOG_QUERY.getKey() + " is null but it is required. Please configure it."); } reportRunner = new ReportStatusRunner(); diff --git a/minifi/minifi-bootstrap/src/main/java/org/apache/nifi/minifi/bootstrap/util/BootstrapTransformer.java b/minifi/minifi-bootstrap/src/main/java/org/apache/nifi/minifi/bootstrap/util/BootstrapTransformer.java deleted file mode 100644 index eb861b77de..0000000000 --- a/minifi/minifi-bootstrap/src/main/java/org/apache/nifi/minifi/bootstrap/util/BootstrapTransformer.java +++ /dev/null @@ -1,102 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.nifi.minifi.bootstrap.util; - -import org.apache.commons.lang3.StringUtils; -import org.apache.nifi.minifi.commons.schema.ProvenanceReportingSchema; -import org.apache.nifi.minifi.commons.schema.SecurityPropertiesSchema; -import org.apache.nifi.minifi.commons.schema.common.CommonPropertyKeys; - -import java.util.HashMap; -import java.util.Map; -import java.util.Optional; -import java.util.Properties; - -import static org.apache.nifi.minifi.commons.schema.common.BootstrapPropertyKeys.BOOTSTRAP_KEYS_TO_YML_KEYS; -import static org.apache.nifi.minifi.commons.schema.common.BootstrapPropertyKeys.BOOTSTRAP_PROVENANCE_REPORTING_KEYS; -import static org.apache.nifi.minifi.commons.schema.common.BootstrapPropertyKeys.BOOTSTRAP_SECURITY_PROPERTY_KEYS; -import static org.apache.nifi.minifi.commons.schema.common.BootstrapPropertyKeys.BOOTSTRAP_SENSITIVE_PROPERTY_KEYS; -import static org.apache.nifi.minifi.commons.schema.common.BootstrapPropertyKeys.USE_PARENT_SSL; - -public class BootstrapTransformer { - - public static Optional buildSecurityPropertiesFromBootstrap(final Properties bootstrapProperties) { - - Optional securityPropsOptional = Optional.empty(); - - final Map securityProperties = new HashMap<>(); - - if (bootstrapProperties != null) { - BOOTSTRAP_SECURITY_PROPERTY_KEYS.stream() - .filter(key -> StringUtils.isNotBlank(bootstrapProperties.getProperty(key))) - .forEach(key -> - securityProperties.put(BOOTSTRAP_KEYS_TO_YML_KEYS.get(key), bootstrapProperties.getProperty(key)) - ); - - if (!securityProperties.isEmpty()) { - // Determine if sensitive properties were provided - final Map sensitiveProperties = new HashMap<>(); - BOOTSTRAP_SENSITIVE_PROPERTY_KEYS.stream() - .filter(key -> StringUtils.isNotBlank(bootstrapProperties.getProperty(key))) - .forEach(key -> - sensitiveProperties.put(BOOTSTRAP_KEYS_TO_YML_KEYS.get(key), bootstrapProperties.getProperty(key)) - ); - if (!sensitiveProperties.isEmpty()) { - securityProperties.put(CommonPropertyKeys.SENSITIVE_PROPS_KEY, sensitiveProperties); - } - - final SecurityPropertiesSchema securityPropertiesSchema = new SecurityPropertiesSchema(securityProperties); - securityPropsOptional = Optional.of(securityPropertiesSchema); - } - } - - return securityPropsOptional; - } - - public static Optional buildProvenanceReportingPropertiesFromBootstrap(final Properties bootstrapProperties) { - - Optional provenanceReportingPropsOptional = Optional.empty(); - - final Map provenanceReportingProperties = new HashMap<>(); - if (bootstrapProperties != null) { - BOOTSTRAP_PROVENANCE_REPORTING_KEYS.stream() - .filter(key -> StringUtils.isNotBlank(bootstrapProperties.getProperty(key))) - .forEach(key -> - provenanceReportingProperties.put(BOOTSTRAP_KEYS_TO_YML_KEYS.get(key), bootstrapProperties.getProperty(key)) - ); - - if (!provenanceReportingProperties.isEmpty()) { - final ProvenanceReportingSchema provenanceReportingSchema = new ProvenanceReportingSchema(provenanceReportingProperties); - provenanceReportingPropsOptional = Optional.of(provenanceReportingSchema); - } - } - - return provenanceReportingPropsOptional; - } - - public static boolean processorSSLOverride(final Properties bootstrapProperties) { - boolean shouldOverride = false; - - if (bootstrapProperties != null) { - shouldOverride = Boolean.parseBoolean(bootstrapProperties.getProperty(USE_PARENT_SSL)); - } - - return shouldOverride; - } - -} diff --git a/minifi/minifi-bootstrap/src/main/java/org/apache/nifi/minifi/bootstrap/util/ConfigTransformer.java b/minifi/minifi-bootstrap/src/main/java/org/apache/nifi/minifi/bootstrap/util/ConfigTransformer.java deleted file mode 100644 index 5b9d0f2a88..0000000000 --- a/minifi/minifi-bootstrap/src/main/java/org/apache/nifi/minifi/bootstrap/util/ConfigTransformer.java +++ /dev/null @@ -1,856 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.nifi.minifi.bootstrap.util; - -import static org.apache.nifi.minifi.bootstrap.configuration.ingestors.PullHttpChangeIngestor.OVERRIDE_SECURITY; -import static org.apache.nifi.minifi.bootstrap.configuration.ingestors.PullHttpChangeIngestor.PULL_HTTP_BASE_KEY; -import static org.apache.nifi.minifi.bootstrap.service.MiNiFiExecCommandProvider.APP_LOG_FILE_EXTENSION; -import static org.apache.nifi.minifi.bootstrap.service.MiNiFiExecCommandProvider.APP_LOG_FILE_NAME; -import static org.apache.nifi.minifi.bootstrap.service.MiNiFiExecCommandProvider.BOOTSTRAP_LOG_FILE_EXTENSION; -import static org.apache.nifi.minifi.bootstrap.service.MiNiFiExecCommandProvider.BOOTSTRAP_LOG_FILE_NAME; -import static org.apache.nifi.minifi.bootstrap.service.MiNiFiExecCommandProvider.DEFAULT_APP_LOG_FILE_NAME; -import static org.apache.nifi.minifi.bootstrap.service.MiNiFiExecCommandProvider.DEFAULT_BOOTSTRAP_LOG_FILE_NAME; -import static org.apache.nifi.minifi.bootstrap.service.MiNiFiExecCommandProvider.DEFAULT_LOG_DIR; -import static org.apache.nifi.minifi.bootstrap.service.MiNiFiExecCommandProvider.DEFAULT_LOG_FILE_EXTENSION; -import static org.apache.nifi.minifi.bootstrap.service.MiNiFiExecCommandProvider.LOG_DIR; - -import java.io.ByteArrayInputStream; -import java.io.FileOutputStream; -import java.io.IOException; -import java.io.InputStream; -import java.io.OutputStream; -import java.io.StringWriter; -import java.nio.ByteBuffer; -import java.nio.file.Files; -import java.nio.file.Path; -import java.nio.file.Paths; -import java.security.SecureRandom; -import java.util.Arrays; -import java.util.Base64; -import java.util.Collection; -import java.util.Collections; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Optional; -import java.util.Properties; -import java.util.stream.Collectors; -import java.util.zip.GZIPOutputStream; -import javax.xml.transform.dom.DOMSource; -import javax.xml.transform.stream.StreamResult; -import org.apache.commons.io.input.TeeInputStream; -import org.apache.commons.io.output.ByteArrayOutputStream; -import org.apache.commons.lang3.StringUtils; -import org.apache.nifi.minifi.bootstrap.RunMiNiFi; -import org.apache.nifi.minifi.bootstrap.configuration.ConfigurationChangeException; -import org.apache.nifi.minifi.bootstrap.exception.InvalidConfigurationException; -import org.apache.nifi.minifi.bootstrap.service.BootstrapFileProvider; -import org.apache.nifi.minifi.commons.schema.ComponentStatusRepositorySchema; -import org.apache.nifi.minifi.commons.schema.ConfigSchema; -import org.apache.nifi.minifi.commons.schema.ConnectionSchema; -import org.apache.nifi.minifi.commons.schema.ContentRepositorySchema; -import org.apache.nifi.minifi.commons.schema.ControllerServiceSchema; -import org.apache.nifi.minifi.commons.schema.CorePropertiesSchema; -import org.apache.nifi.minifi.commons.schema.FlowControllerSchema; -import org.apache.nifi.minifi.commons.schema.FlowFileRepositorySchema; -import org.apache.nifi.minifi.commons.schema.FunnelSchema; -import org.apache.nifi.minifi.commons.schema.PortSchema; -import org.apache.nifi.minifi.commons.schema.ProcessGroupSchema; -import org.apache.nifi.minifi.commons.schema.ProcessorSchema; -import org.apache.nifi.minifi.commons.schema.ProvenanceReportingSchema; -import org.apache.nifi.minifi.commons.schema.ProvenanceRepositorySchema; -import org.apache.nifi.minifi.commons.schema.RemotePortSchema; -import org.apache.nifi.minifi.commons.schema.RemoteProcessGroupSchema; -import org.apache.nifi.minifi.commons.schema.ReportingSchema; -import org.apache.nifi.minifi.commons.schema.SecurityPropertiesSchema; -import org.apache.nifi.minifi.commons.schema.SensitivePropsSchema; -import org.apache.nifi.minifi.commons.schema.SwapSchema; -import org.apache.nifi.minifi.commons.schema.common.ConvertableSchema; -import org.apache.nifi.minifi.commons.schema.common.Schema; -import org.apache.nifi.minifi.commons.schema.common.StringUtil; -import org.apache.nifi.minifi.commons.schema.serialization.SchemaLoader; -import org.apache.nifi.util.NiFiProperties; -import org.apache.nifi.xml.processing.ProcessingException; -import org.apache.nifi.xml.processing.parsers.StandardDocumentProvider; -import org.apache.nifi.xml.processing.transform.StandardTransformProvider; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.w3c.dom.DOMException; -import org.w3c.dom.Document; -import org.w3c.dom.Element; - -public final class ConfigTransformer { - // Underlying version of NIFI will be using - public static final String ROOT_GROUP = "Root-Group"; - - static final String MINIFI_CONFIG_FILE_PATH = "nifi.minifi.config.file"; - static final String MINIFI_BOOTSTRAP_FILE_PATH = "nifi.minifi.bootstrap.file"; - static final String MINIFI_LOG_DIRECTORY = "nifi.minifi.log.directory"; - static final String MINIFI_APP_LOG_FILE = "nifi.minifi.app.log.file"; - static final String MINIFI_BOOTSTRAP_LOG_FILE = "nifi.minifi.bootstrap.log.file"; - - private static final Logger LOGGER = LoggerFactory.getLogger(ConfigTransformer.class); - - private static final String OVERRIDE_CORE_PROPERTIES_KEY = PULL_HTTP_BASE_KEY + ".override.core"; - private static final Base64.Encoder KEY_ENCODER = Base64.getEncoder().withoutPadding(); - private static final int SENSITIVE_PROPERTIES_KEY_LENGTH = 24; - - // Final util classes should have private constructor - private ConfigTransformer() { - } - - public static ByteBuffer generateConfigFiles(InputStream configIs, String configDestinationPath, Properties bootstrapProperties) throws ConfigurationChangeException, IOException { - try (java.io.ByteArrayOutputStream byteArrayOutputStream = new java.io.ByteArrayOutputStream(); - TeeInputStream teeInputStream = new TeeInputStream(configIs, byteArrayOutputStream)) { - - ConfigTransformer.transformConfigFile( - teeInputStream, - configDestinationPath, - bootstrapProperties - ); - - return ByteBuffer.wrap(byteArrayOutputStream.toByteArray()); - } catch (ConfigurationChangeException e){ - throw e; - } catch (Exception e) { - throw new IOException("Unable to successfully transform the provided configuration", e); - } - } - - public static void transformConfigFile(InputStream sourceStream, String destPath, Properties bootstrapProperties) throws Exception { - ConvertableSchema convertableSchemaNew = throwIfInvalid(SchemaLoader.loadConvertableSchemaFromYaml(sourceStream)); - ConfigSchema configSchemaNew = throwIfInvalid(convertableSchemaNew.convert()); - - SecurityPropertiesSchema securityProperties = BootstrapTransformer.buildSecurityPropertiesFromBootstrap(bootstrapProperties).orElse(null); - ProvenanceReportingSchema provenanceReportingProperties = BootstrapTransformer.buildProvenanceReportingPropertiesFromBootstrap(bootstrapProperties).orElse(null); - - // See if we are providing defined properties from the filesystem configurations and use those as the definitive values - if (securityProperties != null) { - configSchemaNew.setSecurityProperties(securityProperties); - LOGGER.info("Bootstrap flow override: Replaced security properties"); - } - - if (provenanceReportingProperties != null) { - configSchemaNew.setProvenanceReportingProperties(provenanceReportingProperties); - LOGGER.info("Bootstrap flow override: Replaced provenance reporting properties"); - } - - // Replace all processor SSL controller services with MiNiFi parent, if bootstrap boolean is set to true - if (BootstrapTransformer.processorSSLOverride(bootstrapProperties)) { - for (ProcessorSchema processorConfig : configSchemaNew.getProcessGroupSchema().getProcessors()) { - processorConfig.getProperties().replace("SSL Context Service", processorConfig.getProperties().get("SSL Context Service"), "SSL-Context-Service"); - LOGGER.info("Bootstrap flow override: Replaced {} SSL Context Service with parent MiNiFi SSL", processorConfig.getName()); - } - } - - Optional.ofNullable(bootstrapProperties) - .map(Properties::entrySet) - .orElse(Collections.emptySet()) - .stream() - .filter(entry -> ((String) entry.getKey()).startsWith("c2")) - .forEach(entry -> configSchemaNew.getNifiPropertiesOverrides().putIfAbsent((String) entry.getKey(), (String) entry.getValue())); - - // Config files and log files - if (bootstrapProperties != null) { - configSchemaNew.getNifiPropertiesOverrides().putIfAbsent(MINIFI_CONFIG_FILE_PATH, bootstrapProperties.getProperty(RunMiNiFi.MINIFI_CONFIG_FILE_KEY)); - } - configSchemaNew.getNifiPropertiesOverrides().putIfAbsent(MINIFI_BOOTSTRAP_FILE_PATH, BootstrapFileProvider.getBootstrapConfFile().getAbsolutePath()); - configSchemaNew.getNifiPropertiesOverrides().putIfAbsent(MINIFI_LOG_DIRECTORY, System.getProperty(LOG_DIR, DEFAULT_LOG_DIR).trim()); - configSchemaNew.getNifiPropertiesOverrides().putIfAbsent(MINIFI_APP_LOG_FILE, - System.getProperty(APP_LOG_FILE_NAME, DEFAULT_APP_LOG_FILE_NAME).trim() + "." + System.getProperty(APP_LOG_FILE_EXTENSION, DEFAULT_LOG_FILE_EXTENSION).trim()); - configSchemaNew.getNifiPropertiesOverrides().putIfAbsent(MINIFI_BOOTSTRAP_LOG_FILE, - System.getProperty(BOOTSTRAP_LOG_FILE_NAME, DEFAULT_BOOTSTRAP_LOG_FILE_NAME).trim() + "." + System.getProperty(BOOTSTRAP_LOG_FILE_EXTENSION, DEFAULT_LOG_FILE_EXTENSION).trim()); - - // Create nifi.properties and flow.xml.gz in memory - ByteArrayOutputStream nifiPropertiesOutputStream = new ByteArrayOutputStream(); - writeNiFiProperties(configSchemaNew, nifiPropertiesOutputStream); - - writeFlowXmlFile(configSchemaNew, destPath); - - // Write nifi.properties and flow.xml.gz - writeNiFiPropertiesFile(nifiPropertiesOutputStream, destPath); - } - - public static T throwIfInvalid(T schema) throws InvalidConfigurationException { - if (!schema.isValid()) { - throw new InvalidConfigurationException("Failed to transform config file due to:[" - + schema.getValidationIssues().stream().sorted().collect(Collectors.joining("], [")) + "]"); - } - return schema; - } - - public static ByteArrayInputStream asByteArrayInputStream(ByteBuffer byteBuffer) { - byte[] config = new byte[byteBuffer.remaining()]; - byteBuffer.get(config); - return new ByteArrayInputStream(config); - } - - protected static void writeNiFiPropertiesFile(ByteArrayOutputStream nifiPropertiesOutputStream, String destPath) throws IOException { - final Path nifiPropertiesPath = Paths.get(destPath, "nifi.properties"); - try (FileOutputStream nifiProperties = new FileOutputStream(nifiPropertiesPath.toString())) { - nifiPropertiesOutputStream.writeTo(nifiProperties); - } finally { - if (nifiPropertiesOutputStream != null) { - nifiPropertiesOutputStream.flush(); - nifiPropertiesOutputStream.close(); - } - } - } - - protected static void writeFlowXmlFile(ConfigSchema configSchema, OutputStream outputStream) throws ConfigTransformerException { - final StreamResult streamResult = new StreamResult(outputStream); - - // configure the transformer and convert the DOM - final StandardTransformProvider transformProvider = new StandardTransformProvider(); - transformProvider.setIndent(true); - - // transform the document to byte stream - transformProvider.transform(createFlowXml(configSchema), streamResult); - } - - protected static void writeFlowXmlFile(ConfigSchema configSchema, String path) throws IOException, ConfigTransformerException { - try (OutputStream fileOut = Files.newOutputStream(Paths.get(path, "flow.xml.gz"))) { - try (OutputStream outStream = new GZIPOutputStream(fileOut)) { - writeFlowXmlFile(configSchema, outStream); - } - } - } - - protected static void writeNiFiProperties(ConfigSchema configSchema, OutputStream outputStream) throws IOException, ConfigurationChangeException { - try { - CorePropertiesSchema coreProperties = configSchema.getCoreProperties(); - FlowFileRepositorySchema flowfileRepoSchema = configSchema.getFlowfileRepositoryProperties(); - SwapSchema swapProperties = flowfileRepoSchema.getSwapProperties(); - ContentRepositorySchema contentRepoProperties = configSchema.getContentRepositoryProperties(); - ComponentStatusRepositorySchema componentStatusRepoProperties = configSchema.getComponentStatusRepositoryProperties(); - SecurityPropertiesSchema securityProperties = configSchema.getSecurityProperties(); - SensitivePropsSchema sensitiveProperties = securityProperties.getSensitiveProps(); - ProvenanceRepositorySchema provenanceRepositorySchema = configSchema.getProvenanceRepositorySchema(); - - OrderedProperties orderedProperties = new OrderedProperties(); - orderedProperties.setProperty("nifi.flow.configuration.file", "./conf/flow.xml.gz", "# Core Properties #" + System.lineSeparator()); - orderedProperties.setProperty("nifi.flow.configuration.archive.enabled", "false"); - orderedProperties.setProperty("nifi.flow.configuration.archive.dir", "./conf/archive/"); - orderedProperties.setProperty("nifi.flowcontroller.autoResumeState", "true"); - orderedProperties.setProperty("nifi.flowcontroller.graceful.shutdown.period", coreProperties.getFlowControllerGracefulShutdownPeriod()); - orderedProperties.setProperty("nifi.flowservice.writedelay.interval", coreProperties.getFlowServiceWriteDelayInterval()); - orderedProperties.setProperty("nifi.administrative.yield.duration", coreProperties.getAdministrativeYieldDuration()); - orderedProperties.setProperty("nifi.variable.registry.properties", coreProperties.getVariableRegistryProperties()); - - orderedProperties.setProperty("nifi.bored.yield.duration", coreProperties.getBoredYieldDuration(), - "# If a component has no work to do (is \"bored\"), how long should we wait before checking again for work?"); - - orderedProperties.setProperty("nifi.authority.provider.configuration.file", "./conf/authority-providers.xml", ""); - orderedProperties.setProperty("nifi.login.identity.provider.configuration.file", "./conf/login-identity-providers.xml"); - orderedProperties.setProperty("nifi.templates.directory", "./conf/templates"); - orderedProperties.setProperty("nifi.ui.banner.text", ""); - orderedProperties.setProperty("nifi.ui.autorefresh.interval", "30 sec"); - orderedProperties.setProperty("nifi.nar.library.directory", "./lib"); - orderedProperties.setProperty("nifi.nar.working.directory", "./work/nar/"); - orderedProperties.setProperty("nifi.nar.library.autoload.directory", "./extensions"); - orderedProperties.setProperty("nifi.documentation.working.directory", "./work/docs/components"); - - orderedProperties.setProperty("nifi.state.management.configuration.file", "./conf/state-management.xml", System.lineSeparator() + - "####################" + - "# State Management #" + - "####################"); - - orderedProperties.setProperty("nifi.state.management.provider.local", "local-provider", "# The ID of the local state provider"); - - orderedProperties.setProperty("nifi.database.directory", "./database_repository", System.lineSeparator() + "# H2 Settings"); - orderedProperties.setProperty("nifi.h2.url.append", ";LOCK_TIMEOUT=25000;WRITE_DELAY=0;AUTO_SERVER=FALSE"); - orderedProperties.setProperty("nifi.flowfile.repository.implementation", String.valueOf(flowfileRepoSchema.getFlowFileRepository()), - System.lineSeparator() + "# FlowFile Repository"); - orderedProperties.setProperty("nifi.flowfile.repository.directory", "./flowfile_repository"); - orderedProperties.setProperty("nifi.flowfile.repository.partitions", String.valueOf(flowfileRepoSchema.getPartitions())); - orderedProperties.setProperty("nifi.flowfile.repository.checkpoint.interval", flowfileRepoSchema.getCheckpointInterval()); - orderedProperties.setProperty("nifi.flowfile.repository.always.sync", Boolean.toString(flowfileRepoSchema.getAlwaysSync())); - - orderedProperties.setProperty("nifi.swap.manager.implementation", "org.apache.nifi.controller.FileSystemSwapManager", ""); - orderedProperties.setProperty("nifi.queue.swap.threshold", String.valueOf(swapProperties.getThreshold())); - orderedProperties.setProperty("nifi.swap.in.period", swapProperties.getInPeriod()); - orderedProperties.setProperty("nifi.swap.in.threads", String.valueOf(swapProperties.getInThreads())); - orderedProperties.setProperty("nifi.swap.out.period", swapProperties.getOutPeriod()); - orderedProperties.setProperty("nifi.swap.out.threads", String.valueOf(swapProperties.getOutThreads())); - - orderedProperties.setProperty("nifi.content.repository.implementation", contentRepoProperties.getContentRepository(), System.lineSeparator() + "# Content Repository"); - orderedProperties.setProperty("nifi.content.claim.max.appendable.size", contentRepoProperties.getContentClaimMaxAppendableSize()); - orderedProperties.setProperty("nifi.content.claim.max.flow.files", String.valueOf(contentRepoProperties.getContentClaimMaxFlowFiles())); - orderedProperties.setProperty("nifi.content.repository.archive.max.retention.period", contentRepoProperties.getContentRepoArchiveMaxRetentionPeriod()); - orderedProperties.setProperty("nifi.content.repository.archive.max.usage.percentage", contentRepoProperties.getContentRepoArchiveMaxUsagePercentage()); - orderedProperties.setProperty("nifi.content.repository.archive.enabled", Boolean.toString(contentRepoProperties.getContentRepoArchiveEnabled())); - orderedProperties.setProperty("nifi.content.repository.directory.default", "./content_repository"); - orderedProperties.setProperty("nifi.content.repository.always.sync", Boolean.toString(contentRepoProperties.getAlwaysSync())); - - orderedProperties.setProperty("nifi.provenance.repository.implementation", provenanceRepositorySchema.getProvenanceRepository(), - System.lineSeparator() + "# Provenance Repository Properties"); - - orderedProperties.setProperty("nifi.provenance.repository.rollover.time", provenanceRepositorySchema.getProvenanceRepoRolloverTimeKey()); - - orderedProperties.setProperty("nifi.provenance.repository.index.shard.size", provenanceRepositorySchema.getProvenanceRepoIndexShardSize()); - orderedProperties.setProperty("nifi.provenance.repository.max.storage.size", provenanceRepositorySchema.getProvenanceRepoMaxStorageSize()); - orderedProperties.setProperty("nifi.provenance.repository.max.storage.time", provenanceRepositorySchema.getProvenanceRepoMaxStorageTime()); - - orderedProperties.setProperty("nifi.provenance.repository.buffer.size", String.valueOf(provenanceRepositorySchema.getProvenanceRepoBufferSize()), - System.lineSeparator() + "# Volatile Provenance Respository Properties"); - - orderedProperties.setProperty("nifi.components.status.repository.implementation", "org.apache.nifi.controller.status.history.VolatileComponentStatusRepository", - System.lineSeparator() + "# Component Status Repository"); - orderedProperties.setProperty("nifi.components.status.repository.buffer.size", String.valueOf(componentStatusRepoProperties.getBufferSize())); - orderedProperties.setProperty("nifi.components.status.snapshot.frequency", componentStatusRepoProperties.getSnapshotFrequency()); - - orderedProperties.setProperty("nifi.web.war.directory", "./lib", System.lineSeparator() + "# web properties #"); - orderedProperties.setProperty("nifi.web.http.host", ""); - orderedProperties.setProperty("nifi.web.http.port", "8081"); - orderedProperties.setProperty("nifi.web.https.host", ""); - orderedProperties.setProperty("nifi.web.https.port", ""); - orderedProperties.setProperty("nifi.web.jetty.working.directory", "./work/jetty"); - orderedProperties.setProperty("nifi.web.jetty.threads", "200"); - - final String sensitivePropertiesKey = sensitiveProperties.getKey(); - final String notnullSensitivePropertiesKey; - // Auto-generate the sensitive properties key if not provided, NiFi security libraries require it - if (StringUtil.isNullOrEmpty(sensitivePropertiesKey)) { - LOGGER.warn("Generating Random Sensitive Properties Key [{}]", NiFiProperties.SENSITIVE_PROPS_KEY); - final SecureRandom secureRandom = new SecureRandom(); - final byte[] sensitivePropertiesKeyBinary = new byte[SENSITIVE_PROPERTIES_KEY_LENGTH]; - secureRandom.nextBytes(sensitivePropertiesKeyBinary); - notnullSensitivePropertiesKey = KEY_ENCODER.encodeToString(sensitivePropertiesKeyBinary); - } else { - notnullSensitivePropertiesKey = sensitivePropertiesKey; - } - orderedProperties.setProperty("nifi.sensitive.props.key", notnullSensitivePropertiesKey, System.lineSeparator() + "# security properties #"); - orderedProperties.setProperty("nifi.sensitive.props.algorithm", sensitiveProperties.getAlgorithm()); - - orderedProperties.setProperty("nifi.security.keystore", securityProperties.getKeystore(), ""); - orderedProperties.setProperty("nifi.security.keystoreType", securityProperties.getKeystoreType()); - orderedProperties.setProperty("nifi.security.keystorePasswd", securityProperties.getKeystorePassword()); - orderedProperties.setProperty("nifi.security.keyPasswd", securityProperties.getKeyPassword()); - orderedProperties.setProperty("nifi.security.truststore", securityProperties.getTruststore()); - orderedProperties.setProperty("nifi.security.truststoreType", securityProperties.getTruststoreType()); - orderedProperties.setProperty("nifi.security.truststorePasswd", securityProperties.getTruststorePassword()); - orderedProperties.setProperty("nifi.security.needClientAuth", ""); - orderedProperties.setProperty("nifi.security.user.credential.cache.duration", "24 hours"); - orderedProperties.setProperty("nifi.security.user.authority.provider", "file-provider"); - orderedProperties.setProperty("nifi.security.user.login.identity.provider", ""); - orderedProperties.setProperty("nifi.security.support.new.account.requests", ""); - - orderedProperties.setProperty("nifi.security.anonymous.authorities", "", "# Valid Authorities include: ROLE_MONITOR,ROLE_DFM,ROLE_ADMIN,ROLE_PROVENANCE,ROLE_NIFI"); - orderedProperties.setProperty("nifi.security.ocsp.responder.url", ""); - orderedProperties.setProperty("nifi.security.ocsp.responder.certificate", ""); - - orderedProperties.setProperty("nifi.cluster.is.node", "false", System.lineSeparator() + System.lineSeparator() + "# cluster node properties (only configure for cluster nodes) #"); - orderedProperties.setProperty("nifi.cluster.is.manager", "false", System.lineSeparator() + "# cluster manager properties (only configure for cluster manager) #"); - - for (Map.Entry entry : configSchema.getNifiPropertiesOverrides().entrySet()) { - orderedProperties.setProperty(entry.getKey(), entry.getValue()); - } - - orderedProperties.store(outputStream, PROPERTIES_FILE_APACHE_2_0_LICENSE); - } catch (NullPointerException e) { - throw new ConfigurationChangeException("Failed to parse the config YAML while creating the nifi.properties", e); - } finally { - outputStream.close(); - } - } - - protected static DOMSource createFlowXml(ConfigSchema configSchema) throws ConfigTransformerException { - try { - // create a new, empty document - final StandardDocumentProvider documentProvider = new StandardDocumentProvider(); - documentProvider.setNamespaceAware(true); - final Document doc = documentProvider.newDocument(); - - // populate document with controller state - final Element rootNode = doc.createElement("flowController"); - doc.appendChild(rootNode); - CorePropertiesSchema coreProperties = configSchema.getCoreProperties(); - addTextElement(rootNode, "maxTimerDrivenThreadCount", String.valueOf(coreProperties.getMaxConcurrentThreads())); - addTextElement(rootNode, "maxEventDrivenThreadCount", String.valueOf(coreProperties.getMaxConcurrentThreads())); - - FlowControllerSchema flowControllerProperties = configSchema.getFlowControllerProperties(); - - final Element element = doc.createElement("rootGroup"); - rootNode.appendChild(element); - - ProcessGroupSchema processGroupSchema = configSchema.getProcessGroupSchema(); - processGroupSchema.setId(ROOT_GROUP); - processGroupSchema.setName(flowControllerProperties.getName()); - processGroupSchema.setComment(flowControllerProperties.getComment()); - - addProcessGroup(doc, element, processGroupSchema, new ParentGroupIdResolver(processGroupSchema)); - - SecurityPropertiesSchema securityProperties = configSchema.getSecurityProperties(); - boolean useSSL = securityProperties.useSSL(); - if (useSSL) { - Element controllerServicesNode = doc.getElementById("controllerServices"); - if (controllerServicesNode == null) { - controllerServicesNode = doc.createElement("controllerServices"); - } - - rootNode.appendChild(controllerServicesNode); - addSSLControllerService(controllerServicesNode, securityProperties); - } - - List reportingTasks = configSchema.getReportingTasksSchema(); - ProvenanceReportingSchema provenanceProperties = configSchema.getProvenanceReportingProperties(); - if (provenanceProperties != null) { - provenanceProperties.setSSL(useSSL); - ReportingSchema provenance = provenanceProperties.convert(); - provenance.setId("Provenance-Reporting"); - provenance.setName("Site-To-Site-Provenance-Reporting"); - reportingTasks.add(provenance); - } - if (reportingTasks != null) { - final Element reportingTasksNode = doc.createElement("reportingTasks"); - rootNode.appendChild(reportingTasksNode); - for (ReportingSchema task : reportingTasks) { - addReportingTask(reportingTasksNode, task); - } - } - - return new DOMSource(doc); - } catch (final ProcessingException | DOMException | IllegalArgumentException e) { - throw new ConfigTransformerException(e); - } catch (Exception e) { - throw new ConfigTransformerException("Failed to parse the config YAML while writing the top level of the flow xml", e); - } - } - - protected static void addSSLControllerService(final Element element, SecurityPropertiesSchema securityProperties) throws ConfigurationChangeException { - try { - final Element serviceElement = element.getOwnerDocument().createElement("controllerService"); - addTextElement(serviceElement, "id", "SSL-Context-Service"); - addTextElement(serviceElement, "name", "SSL-Context-Service"); - addTextElement(serviceElement, "comment", ""); - addTextElement(serviceElement, "class", "org.apache.nifi.ssl.StandardSSLContextService"); - - addTextElement(serviceElement, "enabled", "true"); - - Map attributes = new HashMap<>(); - attributes.put("Keystore Filename", securityProperties.getKeystore()); - attributes.put("Keystore Type", securityProperties.getKeystoreType()); - attributes.put("Keystore Password", securityProperties.getKeystorePassword()); - attributes.put("Truststore Filename", securityProperties.getTruststore()); - attributes.put("Truststore Type", securityProperties.getTruststoreType()); - attributes.put("Truststore Password", securityProperties.getTruststorePassword()); - attributes.put("SSL Protocol", securityProperties.getSslProtocol()); - - addConfiguration(serviceElement, attributes); - - element.appendChild(serviceElement); - } catch (Exception e) { - throw new ConfigurationChangeException("Failed to parse the config YAML while trying to create an SSL Controller Service", e); - } - } - - protected static void addControllerService(final Element element, ControllerServiceSchema controllerServiceSchema) throws ConfigurationChangeException { - try { - final Element serviceElement = element.getOwnerDocument().createElement("controllerService"); - addTextElement(serviceElement, "id", controllerServiceSchema.getId()); - addTextElement(serviceElement, "name", controllerServiceSchema.getName()); - addTextElement(serviceElement, "comment", ""); - addTextElement(serviceElement, "class", controllerServiceSchema.getServiceClass()); - - addTextElement(serviceElement, "enabled", "true"); - - Map attributes = controllerServiceSchema.getProperties(); - - addConfiguration(serviceElement, attributes); - - String annotationData = controllerServiceSchema.getAnnotationData(); - if (annotationData != null && !annotationData.isEmpty()) { - addTextElement(element, "annotationData", annotationData); - } - - element.appendChild(serviceElement); - } catch (Exception e) { - throw new ConfigurationChangeException("Failed to parse the config YAML while trying to create an SSL Controller Service", e); - } - } - - protected static void addProcessGroup(Document doc, Element element, ProcessGroupSchema processGroupSchema, ParentGroupIdResolver parentGroupIdResolver) throws ConfigurationChangeException { - try { - String processGroupId = processGroupSchema.getId(); - addTextElement(element, "id", processGroupId); - addTextElement(element, "name", processGroupSchema.getName()); - addPosition(element); - addTextElement(element, "comment", processGroupSchema.getComment()); - - for (ProcessorSchema processorConfig : processGroupSchema.getProcessors()) { - addProcessor(element, processorConfig); - } - - for (PortSchema portSchema : processGroupSchema.getInputPortSchemas()) { - addPort(doc, element, portSchema, "inputPort"); - } - - for (PortSchema portSchema : processGroupSchema.getOutputPortSchemas()) { - addPort(doc, element, portSchema, "outputPort"); - } - - for (FunnelSchema funnelSchema : processGroupSchema.getFunnels()) { - addFunnel(element, funnelSchema); - } - - for (ProcessGroupSchema child : processGroupSchema.getProcessGroupSchemas()) { - Element processGroups = doc.createElement("processGroup"); - element.appendChild(processGroups); - addProcessGroup(doc, processGroups, child, parentGroupIdResolver); - } - - for (RemoteProcessGroupSchema remoteProcessGroupSchema : processGroupSchema.getRemoteProcessGroups()) { - addRemoteProcessGroup(element, remoteProcessGroupSchema); - } - - for (ConnectionSchema connectionConfig : processGroupSchema.getConnections()) { - addConnection(element, connectionConfig, parentGroupIdResolver); - } - - for (ControllerServiceSchema controllerServiceSchema : processGroupSchema.getControllerServices()) { - addControllerService(element, controllerServiceSchema); - } - } catch (ConfigurationChangeException e) { - throw e; - } catch (Exception e) { - throw new ConfigurationChangeException("Failed to parse the config YAML while trying to creating the root Process Group", e); - } - } - - protected static void addPort(Document doc, Element parentElement, PortSchema portSchema, String tag) { - Element element = doc.createElement(tag); - parentElement.appendChild(element); - - addTextElement(element, "id", portSchema.getId()); - addTextElement(element, "name", portSchema.getName()); - - addPosition(element); - addTextElement(element, "comments", null); - - addTextElement(element, "scheduledState", "RUNNING"); - } - - protected static void addProcessor(final Element parentElement, ProcessorSchema processorConfig) throws ConfigurationChangeException { - try { - final Document doc = parentElement.getOwnerDocument(); - final Element element = doc.createElement("processor"); - parentElement.appendChild(element); - - addTextElement(element, "id", processorConfig.getId()); - addTextElement(element, "name", processorConfig.getName()); - - addPosition(element); - addStyle(element); - - addTextElement(element, "comment", ""); - addTextElement(element, "class", processorConfig.getProcessorClass()); - addTextElement(element, "maxConcurrentTasks", String.valueOf(processorConfig.getMaxConcurrentTasks())); - addTextElement(element, "schedulingPeriod", processorConfig.getSchedulingPeriod()); - addTextElement(element, "penalizationPeriod", processorConfig.getPenalizationPeriod()); - addTextElement(element, "yieldPeriod", processorConfig.getYieldPeriod()); - addTextElement(element, "bulletinLevel", "WARN"); - addTextElement(element, "lossTolerant", "false"); - addTextElement(element, "scheduledState", "RUNNING"); - addTextElement(element, "schedulingStrategy", processorConfig.getSchedulingStrategy()); - addTextElement(element, "runDurationNanos", String.valueOf(processorConfig.getRunDurationNanos())); - - String annotationData = processorConfig.getAnnotationData(); - if (annotationData != null && !annotationData.isEmpty()) { - addTextElement(element, "annotationData", annotationData); - } - - addConfiguration(element, processorConfig.getProperties()); - - Collection autoTerminatedRelationships = processorConfig.getAutoTerminatedRelationshipsList(); - if (autoTerminatedRelationships != null) { - for (String rel : autoTerminatedRelationships) { - addTextElement(element, "autoTerminatedRelationship", rel); - } - } - } catch (Exception e) { - throw new ConfigurationChangeException("Failed to parse the config YAML while trying to add a Processor", e); - } - } - - - protected static void addFunnel(final Element parentElement, FunnelSchema funnelSchema) { - Document document = parentElement.getOwnerDocument(); - Element element = document.createElement("funnel"); - parentElement.appendChild(element); - - addTextElement(element, "id", funnelSchema.getId()); - - addPosition(element); - } - - protected static void addReportingTask(final Element parentElement, ReportingSchema reportingSchema) throws ConfigurationChangeException { - try { - final Document doc = parentElement.getOwnerDocument(); - final Element element = doc.createElement("reportingTask"); - parentElement.appendChild(element); - - addTextElement(element, "id", reportingSchema.getId()); - addTextElement(element, "name", reportingSchema.getName()); - addTextElement(element, "comment", reportingSchema.getComment()); - addTextElement(element, "class", reportingSchema.getReportingClass()); - addTextElement(element, "schedulingPeriod", reportingSchema.getSchedulingPeriod()); - addTextElement(element, "scheduledState", "RUNNING"); - addTextElement(element, "schedulingStrategy", reportingSchema.getSchedulingStrategy()); - - addConfiguration(element, reportingSchema.getProperties()); - - parentElement.appendChild(element); - } catch (Exception e) { - throw new ConfigurationChangeException("Failed to parse the config YAML while trying to add the Provenance Reporting Task", e); - } - } - - protected static void addConfiguration(final Element element, Map elementConfig) { - final Document doc = element.getOwnerDocument(); - if (elementConfig == null) { - return; - } - for (final Map.Entry entry : elementConfig.entrySet()) { - - final Element propElement = doc.createElement("property"); - addTextElement(propElement, "name", entry.getKey()); - if (entry.getValue() != null) { - addTextElement(propElement, "value", entry.getValue().toString()); - } - - element.appendChild(propElement); - } - } - - protected static void addStyle(final Element parentElement) { - final Element element = parentElement.getOwnerDocument().createElement("styles"); - parentElement.appendChild(element); - } - - protected static void addRemoteProcessGroup(final Element parentElement, RemoteProcessGroupSchema remoteProcessGroupProperties) throws ConfigurationChangeException { - try { - final Document doc = parentElement.getOwnerDocument(); - final Element element = doc.createElement("remoteProcessGroup"); - parentElement.appendChild(element); - addTextElement(element, "id", remoteProcessGroupProperties.getId()); - addTextElement(element, "name", remoteProcessGroupProperties.getName()); - addPosition(element); - addTextElement(element, "comment", remoteProcessGroupProperties.getComment()); - // In the case we have multiple urls, select the first - addTextElement(element, "url", Arrays.asList(remoteProcessGroupProperties.getUrls().split(",")).get(0)); - addTextElement(element, "urls", remoteProcessGroupProperties.getUrls()); - addTextElement(element, "timeout", remoteProcessGroupProperties.getTimeout()); - addTextElement(element, "yieldPeriod", remoteProcessGroupProperties.getYieldPeriod()); - addTextElement(element, "transmitting", "true"); - addTextElement(element, "transportProtocol", remoteProcessGroupProperties.getTransportProtocol()); - addTextElement(element, "proxyHost", remoteProcessGroupProperties.getProxyHost()); - if (remoteProcessGroupProperties.getProxyPort() != null) { - addTextElement(element, "proxyPort", Integer.toString(remoteProcessGroupProperties.getProxyPort())); - } - addTextElement(element, "proxyUser", remoteProcessGroupProperties.getProxyUser()); - if (!StringUtils.isEmpty(remoteProcessGroupProperties.getProxyPassword())) { - addTextElement(element, "proxyPassword", remoteProcessGroupProperties.getProxyPassword()); - } - - List remoteInputPorts = remoteProcessGroupProperties.getInputPorts(); - for (RemotePortSchema remoteInputPortSchema : remoteInputPorts) { - addRemoteGroupPort(element, remoteInputPortSchema, "inputPort"); - } - - List remoteOutputPorts = remoteProcessGroupProperties.getOutputPorts(); - for (RemotePortSchema remoteOutputPortSchema : remoteOutputPorts) { - addRemoteGroupPort(element, remoteOutputPortSchema, "outputPort"); - } - addTextElement(element, "networkInterface", remoteProcessGroupProperties.getLocalNetworkInterface()); - - parentElement.appendChild(element); - } catch (Exception e) { - throw new ConfigurationChangeException("Failed to parse the config YAML while trying to add the Remote Process Group", e); - } - } - - protected static void addRemoteGroupPort(final Element parentElement, RemotePortSchema inputPort, String tagName) throws ConfigurationChangeException { - try { - final Document doc = parentElement.getOwnerDocument(); - final Element element = doc.createElement(tagName); - parentElement.appendChild(element); - addTextElement(element, "id", inputPort.getId()); - addTextElement(element, "name", inputPort.getName()); - addPosition(element); - addTextElement(element, "comments", inputPort.getComment()); - addTextElement(element, "scheduledState", "RUNNING"); - addTextElement(element, "maxConcurrentTasks", String.valueOf(inputPort.getMax_concurrent_tasks())); - addTextElement(element, "useCompression", String.valueOf(inputPort.getUseCompression())); - - parentElement.appendChild(element); - } catch (Exception e) { - throw new ConfigurationChangeException("Failed to parse the config YAML while trying to add the input port of the Remote Process Group", e); - } - } - - protected static void addConnection(final Element parentElement, ConnectionSchema connectionProperties, ParentGroupIdResolver parentGroupIdResolver) throws ConfigurationChangeException { - try { - final Document doc = parentElement.getOwnerDocument(); - final Element element = doc.createElement("connection"); - parentElement.appendChild(element); - - addTextElement(element, "id", connectionProperties.getId()); - addTextElement(element, "name", connectionProperties.getName()); - - final Element bendPointsElement = doc.createElement("bendPoints"); - element.appendChild(bendPointsElement); - - addTextElement(element, "labelIndex", "1"); - addTextElement(element, "zIndex", "0"); - - addConnectionSourceOrDestination(element, "source", connectionProperties.getSourceId(), parentGroupIdResolver); - addConnectionSourceOrDestination(element, "destination", connectionProperties.getDestinationId(), parentGroupIdResolver); - - List sourceRelationshipNames = connectionProperties.getSourceRelationshipNames(); - if (sourceRelationshipNames.isEmpty()) { - addTextElement(element, "relationship", null); - } else { - for (String relationshipName : sourceRelationshipNames) { - addTextElement(element, "relationship", relationshipName); - } - } - - addTextElement(element, "maxWorkQueueSize", String.valueOf(connectionProperties.getMaxWorkQueueSize())); - addTextElement(element, "maxWorkQueueDataSize", connectionProperties.getMaxWorkQueueDataSize()); - - addTextElement(element, "flowFileExpiration", connectionProperties.getFlowfileExpiration()); - addTextElementIfNotNullOrEmpty(element, "queuePrioritizerClass", connectionProperties.getQueuePrioritizerClass()); - - parentElement.appendChild(element); - } catch (Exception e) { - throw new ConfigurationChangeException("Failed to parse the config YAML while trying to add the connection from the Processor to the input port of the Remote Process Group", e); - } - } - - protected static void addConnectionSourceOrDestination(Element element, String sourceOrDestination, String id, ParentGroupIdResolver parentGroupIdResolver) { - String idTag = sourceOrDestination + "Id"; - String groupIdTag = sourceOrDestination + "GroupId"; - String typeTag = sourceOrDestination + "Type"; - - String parentId; - String type; - - if ((parentId = parentGroupIdResolver.getRemoteInputPortParentId(id)) != null) { - type = "REMOTE_INPUT_PORT"; - } else if ((parentId = parentGroupIdResolver.getRemoteOutputPortParentId(id)) != null) { - type = "REMOTE_OUTPUT_PORT"; - } else if ((parentId = parentGroupIdResolver.getInputPortParentId(id)) != null) { - type = "INPUT_PORT"; - } else if ((parentId = parentGroupIdResolver.getOutputPortParentId(id)) != null) { - type = "OUTPUT_PORT"; - } else if ((parentId = parentGroupIdResolver.getFunnelParentId(id)) != null) { - type = "FUNNEL"; - } else { - parentId = parentGroupIdResolver.getProcessorParentId(id); - type = "PROCESSOR"; - } - - addTextElement(element, idTag, id); - if (parentId != null) { - addTextElement(element, groupIdTag, parentId); - } - addTextElement(element, typeTag, type); - } - - protected static void addPosition(final Element parentElement) { - final Element element = parentElement.getOwnerDocument().createElement("position"); - element.setAttribute("x", "0"); - element.setAttribute("y", "0"); - parentElement.appendChild(element); - } - - protected static void addTextElementIfNotNullOrEmpty(final Element element, final String name, final String value) { - StringUtil.doIfNotNullOrEmpty(value, s -> addTextElement(element, name, value)); - } - - protected static void addTextElement(final Element element, final String name, final String value) { - final Document doc = element.getOwnerDocument(); - final Element toAdd = doc.createElement(name); - toAdd.setTextContent(value); - element.appendChild(toAdd); - } - - public static ByteBuffer overrideNonFlowSectionsFromOriginalSchema(byte[] newSchema, ByteBuffer currentConfigScheme, Properties bootstrapProperties) - throws InvalidConfigurationException { - try { - boolean overrideCoreProperties = ConfigTransformer.overrideCoreProperties(bootstrapProperties); - boolean overrideSecurityProperties = ConfigTransformer.overrideSecurityProperties(bootstrapProperties); - if (overrideCoreProperties && overrideSecurityProperties) { - return ByteBuffer.wrap(newSchema); - } else { - ConvertableSchema schemaNew = ConfigTransformer - .throwIfInvalid(SchemaLoader.loadConvertableSchemaFromYaml(new ByteArrayInputStream(newSchema))); - ConfigSchema configSchemaNew = ConfigTransformer.throwIfInvalid(schemaNew.convert()); - ConvertableSchema schemaOld = ConfigTransformer - .throwIfInvalid(SchemaLoader.loadConvertableSchemaFromYaml(new ByteBufferInputStream(currentConfigScheme))); - ConfigSchema configSchemaOld = ConfigTransformer.throwIfInvalid(schemaOld.convert()); - - configSchemaNew.setNifiPropertiesOverrides(configSchemaOld.getNifiPropertiesOverrides()); - - if (!overrideCoreProperties) { - LOGGER.debug("Preserving previous core properties..."); - configSchemaNew.setCoreProperties(configSchemaOld.getCoreProperties()); - } - - if (!overrideSecurityProperties) { - LOGGER.debug("Preserving previous security properties..."); - configSchemaNew.setSecurityProperties(configSchemaOld.getSecurityProperties()); - } - - StringWriter writer = new StringWriter(); - SchemaLoader.toYaml(configSchemaNew, writer); - return ByteBuffer.wrap(writer.toString().getBytes()).asReadOnlyBuffer(); - } - } catch (Exception e) { - throw new InvalidConfigurationException("Loading the old and the new schema for merging was not successful", e); - } - } - - private static boolean overrideSecurityProperties(Properties properties) { - String overrideSecurityProperties = (String) properties.getOrDefault(OVERRIDE_SECURITY, "false"); - return Boolean.parseBoolean(overrideSecurityProperties); - } - - private static boolean overrideCoreProperties(Properties properties) { - String overrideCoreProps = (String) properties.getOrDefault(OVERRIDE_CORE_PROPERTIES_KEY, "false"); - return Boolean.parseBoolean(overrideCoreProps); - } - - public static final String PROPERTIES_FILE_APACHE_2_0_LICENSE = - " Licensed to the Apache Software Foundation (ASF) under one or more\n" + - "# contributor license agreements. See the NOTICE file distributed with\n" + - "# this work for additional information regarding copyright ownership.\n" + - "# The ASF licenses this file to You under the Apache License, Version 2.0\n" + - "# (the \"License\"); you may not use this file except in compliance with\n" + - "# the License. You may obtain a copy of the License at\n" + - "#\n" + - "# http://www.apache.org/licenses/LICENSE-2.0\n" + - "#\n" + - "# Unless required by applicable law or agreed to in writing, software\n" + - "# distributed under the License is distributed on an \"AS IS\" BASIS,\n" + - "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n" + - "# See the License for the specific language governing permissions and\n" + - "# limitations under the License.\n" + - "\n"; - -} diff --git a/minifi/minifi-bootstrap/src/main/java/org/apache/nifi/minifi/bootstrap/util/ConfigTransformerException.java b/minifi/minifi-bootstrap/src/main/java/org/apache/nifi/minifi/bootstrap/util/ConfigTransformerException.java deleted file mode 100644 index b56b69b22d..0000000000 --- a/minifi/minifi-bootstrap/src/main/java/org/apache/nifi/minifi/bootstrap/util/ConfigTransformerException.java +++ /dev/null @@ -1,32 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.nifi.minifi.bootstrap.util; - -public class ConfigTransformerException extends Exception { - public ConfigTransformerException(String message) { - super(message); - } - - public ConfigTransformerException(Throwable cause) { - super(cause); - } - - public ConfigTransformerException(String message, Throwable cause) { - super(message, cause); - } -} \ No newline at end of file diff --git a/minifi/minifi-bootstrap/src/main/java/org/apache/nifi/minifi/bootstrap/util/OrderedProperties.java b/minifi/minifi-bootstrap/src/main/java/org/apache/nifi/minifi/bootstrap/util/OrderedProperties.java index 5d5fe51395..89a3c0ca10 100644 --- a/minifi/minifi-bootstrap/src/main/java/org/apache/nifi/minifi/bootstrap/util/OrderedProperties.java +++ b/minifi/minifi-bootstrap/src/main/java/org/apache/nifi/minifi/bootstrap/util/OrderedProperties.java @@ -34,6 +34,7 @@ import java.util.Map; import java.util.Properties; import java.util.Set; import java.util.stream.Collectors; +import org.apache.commons.lang3.StringUtils; public class OrderedProperties extends Properties { private final Map textBeforeMap = new HashMap<>(); @@ -74,7 +75,7 @@ public class OrderedProperties extends Properties { int equalsIndex = line.indexOf('='); if (equalsIndex != -1) { String textBefore = textBeforeMap.get(line.substring(0, equalsIndex)); - if (textBefore != null) { + if (StringUtils.isNotBlank(textBefore)) { bufferedWriter.write(textBefore); bufferedWriter.newLine(); } diff --git a/minifi/minifi-bootstrap/src/main/java/org/apache/nifi/minifi/bootstrap/util/ParentGroupIdResolver.java b/minifi/minifi-bootstrap/src/main/java/org/apache/nifi/minifi/bootstrap/util/ParentGroupIdResolver.java deleted file mode 100644 index d0a7d2957a..0000000000 --- a/minifi/minifi-bootstrap/src/main/java/org/apache/nifi/minifi/bootstrap/util/ParentGroupIdResolver.java +++ /dev/null @@ -1,101 +0,0 @@ -/* - * - * * Licensed to the Apache Software Foundation (ASF) under one or more - * * contributor license agreements. See the NOTICE file distributed with - * * this work for additional information regarding copyright ownership. - * * The ASF licenses this file to You under the Apache License, Version 2.0 - * * (the "License"); you may not use this file except in compliance with - * * the License. You may obtain a copy of the License at - * * - * * http://www.apache.org/licenses/LICENSE-2.0 - * * - * * Unless required by applicable law or agreed to in writing, software - * * distributed under the License is distributed on an "AS IS" BASIS, - * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * * See the License for the specific language governing permissions and - * * limitations under the License. - * - */ - -package org.apache.nifi.minifi.bootstrap.util; - -import org.apache.nifi.minifi.commons.schema.ProcessGroupSchema; -import org.apache.nifi.minifi.commons.schema.RemotePortSchema; -import org.apache.nifi.minifi.commons.schema.RemoteProcessGroupSchema; -import org.apache.nifi.minifi.commons.schema.common.BaseSchemaWithId; - -import java.util.Collection; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.function.Function; - -public class ParentGroupIdResolver { - private final Map processorIdToParentIdMap; - private final Map inputPortIdToParentIdMap; - private final Map outputPortIdToParentIdMap; - private final Map funnelIdToParentIdMap; - private final Map remoteInputPortIdToParentIdMap; - private final Map remoteOutputPortIdToParentIdMap; - - public ParentGroupIdResolver(ProcessGroupSchema processGroupSchema) { - this.processorIdToParentIdMap = getParentIdMap(processGroupSchema, ProcessGroupSchema::getProcessors); - this.inputPortIdToParentIdMap = getParentIdMap(processGroupSchema, ProcessGroupSchema::getInputPortSchemas); - this.outputPortIdToParentIdMap = getParentIdMap(processGroupSchema, ProcessGroupSchema::getOutputPortSchemas); - this.funnelIdToParentIdMap = getParentIdMap(processGroupSchema, ProcessGroupSchema::getFunnels); - this.remoteInputPortIdToParentIdMap = getRemotePortParentIdMap(processGroupSchema, RemoteProcessGroupSchema::getInputPorts); - this.remoteOutputPortIdToParentIdMap = getRemotePortParentIdMap(processGroupSchema, RemoteProcessGroupSchema::getOutputPorts); - } - - protected static Map getParentIdMap(ProcessGroupSchema processGroupSchema, Function> schemaAccessor) { - Map map = new HashMap<>(); - getParentIdMap(processGroupSchema, map, schemaAccessor); - return map; - } - - protected static void getParentIdMap(ProcessGroupSchema processGroupSchema, Map output, Function> schemaAccessor) { - schemaAccessor.apply(processGroupSchema).forEach(p -> output.put(p.getId(), processGroupSchema.getId())); - processGroupSchema.getProcessGroupSchemas().forEach(p -> getParentIdMap(p, output, schemaAccessor)); - } - - protected static Map getRemotePortParentIdMap(ProcessGroupSchema processGroupSchema, Function> getPortsFunction) { - Map result = new HashMap<>(); - getRemotePortParentIdMap(processGroupSchema, result, getPortsFunction); - return result; - } - - protected static void getRemotePortParentIdMap(ProcessGroupSchema processGroupSchema, Map output, Function> getPortsFunction) { - for (RemoteProcessGroupSchema remoteProcessGroupSchema : processGroupSchema.getRemoteProcessGroups()) { - for (RemotePortSchema remotePortSchema : getPortsFunction.apply(remoteProcessGroupSchema)) { - output.put(remotePortSchema.getId(), remoteProcessGroupSchema.getId()); - } - } - processGroupSchema.getProcessGroupSchemas().forEach(p -> getRemotePortParentIdMap(p, output, getPortsFunction)); - } - - public String getRemoteInputPortParentId(String id) { - return remoteInputPortIdToParentIdMap.get(id); - } - - public String getRemoteOutputPortParentId(String id) { - return remoteOutputPortIdToParentIdMap.get(id); - } - - public String getInputPortParentId(String id) { - return inputPortIdToParentIdMap.get(id); - } - - public String getOutputPortParentId(String id) { - return outputPortIdToParentIdMap.get(id); - } - - public String getProcessorParentId(String id) { - return processorIdToParentIdMap.get(id); - } - - public String getFunnelParentId(String id) { - return funnelIdToParentIdMap.get(id); - } -} diff --git a/minifi/minifi-bootstrap/src/test/java/org/apache/nifi/minifi/bootstrap/RunMiNiFiTest.java b/minifi/minifi-bootstrap/src/test/java/org/apache/nifi/minifi/bootstrap/RunMiNiFiTest.java deleted file mode 100644 index 82fcfae8ba..0000000000 --- a/minifi/minifi-bootstrap/src/test/java/org/apache/nifi/minifi/bootstrap/RunMiNiFiTest.java +++ /dev/null @@ -1,129 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.nifi.minifi.bootstrap; - -import org.apache.nifi.minifi.bootstrap.util.BootstrapTransformer; -import org.apache.nifi.minifi.commons.schema.ProvenanceReportingSchema; -import org.apache.nifi.minifi.commons.schema.SecurityPropertiesSchema; -import org.apache.nifi.minifi.commons.schema.SensitivePropsSchema; -import org.junit.jupiter.api.Test; - -import java.io.IOException; -import java.io.InputStream; -import java.util.Optional; -import java.util.Properties; - -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertFalse; -import static org.junit.jupiter.api.Assertions.assertNotNull; -import static org.junit.jupiter.api.Assertions.assertTrue; - -public class RunMiNiFiTest { - - @Test - public void buildSecurityPropertiesNotDefined() throws Exception { - final Properties bootstrapProperties = getTestBootstrapProperties("bootstrap-ssl-ctx/bootstrap.conf.default"); - final Optional securityPropsOptional = BootstrapTransformer.buildSecurityPropertiesFromBootstrap(bootstrapProperties); - assertFalse(securityPropsOptional.isPresent()); - } - - @Test - public void buildSecurityPropertiesDefined() throws Exception { - final Properties bootstrapProperties = getTestBootstrapProperties("bootstrap-ssl-ctx/bootstrap.conf.configured"); - final Optional securityPropsOptional = BootstrapTransformer.buildSecurityPropertiesFromBootstrap(bootstrapProperties); - assertTrue(securityPropsOptional.isPresent()); - - final SecurityPropertiesSchema securityPropertiesSchema = securityPropsOptional.get(); - assertEquals("/my/test/keystore.jks", securityPropertiesSchema.getKeystore()); - assertEquals("JKS", securityPropertiesSchema.getKeystoreType()); - assertEquals("mykeystorepassword", securityPropertiesSchema.getKeystorePassword()); - assertEquals("mykeypassword", securityPropertiesSchema.getKeyPassword()); - - assertEquals("/my/test/truststore.jks", securityPropertiesSchema.getTruststore()); - assertEquals("JKS", securityPropertiesSchema.getTruststoreType()); - assertEquals("mytruststorepassword", securityPropertiesSchema.getTruststorePassword()); - - assertEquals("TLS", securityPropertiesSchema.getSslProtocol()); - - final SensitivePropsSchema sensitiveProps = securityPropertiesSchema.getSensitiveProps(); - assertNotNull(sensitiveProps); - - assertEquals("sensitivepropskey", sensitiveProps.getKey()); - assertEquals("algo", sensitiveProps.getAlgorithm()); - - - assertTrue(securityPropertiesSchema.isValid()); - } - - @Test - public void buildSecurityPropertiesDefinedButInvalid() throws Exception { - final Properties bootstrapProperties = getTestBootstrapProperties("bootstrap-ssl-ctx/bootstrap.conf.configured.invalid"); - final Optional securityPropsOptional = BootstrapTransformer.buildSecurityPropertiesFromBootstrap(bootstrapProperties); - assertTrue(securityPropsOptional.isPresent()); - - final SecurityPropertiesSchema securityPropertiesSchema = securityPropsOptional.get(); - assertEquals("/my/test/keystore.jks", securityPropertiesSchema.getKeystore()); - assertEquals("NOTAKEYSTORETYPE", securityPropertiesSchema.getKeystoreType()); - assertEquals("mykeystorepassword", securityPropertiesSchema.getKeystorePassword()); - assertEquals("mykeypassword", securityPropertiesSchema.getKeyPassword()); - - assertEquals("/my/test/truststore.jks", securityPropertiesSchema.getTruststore()); - assertEquals("JKS", securityPropertiesSchema.getTruststoreType()); - assertEquals("mytruststorepassword", securityPropertiesSchema.getTruststorePassword()); - - final SensitivePropsSchema sensitiveProps = securityPropertiesSchema.getSensitiveProps(); - assertNotNull(sensitiveProps); - - assertEquals("sensitivepropskey", sensitiveProps.getKey()); - assertEquals("algo", sensitiveProps.getAlgorithm()); - - assertFalse(securityPropertiesSchema.isValid()); - } - - @Test - public void buildProvenanceReportingNotDefined() throws Exception { - final Properties bootstrapProperties = getTestBootstrapProperties("bootstrap-provenance-reporting/bootstrap.conf.default"); - final Optional provenanceReportingPropsOptional = BootstrapTransformer.buildProvenanceReportingPropertiesFromBootstrap(bootstrapProperties); - assertFalse(provenanceReportingPropsOptional.isPresent()); - } - - @Test - public void buildProvenanceReportingDefined() throws Exception { - final Properties bootstrapProperties = getTestBootstrapProperties("bootstrap-provenance-reporting/bootstrap.conf.configured"); - final Optional provenanceReportingPropsOptional = BootstrapTransformer.buildProvenanceReportingPropertiesFromBootstrap(bootstrapProperties); - assertTrue(provenanceReportingPropsOptional.isPresent()); - - final ProvenanceReportingSchema provenanceReportingSchema = provenanceReportingPropsOptional.get(); - assertEquals("This is a comment!", provenanceReportingSchema.getComment()); - assertEquals("TIMER_DRIVEN", provenanceReportingSchema.getSchedulingStrategy()); - assertEquals("15 secs", provenanceReportingSchema.getSchedulingPeriod()); - assertEquals("http://localhost:8080/", provenanceReportingSchema.getDestinationUrl()); - assertEquals("provenance", provenanceReportingSchema.getPortName()); - assertEquals("http://${hostname(true)}:8081/nifi", provenanceReportingSchema.getOriginatingUrl()); - assertEquals("10 secs", provenanceReportingSchema.getTimeout()); - } - - - public static Properties getTestBootstrapProperties(final String fileName) throws IOException { - final Properties bootstrapProperties = new Properties(); - try (final InputStream fis = RunMiNiFiTest.class.getClassLoader().getResourceAsStream(fileName)) { - bootstrapProperties.load(fis); - } - return bootstrapProperties; - } - -} \ No newline at end of file diff --git a/minifi/minifi-bootstrap/src/test/java/org/apache/nifi/minifi/bootstrap/configuration/differentiators/WholeConfigDifferentiatorTest.java b/minifi/minifi-bootstrap/src/test/java/org/apache/nifi/minifi/bootstrap/configuration/differentiators/WholeConfigDifferentiatorTest.java index c12869d7e9..23591cded1 100644 --- a/minifi/minifi-bootstrap/src/test/java/org/apache/nifi/minifi/bootstrap/configuration/differentiators/WholeConfigDifferentiatorTest.java +++ b/minifi/minifi-bootstrap/src/test/java/org/apache/nifi/minifi/bootstrap/configuration/differentiators/WholeConfigDifferentiatorTest.java @@ -21,9 +21,7 @@ import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertTrue; import static org.mockito.Mockito.when; -import java.io.FileInputStream; import java.io.IOException; -import java.io.InputStream; import java.nio.ByteBuffer; import java.nio.file.Path; import java.nio.file.Paths; @@ -49,9 +47,9 @@ public class WholeConfigDifferentiatorTest { @BeforeAll public static void setConfiguration() throws IOException { dummyRequest = new Request.Builder() - .get() - .url("https://nifi.apache.org/index.html") - .build(); + .get() + .url("https://nifi.apache.org/index.html") + .build(); defaultConfigBuffer = ByteBuffer.wrap(FileUtils.readFileToByteArray(defaultConfigPath.toFile())); newConfigBuffer = ByteBuffer.wrap(FileUtils.readFileToByteArray(newConfigPath.toFile())); @@ -61,28 +59,6 @@ public class WholeConfigDifferentiatorTest { when(configurationFileHolder.getConfigFileReference()).thenReturn(new AtomicReference<>(defaultConfigBuffer)); } - // InputStream differentiator methods - - @Test - public void TestSameInputStream() throws IOException { - Differentiator differentiator = WholeConfigDifferentiator.getInputStreamDifferentiator(); - differentiator.initialize(configurationFileHolder); - - FileInputStream fileInputStream = new FileInputStream(defaultConfigPath.toFile()); - assertFalse(differentiator.isNew(fileInputStream)); - } - - @Test - public void TestNewInputStream() throws IOException { - Differentiator differentiator = WholeConfigDifferentiator.getInputStreamDifferentiator(); - differentiator.initialize(configurationFileHolder); - - FileInputStream fileInputStream = new FileInputStream(newConfigPath.toFile()); - assertTrue(differentiator.isNew(fileInputStream)); - } - - // Bytebuffer differentiator methods - @Test public void TestSameByteBuffer() throws IOException { Differentiator differentiator = WholeConfigDifferentiator.getByteBufferDifferentiator(); diff --git a/minifi/minifi-bootstrap/src/test/java/org/apache/nifi/minifi/bootstrap/configuration/ingestors/FileChangeIngestorTest.java b/minifi/minifi-bootstrap/src/test/java/org/apache/nifi/minifi/bootstrap/configuration/ingestors/FileChangeIngestorTest.java index 0fbdb03648..cc967e552c 100644 --- a/minifi/minifi-bootstrap/src/test/java/org/apache/nifi/minifi/bootstrap/configuration/ingestors/FileChangeIngestorTest.java +++ b/minifi/minifi-bootstrap/src/test/java/org/apache/nifi/minifi/bootstrap/configuration/ingestors/FileChangeIngestorTest.java @@ -5,20 +5,22 @@ * The ASF licenses this file to You under the Apache License, Version 2.0 * (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ + package org.apache.nifi.minifi.bootstrap.configuration.ingestors; import static java.nio.file.StandardWatchEventKinds.ENTRY_MODIFY; import static org.apache.nifi.minifi.bootstrap.configuration.ingestors.PullHttpChangeIngestor.PULL_HTTP_BASE_KEY; import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.mockito.Mockito.mock; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; @@ -34,6 +36,7 @@ import java.util.concurrent.atomic.AtomicReference; import org.apache.nifi.minifi.bootstrap.ConfigurationFileHolder; import org.apache.nifi.minifi.bootstrap.configuration.ConfigurationChangeNotifier; import org.apache.nifi.minifi.bootstrap.configuration.differentiators.Differentiator; +import org.apache.nifi.minifi.commons.api.MiNiFiProperties; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; @@ -50,12 +53,13 @@ public class FileChangeIngestorTest { private Differentiator mockDifferentiator; private ConfigurationChangeNotifier testNotifier; + @BeforeEach public void setUp() { - mockWatchService = Mockito.mock(WatchService.class); + mockWatchService = mock(WatchService.class); notifierSpy = Mockito.spy(new FileChangeIngestor()); - mockDifferentiator = Mockito.mock(Differentiator.class); - testNotifier = Mockito.mock(ConfigurationChangeNotifier.class); + mockDifferentiator = mock(Differentiator.class); + testNotifier = mock(ConfigurationChangeNotifier.class); setMocks(); @@ -64,6 +68,7 @@ public class FileChangeIngestorTest { testProperties.put(PullHttpChangeIngestor.OVERRIDE_SECURITY, "true"); testProperties.put(PULL_HTTP_BASE_KEY + ".override.core", "true"); testProperties.put(FileChangeIngestor.POLLING_PERIOD_INTERVAL_KEY, FileChangeIngestor.DEFAULT_POLLING_PERIOD_INTERVAL); + testProperties.put(MiNiFiProperties.NIFI_MINIFI_FLOW_CONFIG.getKey(), MiNiFiProperties.NIFI_MINIFI_FLOW_CONFIG.getDefaultValue()); } @AfterEach @@ -74,31 +79,31 @@ public class FileChangeIngestorTest { @Test public void testInitializeInvalidFile() { testProperties.put(FileChangeIngestor.CONFIG_FILE_PATH_KEY, "/land/of/make/believe"); - assertThrows(IllegalStateException.class, () -> notifierSpy.initialize(testProperties, Mockito.mock(ConfigurationFileHolder.class), Mockito.mock(ConfigurationChangeNotifier.class))); + assertThrows(IllegalStateException.class, () -> notifierSpy.initialize(testProperties, mock(ConfigurationFileHolder.class), mock(ConfigurationChangeNotifier.class))); } @Test - public void testInitializeValidFile() { - notifierSpy.initialize(testProperties, Mockito.mock(ConfigurationFileHolder.class), Mockito.mock(ConfigurationChangeNotifier.class)); + public void testInitializeValidFile() { + notifierSpy.initialize(testProperties, mock(ConfigurationFileHolder.class), mock(ConfigurationChangeNotifier.class)); } @Test public void testInitializeInvalidPollingPeriod() { testProperties.put(FileChangeIngestor.POLLING_PERIOD_INTERVAL_KEY, "abc"); - assertThrows(IllegalStateException.class, () -> notifierSpy.initialize(testProperties, Mockito.mock(ConfigurationFileHolder.class), Mockito.mock(ConfigurationChangeNotifier.class))); + assertThrows(IllegalStateException.class, () -> notifierSpy.initialize(testProperties, mock(ConfigurationFileHolder.class), mock(ConfigurationChangeNotifier.class))); } @Test public void testInitializeUseDefaultPolling() { testProperties.remove(FileChangeIngestor.POLLING_PERIOD_INTERVAL_KEY); - notifierSpy.initialize(testProperties, Mockito.mock(ConfigurationFileHolder.class), Mockito.mock(ConfigurationChangeNotifier.class)); + notifierSpy.initialize(testProperties, mock(ConfigurationFileHolder.class), mock(ConfigurationChangeNotifier.class)); } /* Verify handleChange events */ @Test public void testTargetChangedNoModification() throws Exception { when(mockDifferentiator.isNew(Mockito.any(ByteBuffer.class))).thenReturn(false); - final ConfigurationChangeNotifier testNotifier = Mockito.mock(ConfigurationChangeNotifier.class); + final ConfigurationChangeNotifier testNotifier = mock(ConfigurationChangeNotifier.class); // In this case the WatchKey is null because there were no events found establishMockEnvironmentForChangeTests(null); @@ -109,14 +114,14 @@ public class FileChangeIngestorTest { @Test public void testTargetChangedWithModificationEventNonConfigFile() throws Exception { when(mockDifferentiator.isNew(Mockito.any(ByteBuffer.class))).thenReturn(false); - final ConfigurationChangeNotifier testNotifier = Mockito.mock(ConfigurationChangeNotifier.class); + final ConfigurationChangeNotifier testNotifier = mock(ConfigurationChangeNotifier.class); // In this case, we receive a trigger event for the directory monitored, but it was another file not being monitored final WatchKey mockWatchKey = createMockWatchKeyForPath("footage_not_found.yml"); establishMockEnvironmentForChangeTests(mockWatchKey); - notifierSpy.targetChanged(); + notifierSpy.targetFileChanged(); verify(testNotifier, Mockito.never()).notifyListeners(Mockito.any(ByteBuffer.class)); } @@ -129,8 +134,9 @@ public class FileChangeIngestorTest { // Provided as a spy to allow injection of mock objects for some tests when dealing with the finalized FileSystems class establishMockEnvironmentForChangeTests(mockWatchKey); - ConfigurationFileHolder configurationFileHolder = Mockito.mock(ConfigurationFileHolder.class); + ConfigurationFileHolder configurationFileHolder = mock(ConfigurationFileHolder.class); when(configurationFileHolder.getConfigFileReference()).thenReturn(new AtomicReference<>(ByteBuffer.wrap(new byte[0]))); + notifierSpy.initialize(testProperties, configurationFileHolder, testNotifier); setMocks(); @@ -143,8 +149,8 @@ public class FileChangeIngestorTest { /* Helper methods to establish mock environment */ private WatchKey createMockWatchKeyForPath(String configFilePath) { - WatchKey mockWatchKey = Mockito.mock(WatchKey.class); - WatchEvent mockWatchEvent = Mockito.mock(WatchEvent.class); + WatchKey mockWatchKey = mock(WatchKey.class); + WatchEvent mockWatchEvent = mock(WatchEvent.class); // In this case, we receive a trigger event for the directory monitored, and it was the file monitored when(mockWatchEvent.context()).thenReturn(Paths.get(configFilePath)); @@ -157,8 +163,8 @@ public class FileChangeIngestorTest { private void establishMockEnvironmentForChangeTests(final WatchKey watchKey) { // Establish the file mock and its parent directory - final Path mockConfigFilePath = Mockito.mock(Path.class); - final Path mockConfigFileParentPath = Mockito.mock(Path.class); + final Path mockConfigFilePath = mock(Path.class); + final Path mockConfigFileParentPath = mock(Path.class); // When getting the parent of the file, get the directory when(mockConfigFilePath.getParent()).thenReturn(mockConfigFileParentPath); diff --git a/minifi/minifi-bootstrap/src/test/java/org/apache/nifi/minifi/bootstrap/configuration/ingestors/common/PullHttpChangeIngestorCommonTest.java b/minifi/minifi-bootstrap/src/test/java/org/apache/nifi/minifi/bootstrap/configuration/ingestors/PullHttpChangeIngestorCommonTest.java similarity index 70% rename from minifi/minifi-bootstrap/src/test/java/org/apache/nifi/minifi/bootstrap/configuration/ingestors/common/PullHttpChangeIngestorCommonTest.java rename to minifi/minifi-bootstrap/src/test/java/org/apache/nifi/minifi/bootstrap/configuration/ingestors/PullHttpChangeIngestorCommonTest.java index dfc0c69531..ac369f1a61 100644 --- a/minifi/minifi-bootstrap/src/test/java/org/apache/nifi/minifi/bootstrap/configuration/ingestors/common/PullHttpChangeIngestorCommonTest.java +++ b/minifi/minifi-bootstrap/src/test/java/org/apache/nifi/minifi/bootstrap/configuration/ingestors/PullHttpChangeIngestorCommonTest.java @@ -15,15 +15,12 @@ * limitations under the License. */ -package org.apache.nifi.minifi.bootstrap.configuration.ingestors.common; +package org.apache.nifi.minifi.bootstrap.configuration.ingestors; import static org.apache.nifi.minifi.bootstrap.configuration.ingestors.PullHttpChangeIngestor.PATH_KEY; -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; -import java.io.File; import java.io.IOException; import java.io.PrintWriter; import java.nio.ByteBuffer; @@ -32,18 +29,10 @@ import java.util.Collections; import java.util.Properties; import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletResponse; -import org.apache.nifi.minifi.bootstrap.RunMiNiFi; import org.apache.nifi.minifi.bootstrap.configuration.ConfigurationChangeListener; import org.apache.nifi.minifi.bootstrap.configuration.ConfigurationChangeNotifier; import org.apache.nifi.minifi.bootstrap.configuration.ListenerHandleResult; import org.apache.nifi.minifi.bootstrap.configuration.differentiators.Differentiator; -import org.apache.nifi.minifi.bootstrap.configuration.ingestors.PullHttpChangeIngestor; -import org.apache.nifi.minifi.bootstrap.util.ByteBufferInputStream; -import org.apache.nifi.minifi.commons.schema.ConfigSchema; -import org.apache.nifi.minifi.commons.schema.common.ConvertableSchema; -import org.apache.nifi.minifi.commons.schema.exception.SchemaLoaderException; -import org.apache.nifi.minifi.commons.schema.serialization.SchemaLoader; -import org.apache.nifi.util.file.FileUtils; import org.eclipse.jetty.server.Request; import org.eclipse.jetty.server.Server; import org.eclipse.jetty.server.handler.AbstractHandler; @@ -52,11 +41,11 @@ import org.eclipse.jetty.util.thread.QueuedThreadPool; import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; -import org.mockito.ArgumentCaptor; import org.mockito.Mockito; public abstract class PullHttpChangeIngestorCommonTest { + public static final String FLOW_JSON = "/flow.json"; public static volatile Server jetty; public static volatile int port; public static volatile PullHttpChangeIngestor pullHttpChangeIngestor; @@ -64,8 +53,8 @@ public abstract class PullHttpChangeIngestorCommonTest { public static Differentiator mockDifferentiator = Mockito.mock(Differentiator.class); public static final String RESPONSE_STRING = "test"; public static final String PATH_RESPONSE_STRING = "path"; - public static ByteBuffer configBuffer= ByteBuffer.wrap(RESPONSE_STRING.getBytes()); - public static ByteBuffer pathConfigBuffer= ByteBuffer.wrap(PATH_RESPONSE_STRING.getBytes()); + public static ByteBuffer configBuffer = ByteBuffer.wrap(RESPONSE_STRING.getBytes()); + public static ByteBuffer pathConfigBuffer = ByteBuffer.wrap(PATH_RESPONSE_STRING.getBytes()); public static final String ETAG = "testEtag"; public static final String QUOTED_ETAG = "\"testEtag\""; @@ -79,7 +68,7 @@ public abstract class PullHttpChangeIngestorCommonTest { jetty.setHandler(handlerCollection); } - public abstract void pullHttpChangeIngestorInit(Properties properties) throws IOException, SchemaLoaderException; + public abstract void pullHttpChangeIngestorInit(Properties properties) throws IOException; @BeforeEach public void setListeners() { @@ -95,7 +84,7 @@ public abstract class PullHttpChangeIngestorCommonTest { } @Test - public void testNewUpdate() throws IOException, SchemaLoaderException { + public void testNewUpdate() throws IOException { Properties properties = new Properties(); properties.put(PullHttpChangeIngestor.OVERRIDE_SECURITY, "true"); pullHttpChangeIngestorInit(properties); @@ -108,28 +97,7 @@ public abstract class PullHttpChangeIngestorCommonTest { } @Test - public void testSecurityOverride() throws IOException, SchemaLoaderException { - Properties properties = new Properties(); - properties.put(PullHttpChangeIngestor.OVERRIDE_SECURITY, "false"); - properties.put(RunMiNiFi.MINIFI_CONFIG_FILE_KEY, "src/test/resources/config.yml"); - properties.put(PATH_KEY, "/config-minimal.yml"); - pullHttpChangeIngestorInit(properties); - when(mockDifferentiator.isNew(Mockito.any(ByteBuffer.class))).thenReturn(true); - - pullHttpChangeIngestor.run(); - - ArgumentCaptor argument = ArgumentCaptor.forClass(ByteBuffer.class); - verify(testNotifier, Mockito.times(1)).notifyListeners(argument.capture()); - - ConvertableSchema configSchema = SchemaLoader.loadConvertableSchemaFromYaml(new ByteBufferInputStream(argument.getValue())); - ConfigSchema newSchema = configSchema.convert(); - - assertNotNull(newSchema.getSecurityProperties().getKeystore()); - assertEquals(newSchema.getProcessGroupSchema().getProcessors().size(), 2); - } - - @Test - public void testNoUpdate() throws IOException, SchemaLoaderException { + public void testNoUpdate() throws IOException { Properties properties = new Properties(); properties.put(PullHttpChangeIngestor.OVERRIDE_SECURITY, "true"); pullHttpChangeIngestorInit(properties); @@ -142,7 +110,7 @@ public abstract class PullHttpChangeIngestorCommonTest { } @Test - public void testUseEtag() throws IOException, SchemaLoaderException { + public void testUseEtag() throws IOException { Properties properties = new Properties(); properties.put(PullHttpChangeIngestor.OVERRIDE_SECURITY, "true"); pullHttpChangeIngestorInit(properties); @@ -159,13 +127,12 @@ public abstract class PullHttpChangeIngestorCommonTest { pullHttpChangeIngestor.run(); verify(testNotifier, Mockito.times(1)).notifyListeners(Mockito.any()); - } @Test - public void testNewUpdateWithPath() throws IOException, SchemaLoaderException { + public void testNewUpdateWithPath() throws IOException { Properties properties = new Properties(); - properties.put(PATH_KEY, "/config.yml"); + properties.put(PATH_KEY, FLOW_JSON); properties.put(PullHttpChangeIngestor.OVERRIDE_SECURITY, "true"); pullHttpChangeIngestorInit(properties); pullHttpChangeIngestor.setUseEtag(false); @@ -177,10 +144,10 @@ public abstract class PullHttpChangeIngestorCommonTest { } @Test - public void testNoUpdateWithPath() throws IOException, SchemaLoaderException { + public void testNoUpdateWithPath() throws IOException { Properties properties = new Properties(); properties.put(PullHttpChangeIngestor.OVERRIDE_SECURITY, "true"); - properties.put(PATH_KEY, "/config.yml"); + properties.put(PATH_KEY, FLOW_JSON); pullHttpChangeIngestorInit(properties); pullHttpChangeIngestor.setUseEtag(false); when(mockDifferentiator.isNew(Mockito.any(ByteBuffer.class))).thenReturn(false); @@ -191,10 +158,10 @@ public abstract class PullHttpChangeIngestorCommonTest { } @Test - public void testUseEtagWithPath() throws IOException, SchemaLoaderException { + public void testUseEtagWithPath() throws IOException { Properties properties = new Properties(); properties.put(PullHttpChangeIngestor.OVERRIDE_SECURITY, "true"); - properties.put(PATH_KEY, "/config.yml"); + properties.put(PATH_KEY, FLOW_JSON); pullHttpChangeIngestorInit(properties); pullHttpChangeIngestor.setLastEtag(""); @@ -209,33 +176,30 @@ public abstract class PullHttpChangeIngestorCommonTest { pullHttpChangeIngestor.run(); verify(testNotifier, Mockito.times(1)).notifyListeners(Mockito.any()); - } static class JettyHandler extends AbstractHandler { volatile String configResponse; volatile String pathResponse; - public JettyHandler(String configResponse, String pathResponse){ + public JettyHandler(String configResponse, String pathResponse) { this.configResponse = configResponse; this.pathResponse = pathResponse; } @Override public void handle(String target, Request baseRequest, HttpServletRequest request, HttpServletResponse response) - throws IOException { + throws IOException { baseRequest.setHandled(true); if ("GET".equals(request.getMethod())) { - if (QUOTED_ETAG.equals(baseRequest.getHeader("If-None-Match"))){ + if (QUOTED_ETAG.equals(baseRequest.getHeader("If-None-Match"))) { writeOutput(response, null, 304); } else { - if ("/config.yml".equals(baseRequest.getPathInfo())) { + if (FLOW_JSON.equals(baseRequest.getPathInfo())) { writeOutput(response, pathResponse, 200); - } else if ("/config-minimal.yml".equals(baseRequest.getPathInfo())) { - writeFileOutput(response, new File("src/test/resources/config-minimal.yml"), 200); } else { writeOutput(response, configResponse, 200); } @@ -260,17 +224,5 @@ public abstract class PullHttpChangeIngestorCommonTest { } } } - - private void writeFileOutput(HttpServletResponse response, File file, int responseCode) throws IOException { - response.setStatus(responseCode); - response.setHeader("ETag", ETAG); - if (file != null) { - response.setContentType("text/plain"); - response.setContentLength((int) file.length()); - response.setCharacterEncoding(StandardCharsets.UTF_8.displayName()); - FileUtils.copyFile(file, response.getOutputStream(), true, true); - } - } - } } diff --git a/minifi/minifi-bootstrap/src/test/java/org/apache/nifi/minifi/bootstrap/configuration/ingestors/PullHttpChangeIngestorSSLTest.java b/minifi/minifi-bootstrap/src/test/java/org/apache/nifi/minifi/bootstrap/configuration/ingestors/PullHttpChangeIngestorSSLTest.java index 3daa372d05..b414a67a85 100644 --- a/minifi/minifi-bootstrap/src/test/java/org/apache/nifi/minifi/bootstrap/configuration/ingestors/PullHttpChangeIngestorSSLTest.java +++ b/minifi/minifi-bootstrap/src/test/java/org/apache/nifi/minifi/bootstrap/configuration/ingestors/PullHttpChangeIngestorSSLTest.java @@ -18,18 +18,10 @@ package org.apache.nifi.minifi.bootstrap.configuration.ingestors; import static org.apache.nifi.minifi.bootstrap.configuration.ingestors.PullHttpChangeIngestor.PULL_HTTP_BASE_KEY; -import static org.mockito.Mockito.when; import java.io.IOException; -import java.io.StringWriter; -import java.nio.ByteBuffer; import java.util.Properties; -import java.util.concurrent.atomic.AtomicReference; import org.apache.nifi.minifi.bootstrap.ConfigurationFileHolder; -import org.apache.nifi.minifi.bootstrap.configuration.ingestors.common.PullHttpChangeIngestorCommonTest; -import org.apache.nifi.minifi.commons.schema.ConfigSchema; -import org.apache.nifi.minifi.commons.schema.exception.SchemaLoaderException; -import org.apache.nifi.minifi.commons.schema.serialization.SchemaLoader; import org.eclipse.jetty.server.ServerConnector; import org.eclipse.jetty.util.ssl.SslContextFactory; import org.junit.jupiter.api.BeforeAll; @@ -74,7 +66,7 @@ public class PullHttpChangeIngestorSSLTest extends PullHttpChangeIngestorCommonT } @Override - public void pullHttpChangeIngestorInit(Properties properties) throws IOException, SchemaLoaderException { + public void pullHttpChangeIngestorInit(Properties properties) throws IOException { properties.setProperty(PullHttpChangeIngestor.TRUSTSTORE_LOCATION_KEY, "./src/test/resources/localhost-ts.jks"); properties.setProperty(PullHttpChangeIngestor.TRUSTSTORE_PASSWORD_KEY, "localtest"); properties.setProperty(PullHttpChangeIngestor.TRUSTSTORE_TYPE_KEY, "JKS"); @@ -88,12 +80,6 @@ public class PullHttpChangeIngestorSSLTest extends PullHttpChangeIngestorCommonT properties.put(PULL_HTTP_BASE_KEY + ".override.core", "true"); ConfigurationFileHolder configurationFileHolder = Mockito.mock(ConfigurationFileHolder.class); - ConfigSchema configSchema = - SchemaLoader.loadConfigSchemaFromYaml(PullHttpChangeIngestorSSLTest.class.getClassLoader().getResourceAsStream("config.yml")); - StringWriter writer = new StringWriter(); - SchemaLoader.toYaml(configSchema, writer); - when(configurationFileHolder.getConfigFileReference()).thenReturn(new AtomicReference<>(ByteBuffer.wrap(writer.toString().getBytes()))); - pullHttpChangeIngestor = new PullHttpChangeIngestor(); pullHttpChangeIngestor.initialize(properties, configurationFileHolder, testNotifier); diff --git a/minifi/minifi-bootstrap/src/test/java/org/apache/nifi/minifi/bootstrap/configuration/ingestors/PullHttpChangeIngestorTest.java b/minifi/minifi-bootstrap/src/test/java/org/apache/nifi/minifi/bootstrap/configuration/ingestors/PullHttpChangeIngestorTest.java index 3833ea1425..7ef1b8af2d 100644 --- a/minifi/minifi-bootstrap/src/test/java/org/apache/nifi/minifi/bootstrap/configuration/ingestors/PullHttpChangeIngestorTest.java +++ b/minifi/minifi-bootstrap/src/test/java/org/apache/nifi/minifi/bootstrap/configuration/ingestors/PullHttpChangeIngestorTest.java @@ -18,18 +18,10 @@ package org.apache.nifi.minifi.bootstrap.configuration.ingestors; import static org.apache.nifi.minifi.bootstrap.configuration.ingestors.PullHttpChangeIngestor.PULL_HTTP_BASE_KEY; -import static org.mockito.Mockito.when; import java.io.IOException; -import java.io.StringWriter; -import java.nio.ByteBuffer; import java.util.Properties; -import java.util.concurrent.atomic.AtomicReference; import org.apache.nifi.minifi.bootstrap.ConfigurationFileHolder; -import org.apache.nifi.minifi.bootstrap.configuration.ingestors.common.PullHttpChangeIngestorCommonTest; -import org.apache.nifi.minifi.commons.schema.ConfigSchema; -import org.apache.nifi.minifi.commons.schema.exception.SchemaLoaderException; -import org.apache.nifi.minifi.commons.schema.serialization.SchemaLoader; import org.eclipse.jetty.server.ServerConnector; import org.junit.jupiter.api.BeforeAll; import org.mockito.Mockito; @@ -59,7 +51,7 @@ public class PullHttpChangeIngestorTest extends PullHttpChangeIngestorCommonTest @Override - public void pullHttpChangeIngestorInit(Properties properties) throws IOException, SchemaLoaderException { + public void pullHttpChangeIngestorInit(Properties properties) throws IOException { port = ((ServerConnector) jetty.getConnectors()[0]).getLocalPort(); properties.put(PullHttpChangeIngestor.PORT_KEY, String.valueOf(port)); properties.put(PullHttpChangeIngestor.HOST_KEY, "localhost"); @@ -67,12 +59,6 @@ public class PullHttpChangeIngestorTest extends PullHttpChangeIngestorCommonTest properties.put(PULL_HTTP_BASE_KEY + ".override.core", "true"); ConfigurationFileHolder configurationFileHolder = Mockito.mock(ConfigurationFileHolder.class); - ConfigSchema configSchema = - SchemaLoader.loadConfigSchemaFromYaml(PullHttpChangeIngestorTest.class.getClassLoader().getResourceAsStream("config.yml")); - StringWriter writer = new StringWriter(); - SchemaLoader.toYaml(configSchema, writer); - when(configurationFileHolder.getConfigFileReference()).thenReturn(new AtomicReference<>(ByteBuffer.wrap(writer.toString().getBytes()))); - pullHttpChangeIngestor = new PullHttpChangeIngestor(); pullHttpChangeIngestor.initialize(properties, configurationFileHolder, testNotifier); pullHttpChangeIngestor.setDifferentiator(mockDifferentiator); diff --git a/minifi/minifi-bootstrap/src/test/java/org/apache/nifi/minifi/bootstrap/configuration/ingestors/common/RestChangeIngestorCommonTest.java b/minifi/minifi-bootstrap/src/test/java/org/apache/nifi/minifi/bootstrap/configuration/ingestors/RestChangeIngestorCommonTest.java similarity index 84% rename from minifi/minifi-bootstrap/src/test/java/org/apache/nifi/minifi/bootstrap/configuration/ingestors/common/RestChangeIngestorCommonTest.java rename to minifi/minifi-bootstrap/src/test/java/org/apache/nifi/minifi/bootstrap/configuration/ingestors/RestChangeIngestorCommonTest.java index 50e4f21f03..ba085433ea 100644 --- a/minifi/minifi-bootstrap/src/test/java/org/apache/nifi/minifi/bootstrap/configuration/ingestors/common/RestChangeIngestorCommonTest.java +++ b/minifi/minifi-bootstrap/src/test/java/org/apache/nifi/minifi/bootstrap/configuration/ingestors/RestChangeIngestorCommonTest.java @@ -15,7 +15,7 @@ * limitations under the License. */ -package org.apache.nifi.minifi.bootstrap.configuration.ingestors.common; +package org.apache.nifi.minifi.bootstrap.configuration.ingestors; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.mockito.Mockito.verify; @@ -30,11 +30,10 @@ import okhttp3.OkHttpClient; import okhttp3.Request; import okhttp3.RequestBody; import okhttp3.Response; -import org.apache.nifi.minifi.bootstrap.configuration.differentiators.Differentiator; import org.apache.nifi.minifi.bootstrap.configuration.ConfigurationChangeListener; import org.apache.nifi.minifi.bootstrap.configuration.ConfigurationChangeNotifier; import org.apache.nifi.minifi.bootstrap.configuration.ListenerHandleResult; -import org.apache.nifi.minifi.bootstrap.configuration.ingestors.RestChangeIngestor; +import org.apache.nifi.minifi.bootstrap.configuration.differentiators.Differentiator; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.mockito.Mockito; @@ -45,7 +44,7 @@ public abstract class RestChangeIngestorCommonTest { public static OkHttpClient client; public static RestChangeIngestor restChangeIngestor; - public static final MediaType MEDIA_TYPE_MARKDOWN = MediaType.parse("text/x-markdown; charset=utf-8"); + public static final MediaType MEDIA_TYPE_MARKDOWN = MediaType.parse("text/x-markdown; charset=utf-8"); public static String url; public static ConfigurationChangeNotifier testNotifier = Mockito.mock(ConfigurationChangeNotifier.class); public static Differentiator mockDifferentiator = Mockito.mock(Differentiator.class); @@ -61,11 +60,13 @@ public abstract class RestChangeIngestorCommonTest { @Test public void testGet() throws Exception { Request request = new Request.Builder() - .url(url) - .build(); + .url(url) + .build(); try (Response response = client.newCall(request).execute()) { - if (!response.isSuccessful()) throw new IOException("Unexpected code " + response); + if (!response.isSuccessful()) { + throw new IOException("Unexpected code " + response); + } Headers responseHeaders = response.headers(); for (int i = 0; i < responseHeaders.size(); i++) { @@ -82,13 +83,15 @@ public abstract class RestChangeIngestorCommonTest { when(mockDifferentiator.isNew(Mockito.any(ByteBuffer.class))).thenReturn(true); Request request = new Request.Builder() - .url(url) - .post(RequestBody.create(MEDIA_TYPE_MARKDOWN, testString)) - .addHeader("charset","UTF-8") - .build(); + .url(url) + .post(RequestBody.create(MEDIA_TYPE_MARKDOWN, testString)) + .addHeader("charset", "UTF-8") + .build(); try (Response response = client.newCall(request).execute()) { - if (!response.isSuccessful()) throw new IOException("Unexpected code " + response); + if (!response.isSuccessful()) { + throw new IOException("Unexpected code " + response); + } Headers responseHeaders = response.headers(); for (int i = 0; i < responseHeaders.size(); i++) { @@ -106,13 +109,15 @@ public abstract class RestChangeIngestorCommonTest { when(mockDifferentiator.isNew(Mockito.any(ByteBuffer.class))).thenReturn(false); Request request = new Request.Builder() - .url(url) - .post(RequestBody.create(MEDIA_TYPE_MARKDOWN, testString)) - .addHeader("charset","UTF-8") - .build(); + .url(url) + .post(RequestBody.create(MEDIA_TYPE_MARKDOWN, testString)) + .addHeader("charset", "UTF-8") + .build(); try (Response response = client.newCall(request).execute()) { - if (response.isSuccessful()) throw new IOException("Unexpected code " + response); + if (response.isSuccessful()) { + throw new IOException("Unexpected code " + response); + } Headers responseHeaders = response.headers(); for (int i = 0; i < responseHeaders.size(); i++) { diff --git a/minifi/minifi-bootstrap/src/test/java/org/apache/nifi/minifi/bootstrap/configuration/ingestors/RestChangeIngestorSSLTest.java b/minifi/minifi-bootstrap/src/test/java/org/apache/nifi/minifi/bootstrap/configuration/ingestors/RestChangeIngestorSSLTest.java index f58f111677..2f7d644ef3 100644 --- a/minifi/minifi-bootstrap/src/test/java/org/apache/nifi/minifi/bootstrap/configuration/ingestors/RestChangeIngestorSSLTest.java +++ b/minifi/minifi-bootstrap/src/test/java/org/apache/nifi/minifi/bootstrap/configuration/ingestors/RestChangeIngestorSSLTest.java @@ -17,14 +17,24 @@ package org.apache.nifi.minifi.bootstrap.configuration.ingestors; +import static org.apache.nifi.minifi.bootstrap.configuration.ingestors.PullHttpChangeIngestor.PULL_HTTP_BASE_KEY; +import static org.mockito.Mockito.when; + +import java.io.FileInputStream; +import java.io.IOException; import java.nio.ByteBuffer; +import java.security.KeyStore; +import java.util.Collections; +import java.util.Properties; import java.util.concurrent.atomic.AtomicReference; +import javax.net.ssl.SSLContext; +import javax.net.ssl.SSLSocketFactory; +import javax.net.ssl.X509TrustManager; import okhttp3.OkHttpClient; import org.apache.nifi.minifi.bootstrap.ConfigurationFileHolder; import org.apache.nifi.minifi.bootstrap.configuration.ConfigurationChangeListener; import org.apache.nifi.minifi.bootstrap.configuration.ConfigurationChangeNotifier; import org.apache.nifi.minifi.bootstrap.configuration.ListenerHandleResult; -import org.apache.nifi.minifi.bootstrap.configuration.ingestors.common.RestChangeIngestorCommonTest; import org.apache.nifi.security.ssl.StandardKeyStoreBuilder; import org.apache.nifi.security.ssl.StandardSslContextBuilder; import org.apache.nifi.security.ssl.StandardTrustManagerBuilder; @@ -34,18 +44,6 @@ import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.BeforeAll; import org.mockito.Mockito; -import javax.net.ssl.SSLContext; -import javax.net.ssl.SSLSocketFactory; -import javax.net.ssl.X509TrustManager; -import java.io.FileInputStream; -import java.io.IOException; -import java.security.KeyStore; -import java.util.Collections; -import java.util.Properties; - -import static org.apache.nifi.minifi.bootstrap.configuration.ingestors.PullHttpChangeIngestor.PULL_HTTP_BASE_KEY; -import static org.mockito.Mockito.when; - public class RestChangeIngestorSSLTest extends RestChangeIngestorCommonTest { @BeforeAll @@ -82,27 +80,27 @@ public class RestChangeIngestorSSLTest extends RestChangeIngestorCommonTest { final KeyStore keyStore; try (final FileInputStream keyStoreStream = new FileInputStream(tlsConfiguration.getKeystorePath())) { keyStore = new StandardKeyStoreBuilder() - .type(tlsConfiguration.getKeystoreType().getType()) - .inputStream(keyStoreStream) - .password(tlsConfiguration.getKeystorePassword().toCharArray()) - .build(); + .type(tlsConfiguration.getKeystoreType().getType()) + .inputStream(keyStoreStream) + .password(tlsConfiguration.getKeystorePassword().toCharArray()) + .build(); } final KeyStore truststore; try (final FileInputStream trustStoreStream = new FileInputStream(tlsConfiguration.getTruststorePath())) { truststore = new StandardKeyStoreBuilder() - .type(tlsConfiguration.getTruststoreType().getType()) - .inputStream(trustStoreStream) - .password(tlsConfiguration.getTruststorePassword().toCharArray()) - .build(); + .type(tlsConfiguration.getTruststoreType().getType()) + .inputStream(trustStoreStream) + .password(tlsConfiguration.getTruststorePassword().toCharArray()) + .build(); } final X509TrustManager trustManager = new StandardTrustManagerBuilder().trustStore(truststore).build(); final SSLContext sslContext = new StandardSslContextBuilder() - .keyStore(keyStore) - .keyPassword(tlsConfiguration.getKeyPassword().toCharArray()) - .trustStore(truststore) - .build(); + .keyStore(keyStore) + .keyPassword(tlsConfiguration.getKeyPassword().toCharArray()) + .trustStore(truststore) + .build(); final SSLSocketFactory sslSocketFactory = sslContext.getSocketFactory(); clientBuilder.sslSocketFactory(sslSocketFactory, trustManager); diff --git a/minifi/minifi-bootstrap/src/test/java/org/apache/nifi/minifi/bootstrap/configuration/ingestors/RestChangeIngestorTest.java b/minifi/minifi-bootstrap/src/test/java/org/apache/nifi/minifi/bootstrap/configuration/ingestors/RestChangeIngestorTest.java index 65a61cb6c5..64142eccbf 100644 --- a/minifi/minifi-bootstrap/src/test/java/org/apache/nifi/minifi/bootstrap/configuration/ingestors/RestChangeIngestorTest.java +++ b/minifi/minifi-bootstrap/src/test/java/org/apache/nifi/minifi/bootstrap/configuration/ingestors/RestChangeIngestorTest.java @@ -20,19 +20,17 @@ package org.apache.nifi.minifi.bootstrap.configuration.ingestors; import static org.apache.nifi.minifi.bootstrap.configuration.ingestors.PullHttpChangeIngestor.PULL_HTTP_BASE_KEY; import static org.mockito.Mockito.when; +import java.net.MalformedURLException; import java.nio.ByteBuffer; +import java.util.Properties; import java.util.concurrent.atomic.AtomicReference; import okhttp3.OkHttpClient; import org.apache.nifi.minifi.bootstrap.ConfigurationFileHolder; import org.apache.nifi.minifi.bootstrap.configuration.ConfigurationChangeNotifier; -import org.apache.nifi.minifi.bootstrap.configuration.ingestors.common.RestChangeIngestorCommonTest; import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.BeforeAll; import org.mockito.Mockito; -import java.net.MalformedURLException; -import java.util.Properties; - public class RestChangeIngestorTest extends RestChangeIngestorCommonTest { @BeforeAll diff --git a/minifi/minifi-bootstrap/src/test/java/org/apache/nifi/minifi/bootstrap/service/BootstrapCodecTest.java b/minifi/minifi-bootstrap/src/test/java/org/apache/nifi/minifi/bootstrap/service/BootstrapCodecTest.java index 0ebb6c68f9..6304d45bf5 100644 --- a/minifi/minifi-bootstrap/src/test/java/org/apache/nifi/minifi/bootstrap/service/BootstrapCodecTest.java +++ b/minifi/minifi-bootstrap/src/test/java/org/apache/nifi/minifi/bootstrap/service/BootstrapCodecTest.java @@ -61,8 +61,6 @@ class BootstrapCodecTest { @Mock private ConfigurationChangeListener configurationChangeListener; @Mock - private UpdateConfigurationService updateConfigurationService; - @Mock private UpdatePropertiesService updatePropertiesService; @InjectMocks @@ -70,7 +68,6 @@ class BootstrapCodecTest { @BeforeEach void setup() throws IllegalAccessException, NoSuchFieldException { - mockFinal("updateConfigurationService", updateConfigurationService); mockFinal("updatePropertiesService", updatePropertiesService); } @@ -193,19 +190,6 @@ class BootstrapCodecTest { assertEquals(OK, outputStream.toString().trim()); } - @Test - void testUpdateConfigurationCommandShouldHandleUpdateConfiguration() throws IOException { - InputStream inputStream = new ByteArrayInputStream("UPDATE_CONFIGURATION".getBytes(StandardCharsets.UTF_8)); - C2Operation c2Operation = new C2Operation(); - c2Operation.setIdentifier("id"); - ByteArrayOutputStream outputStream = new ByteArrayOutputStream(); - - bootstrapCodec.communicate(inputStream, outputStream); - - assertEquals(OK, outputStream.toString().trim()); - verify(updateConfigurationService).handleUpdate(); - } - @Test void testUpdatePropertiesCommandShouldHandleUpdateProperties() throws IOException { InputStream inputStream = new ByteArrayInputStream("UPDATE_PROPERTIES".getBytes(StandardCharsets.UTF_8)); diff --git a/minifi/minifi-bootstrap/src/test/java/org/apache/nifi/minifi/bootstrap/service/MiNiFiPropertiesGeneratorTest.java b/minifi/minifi-bootstrap/src/test/java/org/apache/nifi/minifi/bootstrap/service/MiNiFiPropertiesGeneratorTest.java new file mode 100644 index 0000000000..1ee026b445 --- /dev/null +++ b/minifi/minifi-bootstrap/src/test/java/org/apache/nifi/minifi/bootstrap/service/MiNiFiPropertiesGeneratorTest.java @@ -0,0 +1,267 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.nifi.minifi.bootstrap.service; + +import static java.lang.Boolean.TRUE; +import static java.nio.file.Files.newInputStream; +import static java.nio.file.Files.newOutputStream; +import static java.util.UUID.randomUUID; +import static java.util.stream.Collectors.toList; +import static java.util.stream.Collectors.toMap; +import static java.util.stream.Stream.concat; +import static org.apache.commons.lang3.StringUtils.EMPTY; +import static org.apache.commons.lang3.StringUtils.SPACE; +import static org.apache.commons.lang3.StringUtils.isNotBlank; +import static org.apache.nifi.minifi.bootstrap.service.MiNiFiPropertiesGenerator.DEFAULT_SENSITIVE_PROPERTIES_ENCODING_ALGORITHM; +import static org.apache.nifi.minifi.bootstrap.service.MiNiFiPropertiesGenerator.MINIFI_TO_NIFI_PROPERTY_MAPPING; +import static org.apache.nifi.minifi.bootstrap.service.MiNiFiPropertiesGenerator.NIFI_PROPERTIES_WITH_DEFAULT_VALUES_AND_COMMENTS; +import static org.apache.nifi.minifi.commons.api.MiNiFiProperties.MINIFI_APP_LOG_FILE; +import static org.apache.nifi.minifi.commons.api.MiNiFiProperties.MINIFI_BOOTSTRAP_FILE_PATH; +import static org.apache.nifi.minifi.commons.api.MiNiFiProperties.MINIFI_BOOTSTRAP_LOG_FILE; +import static org.apache.nifi.minifi.commons.api.MiNiFiProperties.MINIFI_LOG_DIRECTORY; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.io.UncheckedIOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Properties; +import java.util.function.Function; +import java.util.stream.Stream; +import org.apache.commons.lang3.StringUtils; +import org.apache.nifi.minifi.bootstrap.configuration.ConfigurationChangeException; +import org.apache.nifi.minifi.commons.api.MiNiFiProperties; +import org.apache.nifi.util.NiFiProperties; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +public class MiNiFiPropertiesGeneratorTest { + + @TempDir + private Path tempDir; + + private Path configDirectory; + private Path bootstrapPropertiesFile; + private Path minifiPropertiesFile; + + private MiNiFiPropertiesGenerator testPropertiesGenerator; + + @BeforeEach + public void setup() throws IOException { + configDirectory = tempDir.toAbsolutePath().resolve("conf"); + Files.createDirectories(configDirectory); + bootstrapPropertiesFile = configDirectory.resolve("bootstrap.conf"); + minifiPropertiesFile = configDirectory.resolve("minifi.properties"); + + testPropertiesGenerator = new MiNiFiPropertiesGenerator(); + } + + @Test + public void testGenerateDefaultNiFiProperties() throws ConfigurationChangeException { + // given + Properties bootstrapProperties = createBootstrapProperties(Map.of()); + + // when + testPropertiesGenerator.generateMinifiProperties(configDirectory.toString(), bootstrapProperties); + + // then + List expectedMiNiFiProperties = NIFI_PROPERTIES_WITH_DEFAULT_VALUES_AND_COMMENTS.stream() + .map(triplet -> triplet.getLeft() + "=" + triplet.getMiddle()) + .collect(toList()); + List resultMiNiFiProperties = loadMiNiFiProperties().entrySet() + .stream() + .map(entry -> entry.getKey() + "=" + entry.getValue()) + .collect(toList()); + assertTrue(resultMiNiFiProperties.containsAll(expectedMiNiFiProperties)); + } + + @Test + public void testMiNiFiPropertiesMappedToAppropriateNiFiProperties() throws ConfigurationChangeException { + // given + Properties bootstrapProperties = createBootstrapProperties(Stream.of( + MiNiFiProperties.NIFI_MINIFI_FLOW_CONFIG.getKey(), + MiNiFiProperties.NIFI_MINIFI_SECURITY_KEYSTORE.getKey(), + MiNiFiProperties.NIFI_MINIFI_SECURITY_KEYSTORE_TYPE.getKey(), + MiNiFiProperties.NIFI_MINIFI_SECURITY_KEYSTORE_PASSWD.getKey(), + MiNiFiProperties.NIFI_MINIFI_SECURITY_KEY_PASSWD.getKey(), + MiNiFiProperties.NIFI_MINIFI_SECURITY_TRUSTSTORE.getKey(), + MiNiFiProperties.NIFI_MINIFI_SECURITY_TRUSTSTORE_TYPE.getKey(), + MiNiFiProperties.NIFI_MINIFI_SECURITY_TRUSTSTORE_PASSWD.getKey()) + .collect(toMap(Function.identity(), __ -> randomUUID().toString())) + ); + + // when + testPropertiesGenerator.generateMinifiProperties(configDirectory.toString(), bootstrapProperties); + + // then + Properties miNiFiProperties = loadMiNiFiProperties(); + assertTrue( + MINIFI_TO_NIFI_PROPERTY_MAPPING.entrySet().stream() + .allMatch(entry -> Objects.equals(bootstrapProperties.getProperty(entry.getKey()), miNiFiProperties.getProperty(entry.getValue())))); + } + + @Test + public void testSensitivePropertiesAreGeneratedWhenNotProvidedInBootstrap() throws ConfigurationChangeException { + // given + Properties bootstrapProperties = createBootstrapProperties(Map.of()); + + // when + testPropertiesGenerator.generateMinifiProperties(configDirectory.toString(), bootstrapProperties); + + // then + Properties miNiFiProperties = loadMiNiFiProperties(); + assertTrue(isNotBlank(miNiFiProperties.getProperty(NiFiProperties.SENSITIVE_PROPS_KEY))); + assertEquals(DEFAULT_SENSITIVE_PROPERTIES_ENCODING_ALGORITHM, miNiFiProperties.getProperty(NiFiProperties.SENSITIVE_PROPS_ALGORITHM)); + } + + @Test + public void testSensitivePropertiesAreUsedWhenProvidedInBootstrap() throws ConfigurationChangeException { + // given + String sensitivePropertiesKey = "sensitive_properties_key"; + String sensitivePropertiesAlgorithm = "sensitive_properties_algorithm"; + Properties bootstrapProperties = createBootstrapProperties(Map.of( + MiNiFiProperties.NIFI_MINIFI_SENSITIVE_PROPS_KEY.getKey(), sensitivePropertiesKey, + MiNiFiProperties.NIFI_MINIFI_SENSITIVE_PROPS_ALGORITHM.getKey(), sensitivePropertiesAlgorithm + )); + + // when + testPropertiesGenerator.generateMinifiProperties(configDirectory.toString(), bootstrapProperties); + + // then + Properties miNiFiProperties = loadMiNiFiProperties(); + assertEquals(sensitivePropertiesKey, miNiFiProperties.getProperty(NiFiProperties.SENSITIVE_PROPS_KEY)); + assertEquals(sensitivePropertiesAlgorithm, miNiFiProperties.getProperty(NiFiProperties.SENSITIVE_PROPS_ALGORITHM)); + } + + @Test + public void testNonBlankC2PropertiesAreCopiedToMiNiFiProperties() throws ConfigurationChangeException { + // given + Properties bootstrapProperties = createBootstrapProperties(Map.of( + MiNiFiProperties.C2_ENABLE.getKey(), TRUE.toString(), + MiNiFiProperties.C2_AGENT_CLASS.getKey(), EMPTY, + MiNiFiProperties.C2_AGENT_IDENTIFIER.getKey(), SPACE + )); + + // when + testPropertiesGenerator.generateMinifiProperties(configDirectory.toString(), bootstrapProperties); + + // then + Properties miNiFiProperties = loadMiNiFiProperties(); + assertEquals(TRUE.toString(), miNiFiProperties.getProperty(MiNiFiProperties.C2_ENABLE.getKey())); + assertNull(miNiFiProperties.getProperty(MiNiFiProperties.C2_AGENT_CLASS.getKey())); + assertNull(miNiFiProperties.getProperty(MiNiFiProperties.C2_AGENT_IDENTIFIER.getKey())); + assertNull(miNiFiProperties.getProperty(MiNiFiProperties.C2_REST_URL.getKey())); + } + + @Test + public void testDefaultNiFiPropertiesAreOverridden() throws ConfigurationChangeException { + // given + String archiveDir = "/path/to"; + Properties bootstrapProperties = createBootstrapProperties(Map.of( + NiFiProperties.FLOW_CONFIGURATION_ARCHIVE_ENABLED, TRUE.toString(), + NiFiProperties.FLOW_CONFIGURATION_ARCHIVE_DIR, archiveDir + )); + + // when + testPropertiesGenerator.generateMinifiProperties(configDirectory.toString(), bootstrapProperties); + + // then + List expectedMiNiFiProperties = concat( + NIFI_PROPERTIES_WITH_DEFAULT_VALUES_AND_COMMENTS.stream() + .filter(triplet -> + !triplet.getLeft().equals(NiFiProperties.FLOW_CONFIGURATION_ARCHIVE_ENABLED) + && !triplet.getLeft().equals(NiFiProperties.FLOW_CONFIGURATION_ARCHIVE_DIR)) + .map(triplet -> triplet.getLeft() + "=" + triplet.getMiddle()), + Stream.of( + NiFiProperties.FLOW_CONFIGURATION_ARCHIVE_ENABLED + "=" + TRUE, + NiFiProperties.FLOW_CONFIGURATION_ARCHIVE_DIR + "=" + archiveDir + )) + .collect(toList()); + List resultMiNiFiProperties = loadMiNiFiProperties().entrySet() + .stream() + .map(entry -> entry.getKey() + "=" + entry.getValue()) + .collect(toList()); + assertTrue(resultMiNiFiProperties.containsAll(expectedMiNiFiProperties)); + + } + + @Test + public void testArbitraryNiFiPropertyCanBePassedViaBootstrapConf() throws ConfigurationChangeException { + // given + Properties bootstrapProperties = createBootstrapProperties(Map.of( + "nifi.new.property", "new_property_value", + "nifi.other.new.property", "other_new_property_value" + )); + + // when + testPropertiesGenerator.generateMinifiProperties(configDirectory.toString(), bootstrapProperties); + + // then + Properties miNiFiProperties = loadMiNiFiProperties(); + assertEquals("new_property_value", miNiFiProperties.getProperty("nifi.new.property")); + assertEquals("other_new_property_value", miNiFiProperties.getProperty("nifi.other.new.property")); + + } + + @Test + public void bootstrapFileAndLogPropertiesAreGeneratedIntoMiNiFiProperties() throws ConfigurationChangeException { + // given + Properties bootstrapProperties = createBootstrapProperties(Map.of()); + + // when + testPropertiesGenerator.generateMinifiProperties(configDirectory.toString(), bootstrapProperties); + + // then + Properties miNiFiProperties = loadMiNiFiProperties(); + assertTrue(List.of(MINIFI_BOOTSTRAP_FILE_PATH, MINIFI_LOG_DIRECTORY, MINIFI_APP_LOG_FILE, MINIFI_BOOTSTRAP_LOG_FILE) + .stream() + .map(miNiFiProperties::getProperty) + .allMatch(StringUtils::isNotBlank) + ); + } + + private Properties createBootstrapProperties(Map keyValues) { + try (OutputStream outputStream = newOutputStream(bootstrapPropertiesFile)) { + Properties properties = new Properties(); + properties.putAll(keyValues); + properties.store(outputStream, EMPTY); + return properties; + } catch (IOException e) { + throw new UncheckedIOException(e); + } + } + + private Properties loadMiNiFiProperties() { + try (InputStream inputStream = newInputStream(minifiPropertiesFile)) { + Properties properties = new Properties(); + properties.load(inputStream); + return properties; + } catch (IOException e) { + throw new UncheckedIOException(e); + } + } +} + diff --git a/minifi/minifi-bootstrap/src/test/java/org/apache/nifi/minifi/bootstrap/status/reporters/StatusLoggerTest.java b/minifi/minifi-bootstrap/src/test/java/org/apache/nifi/minifi/bootstrap/status/reporters/StatusLoggerTest.java index f47e2c2c83..918edacee7 100644 --- a/minifi/minifi-bootstrap/src/test/java/org/apache/nifi/minifi/bootstrap/status/reporters/StatusLoggerTest.java +++ b/minifi/minifi-bootstrap/src/test/java/org/apache/nifi/minifi/bootstrap/status/reporters/StatusLoggerTest.java @@ -17,6 +17,18 @@ package org.apache.nifi.minifi.bootstrap.status.reporters; +import static org.apache.nifi.minifi.bootstrap.status.reporters.StatusLogger.ENCOUNTERED_IO_EXCEPTION; +import static org.apache.nifi.minifi.commons.api.MiNiFiProperties.NIFI_MINIFI_STATUS_REPORTER_LOG_LEVEL; +import static org.apache.nifi.minifi.commons.api.MiNiFiProperties.NIFI_MINIFI_STATUS_REPORTER_LOG_PERIOD; +import static org.apache.nifi.minifi.commons.api.MiNiFiProperties.NIFI_MINIFI_STATUS_REPORTER_LOG_QUERY; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.mockito.Mockito.verify; + +import java.io.IOException; +import java.util.Properties; +import java.util.concurrent.ScheduledFuture; +import java.util.concurrent.ScheduledThreadPoolExecutor; +import java.util.concurrent.TimeUnit; import org.apache.nifi.logging.LogLevel; import org.apache.nifi.minifi.bootstrap.QueryableStatusAggregator; import org.apache.nifi.minifi.commons.status.FlowStatusReport; @@ -25,22 +37,10 @@ import org.junit.jupiter.api.Test; import org.mockito.Mockito; import org.slf4j.Logger; -import java.io.IOException; -import java.util.Properties; -import java.util.concurrent.ScheduledFuture; -import java.util.concurrent.ScheduledThreadPoolExecutor; -import java.util.concurrent.TimeUnit; - -import static org.apache.nifi.minifi.bootstrap.status.reporters.StatusLogger.ENCOUNTERED_IO_EXCEPTION; -import static org.apache.nifi.minifi.bootstrap.status.reporters.StatusLogger.LOGGING_LEVEL_KEY; -import static org.apache.nifi.minifi.bootstrap.status.reporters.StatusLogger.QUERY_KEY; -import static org.apache.nifi.minifi.bootstrap.status.reporters.StatusLogger.REPORT_PERIOD_KEY; -import static org.junit.jupiter.api.Assertions.assertThrows; -import static org.mockito.Mockito.verify; - public class StatusLoggerTest { - private static final String MOCK_STATUS = "FlowStatusReport{controllerServiceStatusList=null, processorStatusList=[{name='TailFile', processorHealth={runStatus='Running', hasBulletins=false, " + + private static final String MOCK_STATUS = + "FlowStatusReport{controllerServiceStatusList=null, processorStatusList=[{name='TailFile', processorHealth={runStatus='Running', hasBulletins=false, " + "validationErrorList=[]}, processorStats=null, bulletinList=null}], connectionStatusList=null, remoteProcessingGroupStatusList=null, instanceStatus=null, systemDiagnosticsStatus=null," + " reportingTaskStatusList=null, errorsGeneratingReport=[]}"; @@ -69,9 +69,9 @@ public class StatusLoggerTest { @Test public void testFailedInitDueToFatalLogLevel() { Properties properties = new Properties(); - properties.setProperty(REPORT_PERIOD_KEY, "1"); - properties.setProperty(LOGGING_LEVEL_KEY, LogLevel.FATAL.name()); - properties.setProperty(QUERY_KEY, MOCK_QUERY); + properties.setProperty(NIFI_MINIFI_STATUS_REPORTER_LOG_PERIOD.getKey(), "1"); + properties.setProperty(NIFI_MINIFI_STATUS_REPORTER_LOG_LEVEL.getKey(), LogLevel.FATAL.name()); + properties.setProperty(NIFI_MINIFI_STATUS_REPORTER_LOG_QUERY.getKey(), MOCK_QUERY); assertThrows(IllegalStateException.class, () -> statusLogger.initialize(properties, queryableStatusAggregator)); } @@ -79,8 +79,8 @@ public class StatusLoggerTest { @Test public void testFailedInitDueToNoPeriod() { Properties properties = new Properties(); - properties.setProperty(LOGGING_LEVEL_KEY, LogLevel.INFO.name()); - properties.setProperty(QUERY_KEY, MOCK_QUERY); + properties.setProperty(NIFI_MINIFI_STATUS_REPORTER_LOG_LEVEL.getKey(), LogLevel.INFO.name()); + properties.setProperty(NIFI_MINIFI_STATUS_REPORTER_LOG_QUERY.getKey(), MOCK_QUERY); assertThrows(IllegalStateException.class, () -> statusLogger.initialize(properties, queryableStatusAggregator)); } @@ -88,8 +88,8 @@ public class StatusLoggerTest { @Test public void testFailedInitDueToNoQuery() { Properties properties = new Properties(); - properties.setProperty(REPORT_PERIOD_KEY, "1"); - properties.setProperty(LOGGING_LEVEL_KEY, LogLevel.INFO.name()); + properties.setProperty(NIFI_MINIFI_STATUS_REPORTER_LOG_PERIOD.getKey(), "1"); + properties.setProperty(NIFI_MINIFI_STATUS_REPORTER_LOG_LEVEL.getKey(), LogLevel.INFO.name()); assertThrows(IllegalStateException.class, () -> statusLogger.initialize(properties, queryableStatusAggregator)); } @@ -143,9 +143,9 @@ public class StatusLoggerTest { @Test public void testTraceException() throws IOException { Properties properties = new Properties(); - properties.setProperty(REPORT_PERIOD_KEY, "1"); - properties.setProperty(LOGGING_LEVEL_KEY, LogLevel.TRACE.name()); - properties.setProperty(QUERY_KEY, MOCK_QUERY); + properties.setProperty(NIFI_MINIFI_STATUS_REPORTER_LOG_PERIOD.getKey(), "1"); + properties.setProperty(NIFI_MINIFI_STATUS_REPORTER_LOG_LEVEL.getKey(), LogLevel.TRACE.name()); + properties.setProperty(NIFI_MINIFI_STATUS_REPORTER_LOG_QUERY.getKey(), MOCK_QUERY); IOException ioException = new IOException("This is an expected test exception"); Mockito.when(queryableStatusAggregator.statusReport(MOCK_QUERY)).thenThrow(ioException); @@ -207,9 +207,9 @@ public class StatusLoggerTest { private static Properties getProperties(LogLevel logLevel) { Properties properties = new Properties(); - properties.setProperty(REPORT_PERIOD_KEY, "1"); - properties.setProperty(LOGGING_LEVEL_KEY, logLevel.name()); - properties.setProperty(QUERY_KEY, MOCK_QUERY); + properties.setProperty(NIFI_MINIFI_STATUS_REPORTER_LOG_PERIOD.getKey(), "1"); + properties.setProperty(NIFI_MINIFI_STATUS_REPORTER_LOG_LEVEL.getKey(), logLevel.name()); + properties.setProperty(NIFI_MINIFI_STATUS_REPORTER_LOG_QUERY.getKey(), MOCK_QUERY); return properties; } diff --git a/minifi/minifi-bootstrap/src/test/java/org/apache/nifi/minifi/bootstrap/util/ConfigTransformerTest.java b/minifi/minifi-bootstrap/src/test/java/org/apache/nifi/minifi/bootstrap/util/ConfigTransformerTest.java deleted file mode 100644 index d25c98d6a1..0000000000 --- a/minifi/minifi-bootstrap/src/test/java/org/apache/nifi/minifi/bootstrap/util/ConfigTransformerTest.java +++ /dev/null @@ -1,809 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.nifi.minifi.bootstrap.util; - -import org.apache.commons.io.FileUtils; -import org.apache.nifi.minifi.bootstrap.configuration.ConfigurationChangeException; -import org.apache.nifi.minifi.bootstrap.exception.InvalidConfigurationException; -import org.apache.nifi.minifi.commons.schema.ConfigSchema; -import org.apache.nifi.minifi.commons.schema.ConnectionSchema; -import org.apache.nifi.minifi.commons.schema.ControllerServiceSchema; -import org.apache.nifi.minifi.commons.schema.FunnelSchema; -import org.apache.nifi.minifi.commons.schema.PortSchema; -import org.apache.nifi.minifi.commons.schema.ProcessGroupSchema; -import org.apache.nifi.minifi.commons.schema.ProcessorSchema; -import org.apache.nifi.minifi.commons.schema.RemotePortSchema; -import org.apache.nifi.minifi.commons.schema.RemoteProcessGroupSchema; -import org.apache.nifi.minifi.commons.schema.ReportingSchema; -import org.apache.nifi.minifi.commons.schema.common.StringUtil; -import org.apache.nifi.minifi.commons.schema.exception.SchemaLoaderException; -import org.apache.nifi.minifi.commons.schema.serialization.SchemaLoader; -import org.apache.nifi.util.StringUtils; -import org.apache.nifi.xml.processing.parsers.DocumentProvider; -import org.apache.nifi.xml.processing.parsers.StandardDocumentProvider; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; -import org.w3c.dom.Document; -import org.w3c.dom.Element; -import org.w3c.dom.NodeList; - -import javax.xml.xpath.XPath; -import javax.xml.xpath.XPathConstants; -import javax.xml.xpath.XPathExpressionException; -import javax.xml.xpath.XPathFactory; -import java.io.ByteArrayInputStream; -import java.io.ByteArrayOutputStream; -import java.io.File; -import java.io.FileInputStream; -import java.io.IOException; -import java.io.InputStream; -import java.nio.charset.Charset; -import java.util.Arrays; -import java.util.Collections; -import java.util.Comparator; -import java.util.HashMap; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.Properties; -import java.util.Set; -import java.util.stream.Collectors; -import java.util.zip.GZIPInputStream; - -import static org.apache.nifi.minifi.bootstrap.util.ConfigTransformer.MINIFI_APP_LOG_FILE; -import static org.apache.nifi.minifi.bootstrap.util.ConfigTransformer.MINIFI_BOOTSTRAP_FILE_PATH; -import static org.apache.nifi.minifi.bootstrap.util.ConfigTransformer.MINIFI_BOOTSTRAP_LOG_FILE; -import static org.apache.nifi.minifi.bootstrap.util.ConfigTransformer.MINIFI_CONFIG_FILE_PATH; -import static org.apache.nifi.minifi.bootstrap.util.ConfigTransformer.MINIFI_LOG_DIRECTORY; -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertNotEquals; -import static org.junit.jupiter.api.Assertions.assertNull; -import static org.junit.jupiter.api.Assertions.assertTrue; -import static org.junit.jupiter.api.Assertions.fail; - -public class ConfigTransformerTest { - public static final Map PG_ELEMENT_ORDER_MAP = generateOrderMap( - Arrays.asList("processor", "inputPort", "outputPort", "funnel", "processGroup", "remoteProcessGroup", "connection")); - private XPathFactory xPathFactory; - private Element config; - - @BeforeEach - public void setup() { - final DocumentProvider documentProvider = new StandardDocumentProvider(); - final Document document = documentProvider.newDocument(); - config = document.createElement("config"); - xPathFactory = XPathFactory.newInstance(); - } - - @Test - public void testNullQueuePrioritizerNotWritten() throws ConfigurationChangeException, XPathExpressionException { - ConfigTransformer.addConnection(config, new ConnectionSchema(Collections.emptyMap()), new ParentGroupIdResolver(new ProcessGroupSchema(Collections.emptyMap(), ConfigSchema.TOP_LEVEL_NAME))); - XPath xpath = xPathFactory.newXPath(); - String expression = "connection/queuePrioritizerClass"; - assertNull(xpath.evaluate(expression, config, XPathConstants.NODE)); - } - - @Test - public void testEmptyQueuePrioritizerNotWritten() throws ConfigurationChangeException, XPathExpressionException { - Map map = new HashMap<>(); - map.put(ConnectionSchema.QUEUE_PRIORITIZER_CLASS_KEY, ""); - - ConfigTransformer.addConnection(config, new ConnectionSchema(map), new ParentGroupIdResolver(new ProcessGroupSchema(Collections.emptyMap(), ConfigSchema.TOP_LEVEL_NAME))); - XPath xpath = xPathFactory.newXPath(); - String expression = "connection/queuePrioritizerClass"; - assertNull(xpath.evaluate(expression, config, XPathConstants.NODE)); - } - - @Test - public void testQueuePrioritizerWritten() throws ConfigurationChangeException, XPathExpressionException { - Map map = new HashMap<>(); - map.put(ConnectionSchema.QUEUE_PRIORITIZER_CLASS_KEY, "org.apache.nifi.prioritizer.FirstInFirstOutPrioritizer"); - - ConfigTransformer.addConnection(config, new ConnectionSchema(map), new ParentGroupIdResolver(new ProcessGroupSchema(Collections.emptyMap(), ConfigSchema.TOP_LEVEL_NAME))); - XPath xpath = xPathFactory.newXPath(); - String expression = "connection/queuePrioritizerClass/text()"; - assertEquals("org.apache.nifi.prioritizer.FirstInFirstOutPrioritizer", xpath.evaluate(expression, config, XPathConstants.STRING)); - } - - @Test - public void testReportingTasksTransform() throws Exception { - testConfigFileTransform("config-reporting-task.yml"); - } - - @Test - public void testProcessGroupsTransform() throws Exception { - testConfigFileTransform("config-process-groups.yml"); - } - - @Test - public void testFunnelsTransform() throws Exception { - testConfigFileTransform("stress-test-framework-funnel.yml"); - } - - @Test - public void testFunnelAndRpgTransform() throws Exception { - testConfigFileTransform("config-funnel-and-rpg.yml"); - } - - @Test - public void testRpgTransform() throws Exception { - testConfigFileTransform("config-multiple-RPGs.yml"); - } - - @Test - public void testRpgProxyNoPassTransform() throws Exception { - testConfigFileTransform("InvokeHttpMiNiFiProxyNoPasswordTemplateTest.yml"); - } - - @Test - public void testRpgProxyPassTransform() throws Exception { - testConfigFileTransform("InvokeHttpMiNiFiProxyPasswordTemplateTest.yml"); - } - - @Test - public void testRpgOutputPort() throws Exception { - testConfigFileTransform("SimpleRPGToLogAttributes.yml"); - } - - @Test - public void testNifiPropertiesNoOverrides() throws IOException, ConfigurationChangeException, SchemaLoaderException { - Properties pre216Properties = new Properties(); - try (InputStream pre216PropertiesStream = ConfigTransformerTest.class.getClassLoader().getResourceAsStream("MINIFI-216/nifi.properties.before")) { - pre216Properties.load(pre216PropertiesStream); - } - ByteArrayOutputStream outputStream = new ByteArrayOutputStream(); - try (InputStream configStream = ConfigTransformerTest.class.getClassLoader().getResourceAsStream("MINIFI-216/config.yml")) { - ConfigTransformer.writeNiFiProperties(SchemaLoader.loadConfigSchemaFromYaml(configStream), outputStream); - } - Properties properties = new Properties(); - properties.load(new ByteArrayInputStream(outputStream.toByteArray())); - - for (String name : pre216Properties.stringPropertyNames()) { - assertEquals(pre216Properties.getProperty(name), properties.getProperty(name), "Property key " + name + " doesn't match."); - } - } - - @Test - public void testNifiPropertiesOverrides() throws IOException, ConfigurationChangeException, SchemaLoaderException { - Properties pre216Properties = new Properties(); - try (InputStream pre216PropertiesStream = ConfigTransformerTest.class.getClassLoader().getResourceAsStream("MINIFI-216/nifi.properties.before")) { - pre216Properties.load(pre216PropertiesStream); - } - ByteArrayOutputStream outputStream = new ByteArrayOutputStream(); - try (InputStream configStream = ConfigTransformerTest.class.getClassLoader().getResourceAsStream("MINIFI-216/configOverrides.yml")) { - ConfigSchema configSchema = SchemaLoader.loadConfigSchemaFromYaml(configStream); - assertTrue(configSchema.getNifiPropertiesOverrides().size() > 0); - for (Map.Entry entry : configSchema.getNifiPropertiesOverrides().entrySet()) { - pre216Properties.setProperty(entry.getKey(), entry.getValue()); - } - ConfigTransformer.writeNiFiProperties(configSchema, outputStream); - } - Properties properties = new Properties(); - properties.load(new ByteArrayInputStream(outputStream.toByteArray())); - - for (String name : pre216Properties.stringPropertyNames()) { - assertEquals(pre216Properties.getProperty(name), properties.getProperty(name), "Property key " + name + " doesn't match."); - } - } - - @Test - public void testNifiPropertiesVariableRegistry() throws IOException, ConfigurationChangeException, SchemaLoaderException { - Properties initialProperties = new Properties(); - try (InputStream pre216PropertiesStream = ConfigTransformerTest.class.getClassLoader().getResourceAsStream("MINIFI-277/nifi.properties")) { - initialProperties.load(pre216PropertiesStream); - } - ByteArrayOutputStream outputStream = new ByteArrayOutputStream(); - try (InputStream configStream = ConfigTransformerTest.class.getClassLoader().getResourceAsStream("MINIFI-277/config.yml")) { - ConfigSchema configSchema = SchemaLoader.loadConfigSchemaFromYaml(configStream); - ConfigTransformer.writeNiFiProperties(configSchema, outputStream); - } - Properties properties = new Properties(); - properties.load(new ByteArrayInputStream(outputStream.toByteArray())); - - for (String name : initialProperties.stringPropertyNames()) { - assertEquals(initialProperties.getProperty(name), properties.getProperty(name), "Property key " + name + " doesn't match."); - } - } - - @Test - public void doesTransformFile() throws Exception { - File inputFile = new File("./src/test/resources/config.yml"); - ConfigTransformer.transformConfigFile(new FileInputStream(inputFile), "./target/", null); - File nifiPropertiesFile = new File("./target/nifi.properties"); - - assertTrue(nifiPropertiesFile.exists()); - assertTrue(nifiPropertiesFile.canRead()); - - nifiPropertiesFile.deleteOnExit(); - - File flowXml = new File("./target/flow.xml.gz"); - assertTrue(flowXml.exists()); - assertTrue(flowXml.canRead()); - - flowXml.deleteOnExit(); - } - - @Test - public void doesTransformV1File() throws Exception { - File inputFile = new File("./src/test/resources/config-v1.yml"); - ConfigTransformer.transformConfigFile(new FileInputStream(inputFile), "./target/", null); - File nifiPropertiesFile = new File("./target/nifi.properties"); - - assertTrue(nifiPropertiesFile.exists()); - assertTrue(nifiPropertiesFile.canRead()); - - nifiPropertiesFile.deleteOnExit(); - - File flowXml = new File("./target/flow.xml.gz"); - assertTrue(flowXml.exists()); - assertTrue(flowXml.canRead()); - - flowXml.deleteOnExit(); - } - - @Test - public void doesTransformInputStream() throws Exception { - File inputFile = new File("./src/test/resources/config.yml"); - ConfigTransformer.transformConfigFile(new FileInputStream(inputFile), "./target/", null); - - File nifiPropertiesFile = new File("./target/nifi.properties"); - assertTrue(nifiPropertiesFile.exists()); - assertTrue(nifiPropertiesFile.canRead()); - - nifiPropertiesFile.deleteOnExit(); - - File flowXml = new File("./target/flow.xml.gz"); - assertTrue(flowXml.exists()); - assertTrue(flowXml.canRead()); - - flowXml.deleteOnExit(); - } - - @Test - public void doesTransformOnDefaultFile() throws Exception { - File inputFile = new File("./src/test/resources/default.yml"); - ConfigTransformer.transformConfigFile(new FileInputStream(inputFile), "./target/", null); - File nifiPropertiesFile = new File("./target/nifi.properties"); - - assertTrue(nifiPropertiesFile.exists()); - assertTrue(nifiPropertiesFile.canRead()); - - nifiPropertiesFile.deleteOnExit(); - - File flowXml = new File("./target/flow.xml.gz"); - assertTrue(flowXml.exists()); - assertTrue(flowXml.canRead()); - - flowXml.deleteOnExit(); - } - - @Test - public void doesTransformOnMultipleProcessors() throws Exception { - File inputFile = new File("./src/test/resources/config-multiple-processors.yml"); - ConfigTransformer.transformConfigFile(new FileInputStream(inputFile), "./target/", null); - File nifiPropertiesFile = new File("./target/nifi.properties"); - - assertTrue(nifiPropertiesFile.exists()); - assertTrue(nifiPropertiesFile.canRead()); - - nifiPropertiesFile.deleteOnExit(); - - File flowXml = new File("./target/flow.xml.gz"); - assertTrue(flowXml.exists()); - assertTrue(flowXml.canRead()); - - flowXml.deleteOnExit(); - } - - @Test - public void doesTransformOnMultipleRemoteProcessGroups() throws Exception { - File inputFile = new File("./src/test/resources/config-multiple-RPGs.yml"); - ConfigTransformer.transformConfigFile(new FileInputStream(inputFile), "./target/", null); - File nifiPropertiesFile = new File("./target/nifi.properties"); - - assertTrue(nifiPropertiesFile.exists()); - assertTrue(nifiPropertiesFile.canRead()); - - nifiPropertiesFile.deleteOnExit(); - - File flowXml = new File("./target/flow.xml.gz"); - assertTrue(flowXml.exists()); - assertTrue(flowXml.canRead()); - - flowXml.deleteOnExit(); - } - - @Test - public void doesTransformOnMultipleInputPorts() throws Exception { - File inputFile = new File("./src/test/resources/config-multiple-input-ports.yml"); - ConfigTransformer.transformConfigFile(new FileInputStream(inputFile), "./target/", null); - File nifiPropertiesFile = new File("./target/nifi.properties"); - - assertTrue(nifiPropertiesFile.exists()); - assertTrue(nifiPropertiesFile.canRead()); - - nifiPropertiesFile.deleteOnExit(); - - File flowXml = new File("./target/flow.xml.gz"); - assertTrue(flowXml.exists()); - assertTrue(flowXml.canRead()); - - flowXml.deleteOnExit(); - } - - @Test - public void doesTransformOnMinimal() throws Exception { - File inputFile = new File("./src/test/resources/config-minimal.yml"); - ConfigTransformer.transformConfigFile(new FileInputStream(inputFile), "./target/", null); - File nifiPropertiesFile = new File("./target/nifi.properties"); - - assertTrue(nifiPropertiesFile.exists()); - assertTrue(nifiPropertiesFile.canRead()); - - nifiPropertiesFile.deleteOnExit(); - - File flowXml = new File("./target/flow.xml.gz"); - assertTrue(flowXml.exists()); - assertTrue(flowXml.canRead()); - - flowXml.deleteOnExit(); - } - - @Test - public void doesTransformOnProvenanceRepository() throws Exception { - File inputFile = new File("./src/test/resources/config-provenance-repository.yml"); - ConfigTransformer.transformConfigFile(new FileInputStream(inputFile), "./target/", null); - File nifiPropertiesFile = new File("./target/nifi.properties"); - - assertTrue(nifiPropertiesFile.exists()); - assertTrue(nifiPropertiesFile.canRead()); - - String nifi = FileUtils.readFileToString(nifiPropertiesFile, Charset.defaultCharset()); - assertTrue(nifi.contains("nifi.provenance.repository.implementation=org.apache.nifi.provenance.WriteAheadProvenanceRepository")); - - nifiPropertiesFile.deleteOnExit(); - - File flowXml = new File("./target/flow.xml.gz"); - assertTrue(flowXml.exists()); - assertTrue(flowXml.canRead()); - - flowXml.deleteOnExit(); - } - - @Test - public void doesTransformOnCustomProvenanceRepository() throws Exception { - File inputFile = new File("./src/test/resources/config-provenance-custom-repository.yml"); - ConfigTransformer.transformConfigFile(new FileInputStream(inputFile), "./target/", null); - File nifiPropertiesFile = new File("./target/nifi.properties"); - - assertTrue(nifiPropertiesFile.exists()); - assertTrue(nifiPropertiesFile.canRead()); - - String nifi = FileUtils.readFileToString(nifiPropertiesFile, Charset.defaultCharset()); - assertTrue(nifi.contains("nifi.provenance.repository.implementation=org.apache.nifi.provenance.CustomProvenanceRepository")); - - nifiPropertiesFile.deleteOnExit(); - - File flowXml = new File("./target/flow.xml.gz"); - assertTrue(flowXml.exists()); - assertTrue(flowXml.canRead()); - - flowXml.deleteOnExit(); - } - - @Test - public void handleTransformInvalidFile() throws Exception { - try { - File inputFile = new File("./src/test/resources/config-invalid.yml"); - ConfigTransformer.transformConfigFile(new FileInputStream(inputFile), "./target/", null); - fail("Invalid configuration file was not detected."); - } catch (SchemaLoaderException e) { - assertEquals("Provided YAML configuration is not a Map", e.getMessage()); - } - } - - @Test - public void handleTransformMalformedField() throws Exception { - try { - File inputFile = new File("./src/test/resources/config-malformed-field.yml"); - ConfigTransformer.transformConfigFile(new FileInputStream(inputFile), "./target/", null); - fail("Invalid configuration file was not detected."); - } catch (InvalidConfigurationException e) { - assertEquals("Failed to transform config file due to:['threshold' in section 'Swap' because it is found but could not be parsed as a Number]", e.getMessage()); - } - } - - @Test - public void handleTransformEmptyFile() throws Exception { - try { - File inputFile = new File("./src/test/resources/config-empty.yml"); - ConfigTransformer.transformConfigFile(new FileInputStream(inputFile), "./target/", null); - fail("Invalid configuration file was not detected."); - } catch (SchemaLoaderException e) { - assertEquals("Provided YAML configuration is not a Map", e.getMessage()); - } - } - - @Test - public void handleTransformFileMissingRequiredField() throws Exception { - try { - File inputFile = new File("./src/test/resources/config-missing-required-field.yml"); - ConfigTransformer.transformConfigFile(new FileInputStream(inputFile), "./target/", null); - fail("Invalid configuration file was not detected."); - } catch (InvalidConfigurationException e) { - assertEquals("Failed to transform config file due to:['class' in section 'Processors' because it was not found and it is required]", e.getMessage()); - } - } - - @Test - public void handleTransformFileMultipleProblems() throws Exception { - try { - File inputFile = new File("./src/test/resources/config-multiple-problems.yml"); - ConfigTransformer.transformConfigFile(new FileInputStream(inputFile), "./target/", null); - fail("Invalid configuration file was not detected."); - } catch (InvalidConfigurationException e) { - assertEquals("Failed to transform config file due to:['class' in section 'Processors' because it was not found and it is required], " + - "['scheduling strategy' in section 'Provenance Reporting' because it is not a valid scheduling strategy], " + - "['source name' in section 'Connections' because it was not found and it is required]", e.getMessage()); - } - } - - @Test - public void checkSSLOverrides() throws Exception { - File inputFile = new File("./src/test/resources/MINIFI-516/config.yml"); - final Properties bootstrapProperties = getTestBootstrapProperties("MINIFI-516/bootstrap.conf"); - ConfigTransformer.transformConfigFile(new FileInputStream(inputFile), "./target/", bootstrapProperties); - - // nifi.properties testing - File nifiPropertiesFile = new File("./target/nifi.properties"); - assertTrue(nifiPropertiesFile.exists()); - assertTrue(nifiPropertiesFile.canRead()); - - nifiPropertiesFile.deleteOnExit(); - - // flow.xml.gz testing - File flowXml = new File("./target/flow.xml.gz"); - assertTrue(flowXml.exists()); - assertTrue(flowXml.canRead()); - - final DocumentProvider documentProvider = new StandardDocumentProvider(); - final Document xml; - try (final InputStream inputStream = new GZIPInputStream(new FileInputStream(flowXml))) { - xml = documentProvider.parse(inputStream); - } - - XPath xPath = XPathFactory.newInstance().newXPath(); - String result = xPath.evaluate("/flowController/rootGroup/processor/property[name = \"SSL Context Service\"]/value/text()", xml); - - assertEquals(result, "SSL-Context-Service"); - - flowXml.deleteOnExit(); - - } - - @Test - public void checkLogAndConfigFilePathOverrides() throws Exception { - File inputFile = new File("./src/test/resources/config-minimal.yml"); - Properties bootstrapProperties = getTestBootstrapProperties("bootstrap.conf"); - ConfigTransformer.transformConfigFile(new FileInputStream(inputFile), "./target/", bootstrapProperties); - - File nifiPropertiesFile = new File("./target/nifi.properties"); - - assertTrue(nifiPropertiesFile.exists()); - assertTrue(nifiPropertiesFile.canRead()); - - Properties nifiProperties = new Properties(); - nifiProperties.load(new FileInputStream(nifiPropertiesFile)); - - // an absolute path is returned for bootstrap which is different per every environment. only testing that path ends with expected relative path - assertTrue(nifiProperties.getProperty(MINIFI_BOOTSTRAP_FILE_PATH).endsWith("bootstrap.conf")); - assertEquals("./conf/config.yml", nifiProperties.getProperty(MINIFI_CONFIG_FILE_PATH)); - assertEquals("./logs", nifiProperties.getProperty(MINIFI_LOG_DIRECTORY)); - assertEquals("minifi-app.log", nifiProperties.getProperty(MINIFI_APP_LOG_FILE)); - assertEquals("minifi-bootstrap.log", nifiProperties.getProperty(MINIFI_BOOTSTRAP_LOG_FILE)); - } - - public void testConfigFileTransform(String configFile) throws Exception { - ConfigSchema configSchema = SchemaLoader.loadConfigSchemaFromYaml(ConfigTransformerTest.class.getClassLoader().getResourceAsStream(configFile)); - - ByteArrayOutputStream outputStream = new ByteArrayOutputStream(); - ConfigTransformer.writeFlowXmlFile(configSchema, outputStream); - final DocumentProvider documentProvider = new StandardDocumentProvider(); - Document document = documentProvider.parse(new ByteArrayInputStream(outputStream.toByteArray())); - - testProcessGroup((Element) xPathFactory.newXPath().evaluate("flowController/rootGroup", document, XPathConstants.NODE), configSchema.getProcessGroupSchema()); - testReportingTasks((Element) xPathFactory.newXPath().evaluate("flowController/reportingTasks", document, XPathConstants.NODE), configSchema.getReportingTasksSchema()); - } - - private void testProcessGroup(Element element, ProcessGroupSchema processGroupSchema) throws XPathExpressionException { - assertEquals(processGroupSchema.getId(), getText(element, "id")); - assertEquals(processGroupSchema.getName(), getText(element, "name")); - assertEquals(nullToEmpty(processGroupSchema.getComment()), nullToEmpty(getText(element, "comment"))); - - checkOrderOfChildren(element, PG_ELEMENT_ORDER_MAP); - - NodeList processorElements = (NodeList) xPathFactory.newXPath().evaluate("processor", element, XPathConstants.NODESET); - assertEquals(processGroupSchema.getProcessors().size(), processorElements.getLength()); - for (int i = 0; i < processorElements.getLength(); i++) { - testProcessor((Element) processorElements.item(i), processGroupSchema.getProcessors().get(i)); - } - NodeList controllerServiceElements = (NodeList) xPathFactory.newXPath().evaluate("controllerService", element, XPathConstants.NODESET); - assertEquals(processGroupSchema.getControllerServices().size(), controllerServiceElements.getLength()); - for (int i = 0; i < controllerServiceElements.getLength(); i++) { - testControllerService((Element) controllerServiceElements.item(i), processGroupSchema.getControllerServices().get(i)); - } - - NodeList remoteProcessGroupElements = (NodeList) xPathFactory.newXPath().evaluate("remoteProcessGroup", element, XPathConstants.NODESET); - assertEquals(processGroupSchema.getRemoteProcessGroups().size(), remoteProcessGroupElements.getLength()); - for (int i = 0; i < remoteProcessGroupElements.getLength(); i++) { - testRemoteProcessGroups((Element) remoteProcessGroupElements.item(i), processGroupSchema.getRemoteProcessGroups().get(i)); - } - - NodeList funnelElements = (NodeList) xPathFactory.newXPath().evaluate("funnel", element, XPathConstants.NODESET); - assertEquals(processGroupSchema.getFunnels().size(), funnelElements.getLength()); - for (int i = 0; i < funnelElements.getLength(); i++) { - testFunnel((Element) funnelElements.item(i), processGroupSchema.getFunnels().get(i)); - } - - NodeList inputPortElements = (NodeList) xPathFactory.newXPath().evaluate("inputPort", element, XPathConstants.NODESET); - assertEquals(processGroupSchema.getInputPortSchemas().size(), inputPortElements.getLength()); - for (int i = 0; i < inputPortElements.getLength(); i++) { - testPort((Element) inputPortElements.item(i), processGroupSchema.getInputPortSchemas().get(i)); - } - - NodeList outputPortElements = (NodeList) xPathFactory.newXPath().evaluate("outputPort", element, XPathConstants.NODESET); - assertEquals(processGroupSchema.getOutputPortSchemas().size(), outputPortElements.getLength()); - for (int i = 0; i < outputPortElements.getLength(); i++) { - testPort((Element) outputPortElements.item(i), processGroupSchema.getOutputPortSchemas().get(i)); - } - - NodeList processGroupElements = (NodeList) xPathFactory.newXPath().evaluate("processGroup", element, XPathConstants.NODESET); - assertEquals(processGroupSchema.getProcessGroupSchemas().size(), processGroupElements.getLength()); - for (int i = 0; i < processGroupElements.getLength(); i++) { - testProcessGroup((Element) processGroupElements.item(i), processGroupSchema.getProcessGroupSchemas().get(i)); - } - - NodeList connectionElements = (NodeList) xPathFactory.newXPath().evaluate("connection", element, XPathConstants.NODESET); - assertEquals(processGroupSchema.getConnections().size(), connectionElements.getLength()); - for (int i = 0; i < connectionElements.getLength(); i++) { - testConnection((Element) connectionElements.item(i), processGroupSchema.getConnections().get(i)); - } - } - - private void testProcessor(Element element, ProcessorSchema processorSchema) throws XPathExpressionException { - assertEquals(processorSchema.getId(), getText(element, "id")); - assertEquals(processorSchema.getName(), getText(element, "name")); - assertEquals(processorSchema.getProcessorClass(), getText(element, "class")); - assertEquals(processorSchema.getMaxConcurrentTasks().toString(), getText(element, "maxConcurrentTasks")); - assertEquals(processorSchema.getSchedulingPeriod(), getText(element, "schedulingPeriod")); - assertEquals(processorSchema.getPenalizationPeriod(), getText(element, "penalizationPeriod")); - assertEquals(processorSchema.getYieldPeriod(), getText(element, "yieldPeriod")); - assertEquals(processorSchema.getSchedulingStrategy(), getText(element, "schedulingStrategy")); - assertEquals(processorSchema.getRunDurationNanos().toString(), getText(element, "runDurationNanos")); - assertEquals(processorSchema.getAnnotationData(), getText(element, "annotationData")); - - testProperties(element, processorSchema.getProperties()); - } - - private void testReportingTasks(Element element, List reportingSchemas) throws XPathExpressionException { - NodeList taskElements = (NodeList) xPathFactory.newXPath().evaluate("reportingTask", element, XPathConstants.NODESET); - assertEquals(reportingSchemas.size(), taskElements.getLength()); - for (int i = 0; i < taskElements.getLength(); i++) { - testReportingTask((Element) taskElements.item(i), reportingSchemas.get(i)); - } - } - - private void testReportingTask(Element element, ReportingSchema reportingSchema) throws XPathExpressionException { - assertEquals(reportingSchema.getId(), getText(element, "id")); - assertEquals(reportingSchema.getName(), getText(element, "name")); - assertEquals(reportingSchema.getComment(), getText(element, "comment")); - assertEquals(reportingSchema.getReportingClass(), getText(element, "class")); - assertEquals(reportingSchema.getSchedulingPeriod(), getText(element, "schedulingPeriod")); - assertEquals(reportingSchema.getSchedulingStrategy(), getText(element, "schedulingStrategy")); - - testProperties(element, reportingSchema.getProperties()); - } - - private void testControllerService(Element element, ControllerServiceSchema controllerServiceSchema) throws XPathExpressionException { - assertEquals(controllerServiceSchema.getId(), getText(element, "id")); - assertEquals(controllerServiceSchema.getName(), getText(element, "name")); - assertEquals(controllerServiceSchema.getServiceClass(), getText(element, "class")); - assertEquals(controllerServiceSchema.getAnnotationData(), getText(element, "annotationData")); - - testProperties(element, controllerServiceSchema.getProperties()); - } - - private void testRemoteProcessGroups(Element element, RemoteProcessGroupSchema remoteProcessingGroupSchema) throws XPathExpressionException { - assertEquals(remoteProcessingGroupSchema.getId(), getText(element, "id")); - assertEquals(remoteProcessingGroupSchema.getName(), getText(element, "name")); - assertEquals(remoteProcessingGroupSchema.getComment(), getText(element, "comment")); - assertEquals(remoteProcessingGroupSchema.getUrls(), getText(element, "url")); - assertEquals(remoteProcessingGroupSchema.getTimeout(), getText(element, "timeout")); - assertEquals(remoteProcessingGroupSchema.getYieldPeriod(), getText(element, "yieldPeriod")); - assertEquals(remoteProcessingGroupSchema.getTransportProtocol(), getText(element, "transportProtocol")); - assertEquals(remoteProcessingGroupSchema.getProxyHost(), getText(element, "proxyHost")); - String proxyPortText = getText(element, "proxyPort"); - assertEquals(remoteProcessingGroupSchema.getProxyPort(), StringUtil.isNullOrEmpty(proxyPortText) ? null : Integer.parseInt(proxyPortText)); - assertEquals(remoteProcessingGroupSchema.getProxyUser(), getText(element, "proxyUser")); - assertEquals(remoteProcessingGroupSchema.getProxyPassword(), getText(element, "proxyPassword")); - - NodeList inputPortElements = (NodeList) xPathFactory.newXPath().evaluate("inputPort", element, XPathConstants.NODESET); - assertEquals(remoteProcessingGroupSchema.getInputPorts().size(), inputPortElements.getLength()); - for (int i = 0; i < inputPortElements.getLength(); i++) { - testRemotePort((Element) inputPortElements.item(i), remoteProcessingGroupSchema.getInputPorts().get(i)); - } - - NodeList outputPortElements = (NodeList) xPathFactory.newXPath().evaluate("outputPort", element, XPathConstants.NODESET); - assertEquals(remoteProcessingGroupSchema.getOutputPorts().size(), outputPortElements.getLength()); - for (int i = 0; i < outputPortElements.getLength(); i++) { - testRemotePort((Element) outputPortElements.item(i), remoteProcessingGroupSchema.getOutputPorts().get(i)); - } - } - - private void testRemotePort(Element element, RemotePortSchema remoteInputPortSchema) throws XPathExpressionException { - assertEquals(remoteInputPortSchema.getId(), getText(element, "id")); - assertEquals(remoteInputPortSchema.getName(), getText(element, "name")); - assertEquals(remoteInputPortSchema.getComment(), getText(element, "comment")); - assertEquals(remoteInputPortSchema.getMax_concurrent_tasks().toString(), getText(element, "maxConcurrentTasks")); - assertEquals(remoteInputPortSchema.getUseCompression(), Boolean.parseBoolean(getText(element, "useCompression"))); - } - - private void testPort(Element element, PortSchema portSchema) throws XPathExpressionException { - assertEquals(portSchema.getId(), getText(element, "id")); - assertEquals(portSchema.getName(), getText(element, "name")); - assertEquals("RUNNING", getText(element, "scheduledState")); - } - - private void testFunnel(Element element, FunnelSchema funnelSchema) throws XPathExpressionException { - assertEquals(funnelSchema.getId(), getText(element, "id")); - } - - private void testConnection(Element element, ConnectionSchema connectionSchema) throws XPathExpressionException { - assertEquals(connectionSchema.getId(), getText(element, "id")); - assertEquals(connectionSchema.getName(), getText(element, "name")); - - assertEquals(connectionSchema.getSourceId(), getText(element, "sourceId")); - assertEquals(connectionSchema.getDestinationId(), getText(element, "destinationId")); - - NodeList relationshipNodes = (NodeList) xPathFactory.newXPath().evaluate("relationship", element, XPathConstants.NODESET); - Set sourceRelationships = new HashSet<>(); - for (int i = 0; i < relationshipNodes.getLength(); i++) { - String textContent = relationshipNodes.item(i).getTextContent(); - if (!StringUtil.isNullOrEmpty(textContent)) { - sourceRelationships.add(textContent); - } - } - - assertEquals(new HashSet<>(connectionSchema.getSourceRelationshipNames()), sourceRelationships); - - assertEquals(connectionSchema.getMaxWorkQueueSize().toString(), getText(element, "maxWorkQueueSize")); - assertEquals(connectionSchema.getMaxWorkQueueDataSize(), getText(element, "maxWorkQueueDataSize")); - assertEquals(connectionSchema.getFlowfileExpiration(), getText(element, "flowFileExpiration")); - assertEquals(connectionSchema.getQueuePrioritizerClass(), getText(element, "queuePrioritizerClass")); - } - - private void testProperties(Element element, Map expected) throws XPathExpressionException { - NodeList propertyElements = (NodeList) xPathFactory.newXPath().evaluate("property", element, XPathConstants.NODESET); - Map properties = new HashMap<>(); - for (int i = 0; i < propertyElements.getLength(); i++) { - Element item = (Element) propertyElements.item(i); - properties.put(getText(item, "name"), getText(item, "value")); - } - assertEquals(expected.entrySet().stream().collect(Collectors.toMap(Map.Entry::getKey, e -> nullToEmpty(e.getValue()))), properties); - } - - @Test - public void testContentRepoOverride() throws IOException, ConfigurationChangeException, SchemaLoaderException { - Properties pre216Properties = new Properties(); - try (InputStream pre216PropertiesStream = ConfigTransformerTest.class.getClassLoader().getResourceAsStream("MINIFI-245/nifi.properties.before")) { - pre216Properties.load(pre216PropertiesStream); - } - ByteArrayOutputStream outputStream = new ByteArrayOutputStream(); - try (InputStream configStream = ConfigTransformerTest.class.getClassLoader().getResourceAsStream("MINIFI-245/config.yml")) { - ConfigTransformer.writeNiFiProperties(SchemaLoader.loadConfigSchemaFromYaml(configStream), outputStream); - } - Properties properties = new Properties(); - properties.load(new ByteArrayInputStream(outputStream.toByteArray())); - - for (String name : pre216Properties.stringPropertyNames()) { - // Verify the Content Repo property was overridden - if ("nifi.content.repository.implementation".equals(name)) { - assertNotEquals(pre216Properties.getProperty(name), properties.getProperty(name), "Property key " + name + " was not overridden."); - } else { - assertEquals(pre216Properties.getProperty(name), properties.getProperty(name), "Property key " + name + " doesn't match."); - } - } - } - - @Test - public void testFlowFileRepoOverride() throws IOException, ConfigurationChangeException, SchemaLoaderException { - Properties pre216Properties = new Properties(); - try (InputStream pre216PropertiesStream = ConfigTransformerTest.class.getClassLoader().getResourceAsStream("NIFI-8753/nifi.properties.before")) { - pre216Properties.load(pre216PropertiesStream); - } - ByteArrayOutputStream outputStream = new ByteArrayOutputStream(); - try (InputStream configStream = ConfigTransformerTest.class.getClassLoader().getResourceAsStream("NIFI-8753/config.yml")) { - ConfigTransformer.writeNiFiProperties(SchemaLoader.loadConfigSchemaFromYaml(configStream), outputStream); - } - Properties properties = new Properties(); - properties.load(new ByteArrayInputStream(outputStream.toByteArray())); - - for (String name : pre216Properties.stringPropertyNames()) { - // Verify the Content Repo property was overridden - if ("nifi.flowfile.repository.implementation".equals(name)) { - assertNotEquals(pre216Properties.getProperty(name), properties.getProperty(name), "Property key " + name + " was not overridden."); - } else { - assertEquals(pre216Properties.getProperty(name), properties.getProperty(name), "Property key " + name + " doesn't match."); - } - } - } - - @Test - public void testNullSensitiveKey() throws IOException, ConfigurationChangeException, SchemaLoaderException { - final ByteArrayOutputStream outputStream = new ByteArrayOutputStream(); - try (final InputStream configStream = ConfigTransformerTest.class.getClassLoader().getResourceAsStream("MINIFI-537/config.yml")) { - ConfigTransformer.writeNiFiProperties(SchemaLoader.loadConfigSchemaFromYaml(configStream), outputStream); - } - final Properties properties = new Properties(); - properties.load(new ByteArrayInputStream(outputStream.toByteArray())); - // The property should not be empty/null as it is auto-generated when missing - assertTrue(StringUtils.isNotEmpty(properties.getProperty("nifi.sensitive.props.key"))); - } - - private String getText(Element element, String path) throws XPathExpressionException { - return (String) xPathFactory.newXPath().evaluate(path + "/text()", element, XPathConstants.STRING); - } - - private String nullToEmpty(Object val) { - return val == null ? "" : val.toString(); - } - - private static Map generateOrderMap(List elements) { - Map map = new HashMap<>(); - for (int i = 0; i < elements.size(); i++) { - map.put(elements.get(i), i); - } - return Collections.unmodifiableMap(map); - } - - private static void checkOrderOfChildren(Element element, Map orderMap) { - int elementOrderList = 0; - NodeList childNodes = element.getChildNodes(); - String lastOrderedElementName = null; - for (int i = 0; i < childNodes.getLength(); i++) { - String nodeName = childNodes.item(i).getNodeName(); - Integer index = orderMap.get(nodeName); - if (index != null) { - if (elementOrderList > index) { - fail("Found " + nodeName + " after " + lastOrderedElementName + "; expected all " + nodeName + " elements to come before the following elements: " + orderMap.entrySet().stream() - .filter(e -> e.getValue() > index).sorted(Comparator.comparingInt(Map.Entry::getValue)).map(Map.Entry::getKey).collect(Collectors.joining(", "))); - } - lastOrderedElementName = nodeName; - elementOrderList = index; - } - } - } - - public static Properties getTestBootstrapProperties(final String fileName) throws IOException { - final Properties bootstrapProperties = new Properties(); - try (final InputStream fis = ConfigTransformerTest.class.getClassLoader().getResourceAsStream(fileName)) { - bootstrapProperties.load(fis); - } - return bootstrapProperties; - } -} diff --git a/minifi/minifi-bootstrap/src/test/java/org/apache/nifi/minifi/bootstrap/util/OrderedPropertiesTest.java b/minifi/minifi-bootstrap/src/test/java/org/apache/nifi/minifi/bootstrap/util/OrderedPropertiesTest.java index ca18b8bb77..05af746ea8 100644 --- a/minifi/minifi-bootstrap/src/test/java/org/apache/nifi/minifi/bootstrap/util/OrderedPropertiesTest.java +++ b/minifi/minifi-bootstrap/src/test/java/org/apache/nifi/minifi/bootstrap/util/OrderedPropertiesTest.java @@ -17,6 +17,7 @@ package org.apache.nifi.minifi.bootstrap.util; +import org.apache.nifi.minifi.bootstrap.service.MiNiFiPropertiesGenerator; import org.junit.jupiter.api.Test; import java.io.BufferedReader; @@ -25,7 +26,6 @@ import java.io.ByteArrayOutputStream; import java.io.IOException; import java.io.InputStreamReader; -import static org.apache.nifi.minifi.bootstrap.util.ConfigTransformer.PROPERTIES_FILE_APACHE_2_0_LICENSE; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertNull; @@ -41,7 +41,7 @@ public class OrderedPropertiesTest { orderedProperties.setProperty("prop3", "newVal3"); orderedProperties.setProperty("prop1", "newVal1"); ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream(); - orderedProperties.store(byteArrayOutputStream, PROPERTIES_FILE_APACHE_2_0_LICENSE); + orderedProperties.store(byteArrayOutputStream, MiNiFiPropertiesGenerator.PROPERTIES_FILE_APACHE_2_0_LICENSE); try (BufferedReader actualReader = new BufferedReader(new InputStreamReader(new ByteArrayInputStream(byteArrayOutputStream.toByteArray()))); BufferedReader expectedReader = new BufferedReader(new InputStreamReader(OrderedPropertiesTest.class.getClassLoader().getResourceAsStream("orderedPropertiesExpected.properties")))) { diff --git a/minifi/minifi-bootstrap/src/test/java/org/apache/nifi/minifi/bootstrap/util/ParentGroupIdResolverTest.java b/minifi/minifi-bootstrap/src/test/java/org/apache/nifi/minifi/bootstrap/util/ParentGroupIdResolverTest.java deleted file mode 100644 index 4b1336c07e..0000000000 --- a/minifi/minifi-bootstrap/src/test/java/org/apache/nifi/minifi/bootstrap/util/ParentGroupIdResolverTest.java +++ /dev/null @@ -1,200 +0,0 @@ -/* - * - * * Licensed to the Apache Software Foundation (ASF) under one or more - * * contributor license agreements. See the NOTICE file distributed with - * * this work for additional information regarding copyright ownership. - * * The ASF licenses this file to You under the Apache License, Version 2.0 - * * (the "License"); you may not use this file except in compliance with - * * the License. You may obtain a copy of the License at - * * - * * http://www.apache.org/licenses/LICENSE-2.0 - * * - * * Unless required by applicable law or agreed to in writing, software - * * distributed under the License is distributed on an "AS IS" BASIS, - * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * * See the License for the specific language governing permissions and - * * limitations under the License. - * - */ - -package org.apache.nifi.minifi.bootstrap.util; - -import org.apache.nifi.minifi.commons.schema.exception.SchemaLoaderException; -import org.apache.nifi.minifi.commons.schema.serialization.SchemaLoader; -import org.junit.jupiter.api.Test; - -import java.io.ByteArrayInputStream; -import java.io.IOException; -import java.nio.charset.StandardCharsets; -import java.util.ArrayList; -import java.util.List; - -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertNull; - -public class ParentGroupIdResolverTest { - - @Test - public void testRemoteInputPortParentId() throws IOException, SchemaLoaderException { - List configLines = new ArrayList<>(); - configLines.add("MiNiFi Config Version: 2"); - configLines.add("Remote Process Groups:"); - configLines.add("- name: rpgOne"); - configLines.add(" id: rpgOneId"); - configLines.add(" Input Ports:"); - configLines.add(" - id: one"); - configLines.add("Process Groups:"); - configLines.add("- Remote Process Groups:"); - configLines.add(" - name: rpgTwo"); - configLines.add(" id: rpgTwoId"); - configLines.add(" Input Ports:"); - configLines.add(" - id: two"); - ParentGroupIdResolver parentGroupIdResolver = createParentGroupIdResolver(configLines); - assertEquals("rpgOneId", parentGroupIdResolver.getRemoteInputPortParentId("one")); - assertEquals("rpgTwoId", parentGroupIdResolver.getRemoteInputPortParentId("two")); - assertNull(parentGroupIdResolver.getRemoteOutputPortParentId("one")); - assertNull(parentGroupIdResolver.getRemoteOutputPortParentId("two")); - assertNull(parentGroupIdResolver.getInputPortParentId("one")); - assertNull(parentGroupIdResolver.getInputPortParentId("two")); - assertNull(parentGroupIdResolver.getOutputPortParentId("one")); - assertNull(parentGroupIdResolver.getOutputPortParentId("two")); - assertNull(parentGroupIdResolver.getProcessorParentId("one")); - assertNull(parentGroupIdResolver.getProcessorParentId("two")); - assertNull(parentGroupIdResolver.getFunnelParentId("one")); - assertNull(parentGroupIdResolver.getFunnelParentId("two")); - } - @Test - public void testRemoteOutputPortParentId() throws IOException, SchemaLoaderException { - List configLines = new ArrayList<>(); - configLines.add("MiNiFi Config Version: 3"); - configLines.add("Remote Process Groups:"); - configLines.add("- name: rpgOne"); - configLines.add(" id: rpgOneId"); - configLines.add(" Output Ports:"); - configLines.add(" - id: one"); - configLines.add("Process Groups:"); - configLines.add("- Remote Process Groups:"); - configLines.add(" - name: rpgTwo"); - configLines.add(" id: rpgTwoId"); - configLines.add(" Output Ports:"); - configLines.add(" - id: two"); - ParentGroupIdResolver parentGroupIdResolver = createParentGroupIdResolver(configLines); - assertNull(parentGroupIdResolver.getRemoteInputPortParentId("one")); - assertNull(parentGroupIdResolver.getRemoteInputPortParentId("two")); - assertEquals("rpgOneId", parentGroupIdResolver.getRemoteOutputPortParentId("one")); - assertEquals("rpgTwoId", parentGroupIdResolver.getRemoteOutputPortParentId("two")); - assertNull(parentGroupIdResolver.getInputPortParentId("one")); - assertNull(parentGroupIdResolver.getInputPortParentId("two")); - assertNull(parentGroupIdResolver.getOutputPortParentId("one")); - assertNull(parentGroupIdResolver.getOutputPortParentId("two")); - assertNull(parentGroupIdResolver.getProcessorParentId("one")); - assertNull(parentGroupIdResolver.getProcessorParentId("two")); - assertNull(parentGroupIdResolver.getFunnelParentId("one")); - assertNull(parentGroupIdResolver.getFunnelParentId("two")); - } - - @Test - public void testInputPortParentId() throws IOException, SchemaLoaderException { - List configLines = new ArrayList<>(); - configLines.add("MiNiFi Config Version: 2"); - configLines.add("Input Ports:"); - configLines.add("- id: one"); - configLines.add("Process Groups:"); - configLines.add("- id: pgTwo"); - configLines.add(" Input Ports:"); - configLines.add(" - id: two"); - ParentGroupIdResolver parentGroupIdResolver = createParentGroupIdResolver(configLines); - assertNull(parentGroupIdResolver.getRemoteInputPortParentId("one")); - assertNull(parentGroupIdResolver.getRemoteInputPortParentId("two")); - assertNull(parentGroupIdResolver.getRemoteOutputPortParentId("one")); - assertNull(parentGroupIdResolver.getRemoteOutputPortParentId("two")); - assertEquals(ConfigTransformer.ROOT_GROUP, parentGroupIdResolver.getInputPortParentId("one")); - assertEquals("pgTwo", parentGroupIdResolver.getInputPortParentId("two")); - assertNull(parentGroupIdResolver.getOutputPortParentId("one")); - assertNull(parentGroupIdResolver.getOutputPortParentId("two")); - assertNull(parentGroupIdResolver.getProcessorParentId("one")); - assertNull(parentGroupIdResolver.getProcessorParentId("two")); - assertNull(parentGroupIdResolver.getFunnelParentId("one")); - assertNull(parentGroupIdResolver.getFunnelParentId("two")); - } - - @Test - public void testOutputPortParentId() throws IOException, SchemaLoaderException { - List configLines = new ArrayList<>(); - configLines.add("MiNiFi Config Version: 2"); - configLines.add("Output Ports:"); - configLines.add("- id: one"); - configLines.add("Process Groups:"); - configLines.add("- id: pgTwo"); - configLines.add(" Output Ports:"); - configLines.add(" - id: two"); - ParentGroupIdResolver parentGroupIdResolver = createParentGroupIdResolver(configLines); - assertNull(parentGroupIdResolver.getRemoteInputPortParentId("one")); - assertNull(parentGroupIdResolver.getRemoteInputPortParentId("two")); - assertNull(parentGroupIdResolver.getRemoteOutputPortParentId("one")); - assertNull(parentGroupIdResolver.getRemoteOutputPortParentId("two")); - assertNull(parentGroupIdResolver.getInputPortParentId("one")); - assertNull(parentGroupIdResolver.getInputPortParentId("two")); - assertEquals(ConfigTransformer.ROOT_GROUP, parentGroupIdResolver.getOutputPortParentId("one")); - assertEquals("pgTwo", parentGroupIdResolver.getOutputPortParentId("two")); - assertNull(parentGroupIdResolver.getProcessorParentId("one")); - assertNull(parentGroupIdResolver.getProcessorParentId("two")); - assertNull(parentGroupIdResolver.getFunnelParentId("one")); - assertNull(parentGroupIdResolver.getFunnelParentId("two")); - } - - @Test - public void testProcessorParentId() throws IOException, SchemaLoaderException { - List configLines = new ArrayList<>(); - configLines.add("MiNiFi Config Version: 2"); - configLines.add("Processors:"); - configLines.add("- id: one"); - configLines.add("Process Groups:"); - configLines.add("- id: pgTwo"); - configLines.add(" Processors:"); - configLines.add(" - id: two"); - ParentGroupIdResolver parentGroupIdResolver = createParentGroupIdResolver(configLines); - assertNull(parentGroupIdResolver.getRemoteInputPortParentId("one")); - assertNull(parentGroupIdResolver.getRemoteInputPortParentId("two")); - assertNull(parentGroupIdResolver.getRemoteOutputPortParentId("one")); - assertNull(parentGroupIdResolver.getRemoteOutputPortParentId("two")); - assertNull(parentGroupIdResolver.getInputPortParentId("one")); - assertNull(parentGroupIdResolver.getInputPortParentId("two")); - assertNull(parentGroupIdResolver.getOutputPortParentId("one")); - assertNull(parentGroupIdResolver.getOutputPortParentId("two")); - assertEquals(ConfigTransformer.ROOT_GROUP, parentGroupIdResolver.getProcessorParentId("one")); - assertEquals("pgTwo", parentGroupIdResolver.getProcessorParentId("two")); - assertNull(parentGroupIdResolver.getFunnelParentId("one")); - assertNull(parentGroupIdResolver.getFunnelParentId("two")); - } - - @Test - public void testFunnelParentId() throws IOException, SchemaLoaderException { - List configLines = new ArrayList<>(); - configLines.add("MiNiFi Config Version: 2"); - configLines.add("Funnels:"); - configLines.add("- id: one"); - configLines.add("Process Groups:"); - configLines.add("- id: pgTwo"); - configLines.add(" Funnels:"); - configLines.add(" - id: two"); - ParentGroupIdResolver parentGroupIdResolver = createParentGroupIdResolver(configLines); - assertNull(parentGroupIdResolver.getRemoteInputPortParentId("one")); - assertNull(parentGroupIdResolver.getRemoteInputPortParentId("two")); - assertNull(parentGroupIdResolver.getRemoteOutputPortParentId("one")); - assertNull(parentGroupIdResolver.getRemoteOutputPortParentId("two")); - assertNull(parentGroupIdResolver.getInputPortParentId("one")); - assertNull(parentGroupIdResolver.getInputPortParentId("two")); - assertNull(parentGroupIdResolver.getOutputPortParentId("one")); - assertNull(parentGroupIdResolver.getOutputPortParentId("two")); - assertNull(parentGroupIdResolver.getProcessorParentId("one")); - assertNull(parentGroupIdResolver.getProcessorParentId("two")); - assertEquals(ConfigTransformer.ROOT_GROUP, parentGroupIdResolver.getFunnelParentId("one")); - assertEquals("pgTwo", parentGroupIdResolver.getFunnelParentId("two")); - } - - private ParentGroupIdResolver createParentGroupIdResolver(List configLines) throws IOException, SchemaLoaderException { - return new ParentGroupIdResolver(SchemaLoader.loadConfigSchemaFromYaml(new ByteArrayInputStream(String.join("\n", configLines) - .getBytes(StandardCharsets.UTF_8))).getProcessGroupSchema()); - } -} diff --git a/minifi/minifi-c2/README.md b/minifi/minifi-c2/README.md index d1b334a1d0..2d6c541b4e 100644 --- a/minifi/minifi-c2/README.md +++ b/minifi/minifi-c2/README.md @@ -38,17 +38,16 @@ When using the `CacheConfigurationProvider`, by default, the server will look fo The pattern can be configured in [./conf/minifi-c2-context.xml](./minifi-c2-assembly/src/main/resources/conf/minifi-c2-context.xml) and the default value (${class}/config) will replace ${class} with the class query parameter and then look for CLASS/config.CONTENT_TYPE.vVERSION in the directory structure. -Ex: http://localhost:10090/c2/config?class=raspi&version=1 with an Accept header that matches text/yml would result in [./files/raspi3/config.text.yml.v1](./minifi-c2-assembly/src/main/resources/files/raspi3/config.text.yml.v1) and omitting the version parameter would look for the highest version number in the directory. +Ex: http://localhost:10090/c2/config?class=raspi&version=1 with an Accept header that matches application/json would result in [./files/raspi3/config.application.json.v1](./minifi-c2-assembly/src/main/resources/files/raspi3/config.application.json.v1) and omitting the version parameter would look for the highest version number in the directory. The version resolution is cached in memory to accommodate many devices polling periodically. The cache duration can be configured with additional arguments in [./conf/minifi-c2-context.xml](../minifi-integration-tests/src/test/resources/c2/hierarchical/c2-edge2/conf/minifi-c2-context.xml#L55) that call the [overloaded constructor.](./minifi-c2-service/src/main/java/org/apache/nifi/minifi/c2/service/ConfigService.java#L81) MiNiFi Java agents can be configured to poll the C2 server by [configuring the PullHttpChangeIngestor in their bootstrap.conf.](../minifi-integration-tests/src/test/resources/c2/hierarchical/minifi-edge1/bootstrap.conf#L37) ### Configuration Providers: -There are three `ConfigurationProvider` implementations provided out of the box. +There are two `ConfigurationProvider` implementations provided out of the box. 1. The [CacheConfigurationProvider](./minifi-c2-assembly/src/main/resources/conf/minifi-c2-context.xml) looks at a directory on the filesystem or in an Amazon S3 bucket. Which backend storage is used can be selected in `minifi-c2-context.xml` via the constructors to `CacheConfigurationProvider`. 2. The [DelegatingConfigurationProvider](./minifi-c2-integration-tests/src/test/resources/c2-unsecure-delegating/conf/minifi-c2-context.xml) delegates to another C2 server to allow for hierarchical C2 structures to help with scaling and/or bridging networks. -3. The [NiFiRestConfigurationProvider](./minifi-c2-integration-tests/src/test/resources/c2-unsecure-rest/conf/minifi-c2-context.xml) pulls templates from a NiFi instance over its REST API. (Note: sensitive values are NOT included in templates so this is unsuitable for flows with sensitive configuration currently) ### Example network diagram: Below is a network diagram showing the different configurations tested by [our hierarchical integration test docker-compose file.](../minifi-integration-tests/src/test/resources/docker-compose-c2-hierarchical.yml) It consists of a "cluster" network where real processing might occur as well as 3 "edge" networks that can get configuration from the cluster network a few different ways. The edge1 instance can directly access the authoritative C2 server via HTTPS. The edge2 instance is representative of a segmented network where the MiNiFi agents can talk to a local delegating C2 server over HTTP which asks the authoritative C2 server over HTTPS. The edge 3 instance can talk to the authoritative C2 server through a Squid proxy over HTTPS. diff --git a/minifi/minifi-c2/minifi-c2-assembly/README.md b/minifi/minifi-c2/minifi-c2-assembly/README.md index 1c048441df..4150fddb5b 100644 --- a/minifi/minifi-c2/minifi-c2-assembly/README.md +++ b/minifi/minifi-c2/minifi-c2-assembly/README.md @@ -26,7 +26,7 @@ MiNiFi is a child project effort of Apache NiFi. The MiNiFi C2 Server aids in t - [Export Control](#export-control) ## Requirements -* JRE 1.8 +* JRE 17 * Apache Maven 3 ## Getting Started diff --git a/minifi/minifi-c2/minifi-c2-assembly/pom.xml b/minifi/minifi-c2/minifi-c2-assembly/pom.xml index 0fcd576976..2e47aeeb87 100644 --- a/minifi/minifi-c2/minifi-c2-assembly/pom.xml +++ b/minifi/minifi-c2/minifi-c2-assembly/pom.xml @@ -100,13 +100,23 @@ limitations under the License. org.apache.nifi.minifi - minifi-c2-provider-nifi-rest + minifi-c2-jetty ${project.version} org.apache.nifi.minifi - minifi-c2-jetty + minifi-toolkit-configuration ${project.version} + + + org.slf4j + slf4j-log4j12 + + + org.glassfish.jersey.media + jersey-media-json-jackson + + org.eclipse.jetty diff --git a/minifi/minifi-c2/minifi-c2-assembly/src/main/resources/conf/minifi-c2-context.xml b/minifi/minifi-c2/minifi-c2-assembly/src/main/resources/conf/minifi-c2-context.xml index f66d48630f..cae8dcb241 100644 --- a/minifi/minifi-c2/minifi-c2-assembly/src/main/resources/conf/minifi-c2-context.xml +++ b/minifi/minifi-c2/minifi-c2-assembly/src/main/resources/conf/minifi-c2-context.xml @@ -26,7 +26,7 @@ - text/yml + application/json @@ -62,24 +62,6 @@ --> - diff --git a/minifi/minifi-c2/minifi-c2-assembly/src/main/resources/files/raspi3/config.test.json.v1 b/minifi/minifi-c2/minifi-c2-assembly/src/main/resources/files/raspi3/config.test.json.v1 new file mode 100644 index 0000000000..cb62b83ae5 --- /dev/null +++ b/minifi/minifi-c2/minifi-c2-assembly/src/main/resources/files/raspi3/config.test.json.v1 @@ -0,0 +1,55 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the \"License\"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an \"AS IS\" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +{ + "encodingVersion": { + "majorVersion": 2, + "minorVersion": 0 + }, + "maxTimerDrivenThreadCount": 10, + "maxEventDrivenThreadCount": 1, + "registries": [], + "parameterContexts": [], + "parameterProviders": [], + "controllerServices": [], + "reportingTasks": [], + "templates": [], + "rootGroup": { + "identifier": "c1b4e586-2011-3f81-a11e-8d669f084d1c", + "instanceIdentifier": "29db3dbc-0188-1000-7025-4cab8b52d278", + "name": "MiNiFi Flow", + "comments": "", + "position": { + "x": 0.0, + "y": 0.0 + }, + "processGroups": [], + "remoteProcessGroups": [], + "processors": [], + "inputPorts": [], + "outputPorts": [], + "connections": [], + "labels": [], + "funnels": [], + "controllerServices": [], + "variables": {}, + "defaultFlowFileExpiration": "0 sec", + "defaultBackPressureObjectThreshold": 10000, + "defaultBackPressureDataSizeThreshold": "1 GB", + "componentType": "PROCESS_GROUP", + "flowFileConcurrency": "UNBOUNDED", + "flowFileOutboundPolicy": "STREAM_WHEN_AVAILABLE" + } +} \ No newline at end of file diff --git a/minifi/minifi-c2/minifi-c2-assembly/src/main/resources/files/raspi3/config.text.yml.v1 b/minifi/minifi-c2/minifi-c2-assembly/src/main/resources/files/raspi3/config.text.yml.v1 deleted file mode 100644 index 76ce1cd26c..0000000000 --- a/minifi/minifi-c2/minifi-c2-assembly/src/main/resources/files/raspi3/config.text.yml.v1 +++ /dev/null @@ -1,63 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the \"License\"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an \"AS IS\" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -MiNiFi Config Version: 3 -Flow Controller: - name: MiNiFi Flow - comment: '' -Core Properties: - flow controller graceful shutdown period: 10 sec - flow service write delay interval: 500 ms - administrative yield duration: 30 sec - bored yield duration: 10 millis - max concurrent threads: 1 -FlowFile Repository: - partitions: 256 - checkpoint interval: 2 mins - always sync: false - Swap: - threshold: 20000 - in period: 5 sec - in threads: 1 - out period: 5 sec - out threads: 4 -Content Repository: - content claim max appendable size: 10 MB - content claim max flow files: 100 - always sync: false -Provenance Repository: - provenance rollover time: 1 min -Component Status Repository: - buffer size: 1440 - snapshot frequency: 1 min -Security Properties: - keystore: '' - keystore type: '' - keystore password: '' - key password: '' - truststore: '' - truststore type: '' - truststore password: '' - ssl protocol: '' - Sensitive Props: - key: '' - algorithm: NIFI_PBKDF2_AES_GCM_256 - provider: BC -Processors: [] -Process Groups: [] -Funnels: [] -Connections: [] -Remote Process Groups: [] -NiFi Properties Overrides: {} diff --git a/minifi/minifi-c2/minifi-c2-cache/minifi-c2-cache-filesystem/src/test/java/org/apache/nifi/minfi/c2/cache/filesystem/FileSystemConfigurationCacheTest.java b/minifi/minifi-c2/minifi-c2-cache/minifi-c2-cache-filesystem/src/test/java/org/apache/nifi/minfi/c2/cache/filesystem/FileSystemConfigurationCacheTest.java index e773c49491..d731545a54 100644 --- a/minifi/minifi-c2/minifi-c2-cache/minifi-c2-cache-filesystem/src/test/java/org/apache/nifi/minfi/c2/cache/filesystem/FileSystemConfigurationCacheTest.java +++ b/minifi/minifi-c2/minifi-c2-cache/minifi-c2-cache-filesystem/src/test/java/org/apache/nifi/minfi/c2/cache/filesystem/FileSystemConfigurationCacheTest.java @@ -36,7 +36,10 @@ import org.apache.nifi.minifi.c2.cache.filesystem.FileSystemConfigurationCache; import org.junit.jupiter.api.Test; public class FileSystemConfigurationCacheTest { + private static final String PATH_ROOT = "src/test/resources/files"; + private static final String FLOW_CONTENT_TYPE = "test/json"; + public static final String NOT_REGISTERED_CONTENT_TYPE = "test/contenttype"; @Test public void getConfigurationTest() throws IOException, ConfigurationProviderException { @@ -46,11 +49,11 @@ public class FileSystemConfigurationCacheTest { Map> parameters = new HashMap<>(); - ConfigurationCacheFileInfo info = cache.getCacheFileInfo("text/yaml", parameters); + ConfigurationCacheFileInfo info = cache.getCacheFileInfo(FLOW_CONTENT_TYPE, parameters); WriteableConfiguration configuration = info.getConfiguration(1); - assertEquals("config.text.yaml.v1", configuration.getName()); + assertEquals("config.test.json.v1", configuration.getName()); assertEquals("1", configuration.getVersion()); assertTrue(configuration.exists()); } @@ -63,7 +66,7 @@ public class FileSystemConfigurationCacheTest { Map> parameters = new HashMap<>(); - ConfigurationCacheFileInfo info = cache.getCacheFileInfo("test/contenttype", parameters); + ConfigurationCacheFileInfo info = cache.getCacheFileInfo(NOT_REGISTERED_CONTENT_TYPE, parameters); WriteableConfiguration configuration = info.getConfiguration(1); @@ -80,7 +83,7 @@ public class FileSystemConfigurationCacheTest { Map> parameters = new HashMap<>(); - ConfigurationCacheFileInfo info = cache.getCacheFileInfo("text/yaml", parameters); + ConfigurationCacheFileInfo info = cache.getCacheFileInfo(FLOW_CONTENT_TYPE, parameters); Stream configs = info.getCachedConfigurations(); @@ -96,6 +99,6 @@ public class FileSystemConfigurationCacheTest { Map> parameters = new HashMap<>(); - assertThrows(InvalidParameterException.class, () -> cache.getCacheFileInfo("test/contenttype", parameters)); + assertThrows(InvalidParameterException.class, () -> cache.getCacheFileInfo(NOT_REGISTERED_CONTENT_TYPE, parameters)); } } diff --git a/minifi/minifi-c2/minifi-c2-cache/minifi-c2-cache-filesystem/src/test/resources/files/config.test.json.v1 b/minifi/minifi-c2/minifi-c2-cache/minifi-c2-cache-filesystem/src/test/resources/files/config.test.json.v1 new file mode 100644 index 0000000000..c5e9e399c1 --- /dev/null +++ b/minifi/minifi-c2/minifi-c2-cache/minifi-c2-cache-filesystem/src/test/resources/files/config.test.json.v1 @@ -0,0 +1,55 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the \"License\"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an \"AS IS\" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +{ + "encodingVersion": { + "majorVersion": 2, + "minorVersion": 0 + }, + "maxTimerDrivenThreadCount": 10, + "maxEventDrivenThreadCount": 1, + "registries": [], + "parameterContexts": [], + "parameterProviders": [], + "controllerServices": [], + "reportingTasks": [], + "templates": [], + "rootGroup": { + "identifier": "c1b4e586-2011-3f81-a11e-8d669f084d1c", + "instanceIdentifier": "29db3dbc-0188-1000-7025-4cab8b52d278", + "name": "NiFi Flow", + "comments": "", + "position": { + "x": 0.0, + "y": 0.0 + }, + "processGroups": [], + "remoteProcessGroups": [], + "processors": [], + "inputPorts": [], + "outputPorts": [], + "connections": [], + "labels": [], + "funnels": [], + "controllerServices": [], + "variables": {}, + "defaultFlowFileExpiration": "0 sec", + "defaultBackPressureObjectThreshold": 10000, + "defaultBackPressureDataSizeThreshold": "1 GB", + "componentType": "PROCESS_GROUP", + "flowFileConcurrency": "UNBOUNDED", + "flowFileOutboundPolicy": "STREAM_WHEN_AVAILABLE" + } +} \ No newline at end of file diff --git a/minifi/minifi-c2/minifi-c2-cache/minifi-c2-cache-filesystem/src/test/resources/files/config.text.yaml.v1 b/minifi/minifi-c2/minifi-c2-cache/minifi-c2-cache-filesystem/src/test/resources/files/config.text.yaml.v1 deleted file mode 100644 index de7ea51579..0000000000 --- a/minifi/minifi-c2/minifi-c2-cache/minifi-c2-cache-filesystem/src/test/resources/files/config.text.yaml.v1 +++ /dev/null @@ -1,63 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the \"License\"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an \"AS IS\" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -MiNiFi Config Version: 3 -Flow Controller: - name: MiNiFi Flow - comment: '' -Core Properties: - flow controller graceful shutdown period: 10 sec - flow service write delay interval: 500 ms - administrative yield duration: 30 sec - bored yield duration: 10 millis - max concurrent threads: 1 -FlowFile Repository: - partitions: 256 - checkpoint interval: 2 mins - always sync: false - Swap: - threshold: 20000 - in period: 5 sec - in threads: 1 - out period: 5 sec - out threads: 4 -Content Repository: - content claim max appendable size: 10 MB - content claim max flow files: 100 - always sync: false -Provenance Repository: - provenance rollover time: 1 min -Component Status Repository: - buffer size: 1440 - snapshot frequency: 1 min -Security Properties: - keystore: '' - keystore type: '' - keystore password: '' - key password: '' - truststore: '' - truststore type: '' - truststore password: '' - ssl protocol: '' - Sensitive Props: - key: '' - algorithm: NIFI_PBKDF2_AES_GCM_256 - provider: BC -Processors: [] -Process Groups: [] -Funnels: [] -Connections: [] -Remote Process Groups: [] -NiFi Properties Overrides: {} \ No newline at end of file diff --git a/minifi/minifi-c2/minifi-c2-docker/dockerhub/Dockerfile b/minifi/minifi-c2/minifi-c2-docker/dockerhub/Dockerfile index 138ad24989..795afa3d3d 100644 --- a/minifi/minifi-c2/minifi-c2-docker/dockerhub/Dockerfile +++ b/minifi/minifi-c2/minifi-c2-docker/dockerhub/Dockerfile @@ -16,7 +16,7 @@ # under the License. # -FROM eclipse-temurin:11-jre +FROM eclipse-temurin:17-jre LABEL maintainer="Apache NiFi " LABEL site="https://nifi.apache.org" diff --git a/minifi/minifi-c2/minifi-c2-docker/dockermaven/Dockerfile b/minifi/minifi-c2/minifi-c2-docker/dockermaven/Dockerfile index 6af6d02243..d4c5e24aa3 100644 --- a/minifi/minifi-c2/minifi-c2-docker/dockermaven/Dockerfile +++ b/minifi/minifi-c2/minifi-c2-docker/dockermaven/Dockerfile @@ -17,7 +17,7 @@ # ARG IMAGE_NAME=eclipse-temurin -ARG IMAGE_TAG=11-jre +ARG IMAGE_TAG=17-jre FROM ${IMAGE_NAME}:${IMAGE_TAG} as builder LABEL stage=builder diff --git a/minifi/minifi-c2/minifi-c2-integration-tests/pom.xml b/minifi/minifi-c2/minifi-c2-integration-tests/pom.xml index 5828094c2d..32755c4dea 100644 --- a/minifi/minifi-c2/minifi-c2-integration-tests/pom.xml +++ b/minifi/minifi-c2/minifi-c2-integration-tests/pom.xml @@ -30,12 +30,6 @@ limitations under the License. - - org.apache.nifi.minifi - minifi-commons-schema - ${project.version} - test - org.apache.nifi.minifi minifi-c2-api @@ -59,6 +53,15 @@ limitations under the License. 2.0.0 test + + org.apache.nifi + nifi-framework-core-api + + + com.fasterxml.jackson.module + jackson-module-jaxb-annotations + test + @@ -91,7 +94,7 @@ limitations under the License. apache-rat-plugin - src/test/resources/mocknifi/www/nifi-api/flow/templates + **/config.application.json.* diff --git a/minifi/minifi-c2/minifi-c2-integration-tests/src/test/java/org/apache/nifi/minifi/c2/integration/test/AbstractTestSecure.java b/minifi/minifi-c2/minifi-c2-integration-tests/src/test/java/org/apache/nifi/minifi/c2/integration/test/AbstractTestSecure.java index d63274417b..587e418553 100644 --- a/minifi/minifi-c2/minifi-c2-integration-tests/src/test/java/org/apache/nifi/minifi/c2/integration/test/AbstractTestSecure.java +++ b/minifi/minifi-c2/minifi-c2-integration-tests/src/test/java/org/apache/nifi/minifi/c2/integration/test/AbstractTestSecure.java @@ -17,23 +17,11 @@ package org.apache.nifi.minifi.c2.integration.test; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; + import com.palantir.docker.compose.DockerComposeExtension; import com.palantir.docker.compose.connection.DockerPort; -import org.apache.commons.io.IOUtils; -import org.apache.nifi.minifi.commons.schema.ConfigSchema; -import org.apache.nifi.minifi.commons.schema.serialization.SchemaLoader; -import org.apache.nifi.security.util.KeyStoreUtils; -import org.apache.nifi.security.util.KeystoreType; -import org.apache.nifi.security.util.SslContextFactory; -import org.apache.nifi.security.util.StandardTlsConfiguration; -import org.apache.nifi.security.util.TlsConfiguration; -import org.apache.nifi.toolkit.tls.standalone.TlsToolkitStandalone; -import org.apache.nifi.toolkit.tls.standalone.TlsToolkitStandaloneCommandLine; -import org.junit.jupiter.api.Test; - -import javax.net.ssl.HttpsURLConnection; -import javax.net.ssl.SSLContext; -import javax.net.ssl.TrustManagerFactory; import java.io.FileInputStream; import java.io.IOException; import java.io.InputStream; @@ -49,9 +37,19 @@ import java.security.KeyStore; import java.util.ArrayList; import java.util.Arrays; import java.util.List; - -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertThrows; +import javax.net.ssl.HttpsURLConnection; +import javax.net.ssl.SSLContext; +import javax.net.ssl.TrustManagerFactory; +import org.apache.commons.io.IOUtils; +import org.apache.nifi.controller.flow.VersionedDataflow; +import org.apache.nifi.security.util.KeyStoreUtils; +import org.apache.nifi.security.util.KeystoreType; +import org.apache.nifi.security.util.SslContextFactory; +import org.apache.nifi.security.util.StandardTlsConfiguration; +import org.apache.nifi.security.util.TlsConfiguration; +import org.apache.nifi.toolkit.tls.standalone.TlsToolkitStandalone; +import org.apache.nifi.toolkit.tls.standalone.TlsToolkitStandaloneCommandLine; +import org.junit.jupiter.api.Test; public abstract class AbstractTestSecure extends AbstractTestUnsecure { public static final String C2_URL = "https://c2:10443/c2/config"; @@ -73,7 +71,7 @@ public abstract class AbstractTestSecure extends AbstractTestUnsecure { public static SSLContext initCertificates(Path certificatesDirectory, List serverHostnames) throws Exception { List toolkitCommandLine = new ArrayList<>(Arrays.asList("-O", "-o", certificatesDirectory.toFile().getAbsolutePath(), - "-C", "CN=user1", "-C", "CN=user2", "-C", "CN=user3", "-C", "CN=user4", "-S", "badKeystorePass", "-K", "badKeyPass", "-P", "badTrustPass")); + "-C", "CN=user1", "-C", "CN=user2", "-C", "CN=user3", "-C", "CN=user4", "-S", "badKeystorePass", "-K", "badKeyPass", "-P", "badTrustPass")); for (String serverHostname : serverHostnames) { toolkitCommandLine.add("-n"); toolkitCommandLine.add(serverHostname); @@ -95,8 +93,8 @@ public abstract class AbstractTestSecure extends AbstractTestUnsecure { trustManagerFactory.init(trustStore); TlsConfiguration tlsConfiguration = new StandardTlsConfiguration( - null, null, null, - certificatesDirectory.resolve("c2").resolve("truststore.jks").toFile().getAbsolutePath(), "badTrustPass", KeystoreType.JKS); + null, null, null, + certificatesDirectory.resolve("c2").resolve("truststore.jks").toFile().getAbsolutePath(), "badTrustPass", KeystoreType.JKS); return SslContextFactory.createSslContext(tlsConfiguration); @@ -115,8 +113,8 @@ public abstract class AbstractTestSecure extends AbstractTestUnsecure { assertReturnCode("", sslContext, 403); - ConfigSchema configSchema = assertReturnCode("?class=raspi2", sslContext, 200); - assertEquals("raspi2.v1", configSchema.getFlowControllerProperties().getName()); + VersionedDataflow versionedDataflow = assertReturnCode("?class=raspi2", sslContext, 200); + assertEquals("raspi2.v1", versionedDataflow.getRootGroup().getName()); assertReturnCode("?class=raspi3", sslContext, 403); } @@ -128,8 +126,8 @@ public abstract class AbstractTestSecure extends AbstractTestUnsecure { assertReturnCode("", sslContext, 403); assertReturnCode("?class=raspi2", sslContext, 403); - ConfigSchema configSchema = assertReturnCode("?class=raspi3", sslContext, 200); - assertEquals("raspi3.v2", configSchema.getFlowControllerProperties().getName()); + VersionedDataflow versionedDataflow = assertReturnCode("?class=raspi3", sslContext, 200); + assertEquals("raspi3.v2", versionedDataflow.getRootGroup().getName()); } @Test @@ -138,11 +136,11 @@ public abstract class AbstractTestSecure extends AbstractTestUnsecure { assertReturnCode("", sslContext, 400); - ConfigSchema configSchema = assertReturnCode("?class=raspi2", sslContext, 200); - assertEquals("raspi2.v1", configSchema.getFlowControllerProperties().getName()); + VersionedDataflow dataflowRaspi2 = assertReturnCode("?class=raspi2", sslContext, 200); + assertEquals("raspi2.v1", dataflowRaspi2.getRootGroup().getName()); - configSchema = assertReturnCode("?class=raspi3", sslContext, 200); - assertEquals("raspi3.v2", configSchema.getFlowControllerProperties().getName()); + VersionedDataflow dataflowRaspi3 = assertReturnCode("?class=raspi3", sslContext, 200); + assertEquals("raspi3.v2", dataflowRaspi3.getRootGroup().getName()); } @Test @@ -169,22 +167,25 @@ public abstract class AbstractTestSecure extends AbstractTestUnsecure { keystorePasswd = IOUtils.toString(inputStream, StandardCharsets.UTF_8); } TlsConfiguration tlsConfiguration = new StandardTlsConfiguration( - directory.resolve("CN=" + username + ".p12").toFile().getAbsolutePath(), - keystorePasswd, - KeystoreType.PKCS12, - certificatesDirectory.resolve("c2").resolve("truststore.jks").toFile().getAbsolutePath(), - "badTrustPass", - KeystoreType.JKS); + directory.resolve("CN=" + username + ".p12").toFile().getAbsolutePath(), + keystorePasswd, + KeystoreType.PKCS12, + certificatesDirectory.resolve("c2").resolve("truststore.jks").toFile().getAbsolutePath(), + "badTrustPass", + KeystoreType.JKS); return SslContextFactory.createSslContext(tlsConfiguration); } - protected ConfigSchema assertReturnCode(String query, SSLContext sslContext, int expectedReturnCode) throws Exception { + protected VersionedDataflow assertReturnCode(String query, SSLContext sslContext, int expectedReturnCode) throws Exception { HttpsURLConnection httpsURLConnection = openUrlConnection(C2_URL + query, sslContext); + httpsURLConnection.setRequestProperty("Accept", "application/json"); try { assertEquals(expectedReturnCode, httpsURLConnection.getResponseCode()); if (expectedReturnCode == 200) { - return SchemaLoader.loadConfigSchemaFromYaml(httpsURLConnection.getInputStream()); + try (InputStream inputStream = httpsURLConnection.getInputStream()) { + return toVersionedDataFlow(inputStream); + } } } finally { httpsURLConnection.disconnect(); @@ -195,7 +196,7 @@ public abstract class AbstractTestSecure extends AbstractTestUnsecure { protected HttpsURLConnection openUrlConnection(String url, SSLContext sslContext) throws IOException { DockerPort dockerPort = docker.containers().container("squid").port(3128); HttpsURLConnection httpURLConnection = (HttpsURLConnection) new URL(url).openConnection( - new Proxy(Proxy.Type.HTTP, new InetSocketAddress(dockerPort.getIp(), dockerPort.getExternalPort()))); + new Proxy(Proxy.Type.HTTP, new InetSocketAddress(dockerPort.getIp(), dockerPort.getExternalPort()))); httpURLConnection.setSSLSocketFactory(sslContext.getSocketFactory()); return httpURLConnection; } diff --git a/minifi/minifi-c2/minifi-c2-integration-tests/src/test/java/org/apache/nifi/minifi/c2/integration/test/AbstractTestUnsecure.java b/minifi/minifi-c2/minifi-c2-integration-tests/src/test/java/org/apache/nifi/minifi/c2/integration/test/AbstractTestUnsecure.java index 4fead5fb19..fd174c9344 100644 --- a/minifi/minifi-c2/minifi-c2-integration-tests/src/test/java/org/apache/nifi/minifi/c2/integration/test/AbstractTestUnsecure.java +++ b/minifi/minifi-c2/minifi-c2-integration-tests/src/test/java/org/apache/nifi/minifi/c2/integration/test/AbstractTestUnsecure.java @@ -17,20 +17,21 @@ package org.apache.nifi.minifi.c2.integration.test; +import static org.junit.jupiter.api.Assertions.assertEquals; + +import com.fasterxml.jackson.databind.DeserializationFeature; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.module.jaxb.JaxbAnnotationIntrospector; import com.palantir.docker.compose.DockerComposeExtension; import com.palantir.docker.compose.connection.Container; import com.palantir.docker.compose.connection.DockerPort; -import org.apache.nifi.minifi.commons.schema.ConfigSchema; -import org.apache.nifi.minifi.commons.schema.exception.SchemaLoaderException; -import org.apache.nifi.minifi.commons.schema.serialization.SchemaLoader; -import org.junit.jupiter.api.Test; - import java.io.IOException; import java.io.InputStream; import java.net.HttpURLConnection; import java.net.URL; - -import static org.junit.jupiter.api.Assertions.assertEquals; +import org.apache.commons.io.IOUtils; +import org.apache.nifi.controller.flow.VersionedDataflow; +import org.junit.jupiter.api.Test; public abstract class AbstractTestUnsecure { protected String c2Url; @@ -49,31 +50,28 @@ public abstract class AbstractTestUnsecure { } @Test - public void testCurrentVersion() throws IOException, SchemaLoaderException { - ConfigSchema configSchema = getConfigSchema(c2Url + "?class=raspi3"); - assertEquals(3, configSchema.getVersion()); - assertEquals("raspi3.v2", configSchema.getFlowControllerProperties().getName()); + public void testCurrentVersion() throws IOException { + VersionedDataflow versionedDataflow = getFlowDefinition(c2Url + "?class=raspi3"); + assertEquals("raspi3.v2", versionedDataflow.getRootGroup().getName()); } @Test - public void testVersion1() throws IOException, SchemaLoaderException { - ConfigSchema configSchema = getConfigSchema(c2Url + "?class=raspi3&version=1"); - assertEquals(3, configSchema.getVersion()); - assertEquals("raspi3.v1", configSchema.getFlowControllerProperties().getName()); + public void testVersion1() throws IOException { + VersionedDataflow versionedDataflow = getFlowDefinition(c2Url + "?class=raspi3&version=1"); + assertEquals("raspi3.v1", versionedDataflow.getRootGroup().getName()); } @Test - public void testVersion2() throws IOException, SchemaLoaderException { - ConfigSchema configSchema = getConfigSchema(c2Url + "?class=raspi3&version=2"); - assertEquals(3, configSchema.getVersion()); - assertEquals("raspi3.v2", configSchema.getFlowControllerProperties().getName()); + public void testVersion2() throws IOException { + VersionedDataflow versionedDataflow = getFlowDefinition(c2Url + "?class=raspi3&version=2"); + assertEquals("raspi3.v2", versionedDataflow.getRootGroup().getName()); } @Test public void testUnacceptable() throws IOException { HttpURLConnection urlConnection = openSuperUserUrlConnection(c2Url + "?class=raspi3"); try { - urlConnection.setRequestProperty("Accept", "text/xml"); + urlConnection.setRequestProperty("Accept", "text/invalid"); assertEquals(406, urlConnection.getResponseCode()); } finally { urlConnection.disconnect(); @@ -90,18 +88,24 @@ public abstract class AbstractTestUnsecure { } } - public ConfigSchema getConfigSchema(String urlString) throws IOException, SchemaLoaderException { + public VersionedDataflow getFlowDefinition(String urlString) throws IOException { HttpURLConnection urlConnection = openSuperUserUrlConnection(urlString); - ConfigSchema configSchema; + urlConnection.setRequestProperty("Accept", "application/json"); try (InputStream inputStream = urlConnection.getInputStream()) { - configSchema = SchemaLoader.loadConfigSchemaFromYaml(inputStream); + return toVersionedDataFlow(inputStream); } finally { urlConnection.disconnect(); } - return configSchema; } protected HttpURLConnection openSuperUserUrlConnection(String url) throws IOException { return (HttpURLConnection) new URL(url).openConnection(); } + + protected VersionedDataflow toVersionedDataFlow(InputStream inputStream) throws IOException { + ObjectMapper objectMapper = new ObjectMapper(); + objectMapper.setAnnotationIntrospector(new JaxbAnnotationIntrospector(objectMapper.getTypeFactory())); + objectMapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false); + return objectMapper.readValue(IOUtils.toByteArray(inputStream), VersionedDataflow.class); + } } diff --git a/minifi/minifi-c2/minifi-c2-integration-tests/src/test/java/org/apache/nifi/minifi/c2/integration/test/NiFiRestConfigurationProviderSecureTest.java b/minifi/minifi-c2/minifi-c2-integration-tests/src/test/java/org/apache/nifi/minifi/c2/integration/test/NiFiRestConfigurationProviderSecureTest.java deleted file mode 100644 index 0ddd58a51c..0000000000 --- a/minifi/minifi-c2/minifi-c2-integration-tests/src/test/java/org/apache/nifi/minifi/c2/integration/test/NiFiRestConfigurationProviderSecureTest.java +++ /dev/null @@ -1,118 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.nifi.minifi.c2.integration.test; - -import com.palantir.docker.compose.DockerComposeExtension; -import org.apache.nifi.minifi.c2.integration.test.health.HttpsStatusCodeHealthCheck; -import org.apache.nifi.security.util.KeystoreType; -import org.apache.nifi.security.util.SslContextFactory; -import org.apache.nifi.security.util.StandardTlsConfiguration; -import org.apache.nifi.security.util.TlsConfiguration; -import org.apache.nifi.toolkit.tls.standalone.TlsToolkitStandalone; -import org.bouncycastle.openssl.jcajce.JcaMiscPEMGenerator; -import org.bouncycastle.util.io.pem.PemWriter; -import org.junit.jupiter.api.AfterAll; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.BeforeAll; - -import javax.net.ssl.SSLContext; -import javax.net.ssl.SSLSocketFactory; -import java.io.InputStream; -import java.io.OutputStreamWriter; -import java.nio.file.Files; -import java.nio.file.Path; -import java.nio.file.Paths; -import java.security.KeyStore; -import java.security.cert.Certificate; -import java.util.Arrays; - -public class NiFiRestConfigurationProviderSecureTest extends AbstractTestSecure { - private static SSLSocketFactory healthCheckSocketFactory; - private static Path certificatesDirectory; - private static SSLContext trustSslContext; - - // Not annotated as rule because we need to generate certificatesDirectory first - public static DockerComposeExtension docker = DockerComposeExtension.builder() - .file("target/test-classes/docker-compose-NiFiRestConfigurationProviderSecureTest.yml") - .waitingForServices(Arrays.asList("squid", "mocknifi"), - new HttpsStatusCodeHealthCheck(container -> "https://mocknifi:8443/", containers -> containers.get(0), containers -> containers.get(1), () -> { - Path c2 = certificatesDirectory.resolve("c2"); - try { - TlsConfiguration tlsConfiguration = new StandardTlsConfiguration( - c2.resolve("keystore.jks").toFile().getAbsolutePath(), - "badKeystorePass", - "badKeyPass", KeystoreType.JKS, - c2.resolve("truststore.jks").toFile().getAbsolutePath(), - "badTrustPass", - KeystoreType.JKS); - - return SslContextFactory.createSslContext(tlsConfiguration).getSocketFactory(); - } catch (Exception e) { - throw new RuntimeException(e); - } - }, 200)) - .waitingForServices(Arrays.asList("squid", "c2"), - new HttpsStatusCodeHealthCheck(container -> C2_URL, containers -> containers.get(0), containers -> containers.get(1), () -> healthCheckSocketFactory, 403)) - .build(); - - public NiFiRestConfigurationProviderSecureTest() { - super(docker, certificatesDirectory, trustSslContext); - } - - /** - * Generates certificates with the tls-toolkit and then starts up the docker compose file - */ - @BeforeAll - public static void initCertificates() throws Exception { - certificatesDirectory = Paths.get(NiFiRestConfigurationProviderSecureTest.class.getClassLoader() - .getResource("docker-compose-NiFiRestConfigurationProviderSecureTest.yml").getFile()).getParent().toAbsolutePath().resolve("certificates-NiFiRestConfigurationProviderSecureTest"); - trustSslContext = initCertificates(certificatesDirectory, Arrays.asList("c2", "mocknifi")); - healthCheckSocketFactory = trustSslContext.getSocketFactory(); - - KeyStore mockNiFiKeyStore = KeyStore.getInstance("JKS"); - try (InputStream inputStream = Files.newInputStream(certificatesDirectory.resolve("mocknifi").resolve("keystore.jks"))) { - mockNiFiKeyStore.load(inputStream, "badKeystorePass".toCharArray()); - } - try (PemWriter pemWriter = new PemWriter(new OutputStreamWriter(Files.newOutputStream(certificatesDirectory.resolve("mocknifi").resolve("cert.pem"))))) { - pemWriter.writeObject(new JcaMiscPEMGenerator(mockNiFiKeyStore.getKey(TlsToolkitStandalone.NIFI_KEY, "badKeyPass".toCharArray()))); - for (Certificate certificate : mockNiFiKeyStore.getCertificateChain(TlsToolkitStandalone.NIFI_KEY)) { - pemWriter.writeObject(new JcaMiscPEMGenerator(certificate)); - } - } - - KeyStore mockNiFiTrustStore = KeyStore.getInstance("JKS"); - try (InputStream inputStream = Files.newInputStream(certificatesDirectory.resolve("mocknifi").resolve("truststore.jks"))) { - mockNiFiTrustStore.load(inputStream, "badTrustPass".toCharArray()); - } - try (PemWriter pemWriter = new PemWriter(new OutputStreamWriter(Files.newOutputStream(certificatesDirectory.resolve("mocknifi").resolve("ca.pem"))))) { - pemWriter.writeObject(new JcaMiscPEMGenerator(mockNiFiTrustStore.getCertificate(TlsToolkitStandalone.NIFI_CERT))); - } - - docker.before(); - } - - @AfterAll - public static void cleanup() { - docker.after(); - } - - @BeforeEach - public void setup() { - super.setup(docker); - } -} diff --git a/minifi/minifi-c2/minifi-c2-integration-tests/src/test/java/org/apache/nifi/minifi/c2/integration/test/NiFiRestConfigurationProviderUnsecureTest.java b/minifi/minifi-c2/minifi-c2-integration-tests/src/test/java/org/apache/nifi/minifi/c2/integration/test/NiFiRestConfigurationProviderUnsecureTest.java deleted file mode 100644 index 372419df81..0000000000 --- a/minifi/minifi-c2/minifi-c2-integration-tests/src/test/java/org/apache/nifi/minifi/c2/integration/test/NiFiRestConfigurationProviderUnsecureTest.java +++ /dev/null @@ -1,39 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.nifi.minifi.c2.integration.test; - -import com.palantir.docker.compose.DockerComposeExtension; -import com.palantir.docker.compose.connection.waiting.HealthChecks; -import org.apache.nifi.minifi.c2.integration.test.health.HttpStatusCodeHealthCheck; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.extension.RegisterExtension; - -public class NiFiRestConfigurationProviderUnsecureTest extends AbstractTestUnsecure { - @RegisterExtension - public static DockerComposeExtension docker = DockerComposeExtension.builder() - .file("target/test-classes/docker-compose-NiFiRestConfigurationProviderUnsecureTest.yml") - .waitingForService("mocknifi", HealthChecks.toRespond2xxOverHttp(8080, - dockerPort -> "http://" + dockerPort.getIp() + ":" + dockerPort.getExternalPort() + "/")) - .waitingForService("c2", new HttpStatusCodeHealthCheck(FileSystemCacheProviderUnsecureTest::getUnsecureConfigUrl, 400)) - .build(); - - @BeforeEach - public void setup() { - super.setup(docker); - } -} diff --git a/minifi/minifi-c2/minifi-c2-integration-tests/src/test/resources/c2-secure-rest/conf/minifi-c2-context.xml b/minifi/minifi-c2/minifi-c2-integration-tests/src/test/resources/c2-secure-rest/conf/minifi-c2-context.xml deleted file mode 100644 index d5436a5a2e..0000000000 --- a/minifi/minifi-c2/minifi-c2-integration-tests/src/test/resources/c2-secure-rest/conf/minifi-c2-context.xml +++ /dev/null @@ -1,60 +0,0 @@ - - - - - - - - - - - - - - ./cache - - - ${class}/${class} - - - - - https://mocknifi:8443/nifi-api - - - ${class}.v${version} - - - - - - - - - - - diff --git a/minifi/minifi-c2/minifi-c2-integration-tests/src/test/resources/c2-unsecure-rest/conf/minifi-c2-context.xml b/minifi/minifi-c2/minifi-c2-integration-tests/src/test/resources/c2-unsecure-rest/conf/minifi-c2-context.xml deleted file mode 100644 index df02d77a9a..0000000000 --- a/minifi/minifi-c2/minifi-c2-integration-tests/src/test/resources/c2-unsecure-rest/conf/minifi-c2-context.xml +++ /dev/null @@ -1,59 +0,0 @@ - - - - - - - - - - - - - ./cache - - - ${class}/${class} - - - - - http://mocknifi:8080/nifi-api - - - ${class}.v${version} - - - - - - - - - - - diff --git a/minifi/minifi-c2/minifi-c2-integration-tests/src/test/resources/c2/files/raspi2/config.application.json.v1 b/minifi/minifi-c2/minifi-c2-integration-tests/src/test/resources/c2/files/raspi2/config.application.json.v1 new file mode 100644 index 0000000000..b41ffa5a56 --- /dev/null +++ b/minifi/minifi-c2/minifi-c2-integration-tests/src/test/resources/c2/files/raspi2/config.application.json.v1 @@ -0,0 +1,40 @@ +{ + "encodingVersion": { + "majorVersion": 2, + "minorVersion": 0 + }, + "maxTimerDrivenThreadCount": 10, + "maxEventDrivenThreadCount": 1, + "registries": [], + "parameterContexts": [], + "parameterProviders": [], + "controllerServices": [], + "reportingTasks": [], + "templates": [], + "rootGroup": { + "identifier": "c1b4e586-2011-3f81-a11e-8d669f084d1c", + "instanceIdentifier": "29db3dbc-0188-1000-7025-4cab8b52d278", + "name": "raspi2.v1", + "comments": "", + "position": { + "x": 0.0, + "y": 0.0 + }, + "processGroups": [], + "remoteProcessGroups": [], + "processors": [], + "inputPorts": [], + "outputPorts": [], + "connections": [], + "labels": [], + "funnels": [], + "controllerServices": [], + "variables": {}, + "defaultFlowFileExpiration": "0 sec", + "defaultBackPressureObjectThreshold": 10000, + "defaultBackPressureDataSizeThreshold": "1 GB", + "componentType": "PROCESS_GROUP", + "flowFileConcurrency": "UNBOUNDED", + "flowFileOutboundPolicy": "STREAM_WHEN_AVAILABLE" + } +} \ No newline at end of file diff --git a/minifi/minifi-c2/minifi-c2-integration-tests/src/test/resources/c2/files/raspi2/config.text.yml.v1 b/minifi/minifi-c2/minifi-c2-integration-tests/src/test/resources/c2/files/raspi2/config.text.yml.v1 deleted file mode 100644 index 6c01f107eb..0000000000 --- a/minifi/minifi-c2/minifi-c2-integration-tests/src/test/resources/c2/files/raspi2/config.text.yml.v1 +++ /dev/null @@ -1,63 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the \"License\"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an \"AS IS\" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -MiNiFi Config Version: 3 -Flow Controller: - name: raspi2.v1 - comment: '' -Core Properties: - flow controller graceful shutdown period: 10 sec - flow service write delay interval: 500 ms - administrative yield duration: 30 sec - bored yield duration: 10 millis - max concurrent threads: 1 -FlowFile Repository: - partitions: 256 - checkpoint interval: 2 mins - always sync: false - Swap: - threshold: 20000 - in period: 5 sec - in threads: 1 - out period: 5 sec - out threads: 4 -Content Repository: - content claim max appendable size: 10 MB - content claim max flow files: 100 - always sync: false -Provenance Repository: - provenance rollover time: 1 min -Component Status Repository: - buffer size: 1440 - snapshot frequency: 1 min -Security Properties: - keystore: '' - keystore type: '' - keystore password: '' - key password: '' - truststore: '' - truststore type: '' - truststore password: '' - ssl protocol: '' - Sensitive Props: - key: '' - algorithm: NIFI_PBKDF2_AES_GCM_256 - provider: BC -Processors: [] -Process Groups: [] -Funnels: [] -Connections: [] -Remote Process Groups: [] -NiFi Properties Overrides: {} diff --git a/minifi/minifi-c2/minifi-c2-integration-tests/src/test/resources/c2/files/raspi3/config.application.json.v1 b/minifi/minifi-c2/minifi-c2-integration-tests/src/test/resources/c2/files/raspi3/config.application.json.v1 new file mode 100644 index 0000000000..ac13de66c8 --- /dev/null +++ b/minifi/minifi-c2/minifi-c2-integration-tests/src/test/resources/c2/files/raspi3/config.application.json.v1 @@ -0,0 +1,40 @@ +{ + "encodingVersion": { + "majorVersion": 2, + "minorVersion": 0 + }, + "maxTimerDrivenThreadCount": 10, + "maxEventDrivenThreadCount": 1, + "registries": [], + "parameterContexts": [], + "parameterProviders": [], + "controllerServices": [], + "reportingTasks": [], + "templates": [], + "rootGroup": { + "identifier": "c1b4e586-2011-3f81-a11e-8d669f084d1c", + "instanceIdentifier": "29db3dbc-0188-1000-7025-4cab8b52d278", + "name": "raspi3.v1", + "comments": "", + "position": { + "x": 0.0, + "y": 0.0 + }, + "processGroups": [], + "remoteProcessGroups": [], + "processors": [], + "inputPorts": [], + "outputPorts": [], + "connections": [], + "labels": [], + "funnels": [], + "controllerServices": [], + "variables": {}, + "defaultFlowFileExpiration": "0 sec", + "defaultBackPressureObjectThreshold": 10000, + "defaultBackPressureDataSizeThreshold": "1 GB", + "componentType": "PROCESS_GROUP", + "flowFileConcurrency": "UNBOUNDED", + "flowFileOutboundPolicy": "STREAM_WHEN_AVAILABLE" + } +} \ No newline at end of file diff --git a/minifi/minifi-c2/minifi-c2-integration-tests/src/test/resources/c2/files/raspi3/config.application.json.v2 b/minifi/minifi-c2/minifi-c2-integration-tests/src/test/resources/c2/files/raspi3/config.application.json.v2 new file mode 100644 index 0000000000..2afdf48f1f --- /dev/null +++ b/minifi/minifi-c2/minifi-c2-integration-tests/src/test/resources/c2/files/raspi3/config.application.json.v2 @@ -0,0 +1,40 @@ +{ + "encodingVersion": { + "majorVersion": 2, + "minorVersion": 0 + }, + "maxTimerDrivenThreadCount": 10, + "maxEventDrivenThreadCount": 1, + "registries": [], + "parameterContexts": [], + "parameterProviders": [], + "controllerServices": [], + "reportingTasks": [], + "templates": [], + "rootGroup": { + "identifier": "c1b4e586-2011-3f81-a11e-8d669f084d1c", + "instanceIdentifier": "29db3dbc-0188-1000-7025-4cab8b52d278", + "name": "raspi3.v2", + "comments": "", + "position": { + "x": 0.0, + "y": 0.0 + }, + "processGroups": [], + "remoteProcessGroups": [], + "processors": [], + "inputPorts": [], + "outputPorts": [], + "connections": [], + "labels": [], + "funnels": [], + "controllerServices": [], + "variables": {}, + "defaultFlowFileExpiration": "0 sec", + "defaultBackPressureObjectThreshold": 10000, + "defaultBackPressureDataSizeThreshold": "1 GB", + "componentType": "PROCESS_GROUP", + "flowFileConcurrency": "UNBOUNDED", + "flowFileOutboundPolicy": "STREAM_WHEN_AVAILABLE" + } +} \ No newline at end of file diff --git a/minifi/minifi-c2/minifi-c2-integration-tests/src/test/resources/c2/files/raspi3/config.text.yml.v1 b/minifi/minifi-c2/minifi-c2-integration-tests/src/test/resources/c2/files/raspi3/config.text.yml.v1 deleted file mode 100644 index 5daf281d6f..0000000000 --- a/minifi/minifi-c2/minifi-c2-integration-tests/src/test/resources/c2/files/raspi3/config.text.yml.v1 +++ /dev/null @@ -1,63 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the \"License\"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an \"AS IS\" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -MiNiFi Config Version: 3 -Flow Controller: - name: raspi3.v1 - comment: '' -Core Properties: - flow controller graceful shutdown period: 10 sec - flow service write delay interval: 500 ms - administrative yield duration: 30 sec - bored yield duration: 10 millis - max concurrent threads: 1 -FlowFile Repository: - partitions: 256 - checkpoint interval: 2 mins - always sync: false - Swap: - threshold: 20000 - in period: 5 sec - in threads: 1 - out period: 5 sec - out threads: 4 -Content Repository: - content claim max appendable size: 10 MB - content claim max flow files: 100 - always sync: false -Provenance Repository: - provenance rollover time: 1 min -Component Status Repository: - buffer size: 1440 - snapshot frequency: 1 min -Security Properties: - keystore: '' - keystore type: '' - keystore password: '' - key password: '' - truststore: '' - truststore type: '' - truststore password: '' - ssl protocol: '' - Sensitive Props: - key: '' - algorithm: NIFI_PBKDF2_AES_GCM_256 - provider: BC -Processors: [] -Process Groups: [] -Funnels: [] -Connections: [] -Remote Process Groups: [] -NiFi Properties Overrides: {} diff --git a/minifi/minifi-c2/minifi-c2-integration-tests/src/test/resources/c2/files/raspi3/config.text.yml.v2 b/minifi/minifi-c2/minifi-c2-integration-tests/src/test/resources/c2/files/raspi3/config.text.yml.v2 deleted file mode 100644 index 83230e1a19..0000000000 --- a/minifi/minifi-c2/minifi-c2-integration-tests/src/test/resources/c2/files/raspi3/config.text.yml.v2 +++ /dev/null @@ -1,63 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the \"License\"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an \"AS IS\" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -MiNiFi Config Version: 3 -Flow Controller: - name: raspi3.v2 - comment: '' -Core Properties: - flow controller graceful shutdown period: 10 sec - flow service write delay interval: 500 ms - administrative yield duration: 30 sec - bored yield duration: 10 millis - max concurrent threads: 1 -FlowFile Repository: - partitions: 256 - checkpoint interval: 2 mins - always sync: false - Swap: - threshold: 20000 - in period: 5 sec - in threads: 1 - out period: 5 sec - out threads: 4 -Content Repository: - content claim max appendable size: 10 MB - content claim max flow files: 100 - always sync: false -Provenance Repository: - provenance rollover time: 1 min -Component Status Repository: - buffer size: 1440 - snapshot frequency: 1 min -Security Properties: - keystore: '' - keystore type: '' - keystore password: '' - key password: '' - truststore: '' - truststore type: '' - truststore password: '' - ssl protocol: '' - Sensitive Props: - key: '' - algorithm: NIFI_PBKDF2_AES_GCM_256 - provider: BC -Processors: [] -Process Groups: [] -Funnels: [] -Connections: [] -Remote Process Groups: [] -NiFi Properties Overrides: {} diff --git a/minifi/minifi-c2/minifi-c2-integration-tests/src/test/resources/docker-compose-NiFiRestConfigurationProviderSecureTest.yml b/minifi/minifi-c2/minifi-c2-integration-tests/src/test/resources/docker-compose-NiFiRestConfigurationProviderSecureTest.yml deleted file mode 100644 index 66bfecf96b..0000000000 --- a/minifi/minifi-c2/minifi-c2-integration-tests/src/test/resources/docker-compose-NiFiRestConfigurationProviderSecureTest.yml +++ /dev/null @@ -1,55 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the \"License\"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an \"AS IS\" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -version: "2" - -services: - c2: - image: apache/nifi-minifi-c2:${minifi.c2.version}-maven - ports: - - "10443" - hostname: c2 - volumes: - - ./c2-secure/conf/c2.properties:/opt/minifi-c2/minifi-c2-${minifi.c2.version}/conf/c2.properties - - ./c2-secure/conf/authorities.yaml:/opt/minifi-c2/minifi-c2-${minifi.c2.version}/conf/authorities.yaml - - ./c2-secure/conf/authorizations.yaml:/opt/minifi-c2/minifi-c2-${minifi.c2.version}/conf/authorizations.yaml - - ./c2-secure-rest/conf/minifi-c2-context.xml:/opt/minifi-c2/minifi-c2-${minifi.c2.version}/conf/minifi-c2-context.xml - - - ./certificates-NiFiRestConfigurationProviderSecureTest/c2/keystore.jks:/opt/minifi-c2/minifi-c2-${minifi.c2.version}/conf/keystore.jks - - ./certificates-NiFiRestConfigurationProviderSecureTest/c2/truststore.jks:/opt/minifi-c2/minifi-c2-${minifi.c2.version}/conf/truststore.jks - - squid: - image: chrisdaish/squid - ports: - - "3128" - hostname: squid - volumes: - - ./squid/squid.conf:/etc/squid/squid.conf - - mocknifi: - image: python:2 - ports: - - "8443" - hostname: mocknifi - volumes: - - ./mocknifi-secure/server.py:/root/server.py - - ./mocknifi/www:/var/www - - - ./certificates-NiFiRestConfigurationProviderSecureTest/mocknifi/cert.pem:/root/cert.pem - - ./certificates-NiFiRestConfigurationProviderSecureTest/mocknifi/ca.pem:/root/ca.pem - working_dir: /var/www - entrypoint: - - python - - /root/server.py \ No newline at end of file diff --git a/minifi/minifi-c2/minifi-c2-integration-tests/src/test/resources/docker-compose-NiFiRestConfigurationProviderUnsecureTest.yml b/minifi/minifi-c2/minifi-c2-integration-tests/src/test/resources/docker-compose-NiFiRestConfigurationProviderUnsecureTest.yml deleted file mode 100644 index 811791f77b..0000000000 --- a/minifi/minifi-c2/minifi-c2-integration-tests/src/test/resources/docker-compose-NiFiRestConfigurationProviderUnsecureTest.yml +++ /dev/null @@ -1,37 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the \"License\"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an \"AS IS\" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -version: "2" - -services: - c2: - image: apache/nifi-minifi-c2:${minifi.c2.version}-maven - ports: - - "10090" - hostname: c2 - volumes: - - ./c2-unsecure-rest/conf/minifi-c2-context.xml:/opt/minifi-c2/minifi-c2-${minifi.c2.version}/conf/minifi-c2-context.xml - mocknifi: - image: python:2 - ports: - - "8080" - hostname: mocknifi - volumes: - - ./mocknifi-unsecure/server.py:/root/server.py - - ./mocknifi/www:/var/www - working_dir: /var/www - entrypoint: - - python - - /root/server.py \ No newline at end of file diff --git a/minifi/minifi-c2/minifi-c2-integration-tests/src/test/resources/mocknifi-secure/server.py b/minifi/minifi-c2/minifi-c2-integration-tests/src/test/resources/mocknifi-secure/server.py deleted file mode 100644 index fe869a8907..0000000000 --- a/minifi/minifi-c2/minifi-c2-integration-tests/src/test/resources/mocknifi-secure/server.py +++ /dev/null @@ -1,48 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the \"License\"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an \"AS IS\" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -#!/usr/bin/env python - -import logging - -from argparse import ArgumentParser -from BaseHTTPServer import HTTPServer -from os import chdir -from SimpleHTTPServer import SimpleHTTPRequestHandler -from SocketServer import ThreadingMixIn -from ssl import CERT_REQUIRED, wrap_socket - -# Needs to be threaded or health check hangs the server -class ThreadedHTTPServer(ThreadingMixIn, HTTPServer): - pass - -if __name__ == '__main__': - logging.basicConfig(level=logging.DEBUG) - parser = ArgumentParser(description='Serve up directory with ssl') - parser.add_argument('--dir', default='/var/www') - parser.add_argument('--port', type=int, default=8443) - parser.add_argument('--cert', default='/root/cert.pem') - parser.add_argument('--ca', default='/root/ca.pem') - - logging.debug('About to parse arguments') - args = parser.parse_args() - - logging.debug('Serving directory ' + args.dir + ' via HTTPS at port ' + str(args.port)) - - chdir(args.dir) - - server = ThreadedHTTPServer(('', args.port), SimpleHTTPRequestHandler) - server.socket = wrap_socket(server.socket, certfile=args.cert, ca_certs=args.ca, server_side=True, cert_reqs=CERT_REQUIRED) - server.serve_forever() \ No newline at end of file diff --git a/minifi/minifi-c2/minifi-c2-integration-tests/src/test/resources/mocknifi-unsecure/server.py b/minifi/minifi-c2/minifi-c2-integration-tests/src/test/resources/mocknifi-unsecure/server.py deleted file mode 100644 index 82d5f6a635..0000000000 --- a/minifi/minifi-c2/minifi-c2-integration-tests/src/test/resources/mocknifi-unsecure/server.py +++ /dev/null @@ -1,44 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the \"License\"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an \"AS IS\" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -#!/usr/bin/env python - -import logging - -from argparse import ArgumentParser -from BaseHTTPServer import HTTPServer -from os import chdir -from SimpleHTTPServer import SimpleHTTPRequestHandler -from SocketServer import ThreadingMixIn - -# Needs to be threaded or health check hangs the server -class ThreadedHTTPServer(ThreadingMixIn, HTTPServer): - pass - -if __name__ == '__main__': - logging.basicConfig(level=logging.DEBUG) - parser = ArgumentParser(description='Serve up directory over http') - parser.add_argument('--dir', default='/var/www') - parser.add_argument('--port', type=int, default=8080) - - logging.debug('About to parse arguments') - args = parser.parse_args() - - logging.debug('Serving directory ' + args.dir + ' via HTTP at port ' + str(args.port)) - - chdir(args.dir) - - server = ThreadedHTTPServer(('', args.port), SimpleHTTPRequestHandler) - server.serve_forever() \ No newline at end of file diff --git a/minifi/minifi-c2/minifi-c2-integration-tests/src/test/resources/mocknifi/www/nifi-api/flow/templates b/minifi/minifi-c2/minifi-c2-integration-tests/src/test/resources/mocknifi/www/nifi-api/flow/templates deleted file mode 100644 index 496665b8f5..0000000000 --- a/minifi/minifi-c2/minifi-c2-integration-tests/src/test/resources/mocknifi/www/nifi-api/flow/templates +++ /dev/null @@ -1,53 +0,0 @@ -{ - "generated": "15:41:45 EDT", - "templates": [ - { - "id": "f080ec50-ca32-4b36-8453-5a7145bec4c5", - "permissions": { - "canRead": true, - "canWrite": true - }, - "template": { - "description": "", - "encoding-version": "1.0", - "groupId": "8f8eda5e-015a-1000-a9c1-b7e4fe10ae83", - "id": "f080ec50-ca32-4b36-8453-5a7145bec4c5", - "name": "raspi3.v1", - "timestamp": "03/07/2017 11:13:03 EST", - "uri": "http://localhost:8080/nifi-api/templates/f080ec50-ca32-4b36-8453-5a7145bec4c5" - } - }, - { - "id": "87048385-a6ca-42fe-b2d8-6a563cedd036", - "permissions": { - "canRead": true, - "canWrite": true - }, - "template": { - "description": "", - "encoding-version": "1.0", - "groupId": "8f8eda5e-015a-1000-a9c1-b7e4fe10ae83", - "id": "87048385-a6ca-42fe-b2d8-6a563cedd036", - "name": "raspi2.v1", - "timestamp": "03/17/2017 15:41:33 EDT", - "uri": "http://localhost:8080/nifi-api/templates/87048385-a6ca-42fe-b2d8-6a563cedd036" - } - }, - { - "id": "dd737a3e-333e-40df-a0bc-d7e28c8e6843", - "permissions": { - "canRead": true, - "canWrite": true - }, - "template": { - "description": "", - "encoding-version": "1.0", - "groupId": "8f8eda5e-015a-1000-a9c1-b7e4fe10ae83", - "id": "dd737a3e-333e-40df-a0bc-d7e28c8e6843", - "name": "raspi3.v2", - "timestamp": "03/17/2017 13:22:58 EDT", - "uri": "http://localhost:8080/nifi-api/templates/dd737a3e-333e-40df-a0bc-d7e28c8e6843" - } - } - ] -} \ No newline at end of file diff --git a/minifi/minifi-c2/minifi-c2-integration-tests/src/test/resources/mocknifi/www/nifi-api/templates/87048385-a6ca-42fe-b2d8-6a563cedd036/download b/minifi/minifi-c2/minifi-c2-integration-tests/src/test/resources/mocknifi/www/nifi-api/templates/87048385-a6ca-42fe-b2d8-6a563cedd036/download deleted file mode 100644 index 5d36113282..0000000000 --- a/minifi/minifi-c2/minifi-c2-integration-tests/src/test/resources/mocknifi/www/nifi-api/templates/87048385-a6ca-42fe-b2d8-6a563cedd036/download +++ /dev/null @@ -1,203 +0,0 @@ - - - \ No newline at end of file diff --git a/minifi/minifi-c2/minifi-c2-integration-tests/src/test/resources/mocknifi/www/nifi-api/templates/dd737a3e-333e-40df-a0bc-d7e28c8e6843/download b/minifi/minifi-c2/minifi-c2-integration-tests/src/test/resources/mocknifi/www/nifi-api/templates/dd737a3e-333e-40df-a0bc-d7e28c8e6843/download deleted file mode 100644 index 6b364d0282..0000000000 --- a/minifi/minifi-c2/minifi-c2-integration-tests/src/test/resources/mocknifi/www/nifi-api/templates/dd737a3e-333e-40df-a0bc-d7e28c8e6843/download +++ /dev/null @@ -1,203 +0,0 @@ - - - \ No newline at end of file diff --git a/minifi/minifi-c2/minifi-c2-integration-tests/src/test/resources/mocknifi/www/nifi-api/templates/f080ec50-ca32-4b36-8453-5a7145bec4c5/download b/minifi/minifi-c2/minifi-c2-integration-tests/src/test/resources/mocknifi/www/nifi-api/templates/f080ec50-ca32-4b36-8453-5a7145bec4c5/download deleted file mode 100644 index ad65498cb2..0000000000 --- a/minifi/minifi-c2/minifi-c2-integration-tests/src/test/resources/mocknifi/www/nifi-api/templates/f080ec50-ca32-4b36-8453-5a7145bec4c5/download +++ /dev/null @@ -1,202 +0,0 @@ - - - \ No newline at end of file diff --git a/minifi/minifi-c2/minifi-c2-provider/minifi-c2-provider-delegating/src/main/java/org/apache/nifi/minifi/c2/provider/delegating/DelegatingConfigurationProvider.java b/minifi/minifi-c2/minifi-c2-provider/minifi-c2-provider-delegating/src/main/java/org/apache/nifi/minifi/c2/provider/delegating/DelegatingConfigurationProvider.java index 04f6017c0e..b749461cbc 100644 --- a/minifi/minifi-c2/minifi-c2-provider/minifi-c2-provider-delegating/src/main/java/org/apache/nifi/minifi/c2/provider/delegating/DelegatingConfigurationProvider.java +++ b/minifi/minifi-c2/minifi-c2-provider/minifi-c2-provider-delegating/src/main/java/org/apache/nifi/minifi/c2/provider/delegating/DelegatingConfigurationProvider.java @@ -38,7 +38,6 @@ import java.io.UnsupportedEncodingException; import java.net.HttpURLConnection; import java.net.URLEncoder; import java.nio.charset.StandardCharsets; -import java.security.GeneralSecurityException; import java.util.Comparator; import java.util.List; import java.util.Map; @@ -52,10 +51,6 @@ public class DelegatingConfigurationProvider implements ConfigurationProvider { private final HttpConnector httpConnector; private final ObjectMapper objectMapper; - public DelegatingConfigurationProvider(ConfigurationCache configurationCache, String delegateUrl) throws InvalidParameterException, GeneralSecurityException, IOException { - this(configurationCache, new HttpConnector(delegateUrl)); - } - public DelegatingConfigurationProvider(ConfigurationCache configurationCache, HttpConnector httpConnector) { this.configurationCache = configurationCache; this.httpConnector = httpConnector; diff --git a/minifi/minifi-c2/minifi-c2-provider/minifi-c2-provider-delegating/src/test/java/org/apache/nifi/minifi/c2/provider/delegating/DelegatingConfigurationProviderTest.java b/minifi/minifi-c2/minifi-c2-provider/minifi-c2-provider-delegating/src/test/java/org/apache/nifi/minifi/c2/provider/delegating/DelegatingConfigurationProviderTest.java index cae447a89d..6e88a8fa64 100644 --- a/minifi/minifi-c2/minifi-c2-provider/minifi-c2-provider-delegating/src/test/java/org/apache/nifi/minifi/c2/provider/delegating/DelegatingConfigurationProviderTest.java +++ b/minifi/minifi-c2/minifi-c2-provider/minifi-c2-provider-delegating/src/test/java/org/apache/nifi/minifi/c2/provider/delegating/DelegatingConfigurationProviderTest.java @@ -57,7 +57,7 @@ public class DelegatingConfigurationProviderTest { @BeforeEach public void setup() throws ConfigurationProviderException { - contentType = "text/yml"; + contentType = "test/json"; version = 2; parameters = new HashMap<>(); parameters.put("net", Collections.singletonList("edge")); diff --git a/minifi/minifi-c2/minifi-c2-provider/minifi-c2-provider-nifi-rest/src/main/java/org/apache/nifi/minifi/c2/provider/nifi/rest/NiFiRestConfigurationProvider.java b/minifi/minifi-c2/minifi-c2-provider/minifi-c2-provider-nifi-rest/src/main/java/org/apache/nifi/minifi/c2/provider/nifi/rest/NiFiRestConfigurationProvider.java deleted file mode 100644 index 6067d7ca5e..0000000000 --- a/minifi/minifi-c2/minifi-c2-provider/minifi-c2-provider-nifi-rest/src/main/java/org/apache/nifi/minifi/c2/provider/nifi/rest/NiFiRestConfigurationProvider.java +++ /dev/null @@ -1,188 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.nifi.minifi.c2.provider.nifi.rest; - -import com.fasterxml.jackson.core.JsonFactory; -import org.apache.nifi.minifi.c2.api.Configuration; -import org.apache.nifi.minifi.c2.api.ConfigurationProvider; -import org.apache.nifi.minifi.c2.api.ConfigurationProviderException; -import org.apache.nifi.minifi.c2.api.InvalidParameterException; -import org.apache.nifi.minifi.c2.api.cache.ConfigurationCache; -import org.apache.nifi.minifi.c2.api.cache.WriteableConfiguration; -import org.apache.nifi.minifi.c2.api.util.Pair; -import org.apache.nifi.minifi.c2.provider.util.HttpConnector; -import org.apache.nifi.minifi.commons.schema.ConfigSchema; -import org.apache.nifi.minifi.commons.schema.serialization.SchemaSaver; -import org.apache.nifi.minifi.toolkit.configuration.ConfigMain; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import javax.xml.bind.JAXBException; -import java.io.Closeable; -import java.io.IOException; -import java.io.InputStream; -import java.net.HttpURLConnection; -import java.security.GeneralSecurityException; -import java.util.Arrays; -import java.util.Collections; -import java.util.Comparator; -import java.util.List; -import java.util.Map; -import java.util.Objects; -import java.util.Spliterator; -import java.util.Spliterators; -import java.util.regex.Matcher; -import java.util.regex.Pattern; -import java.util.stream.Collectors; -import java.util.stream.Stream; -import java.util.stream.StreamSupport; - -public class NiFiRestConfigurationProvider implements ConfigurationProvider { - public static final String CONTENT_TYPE = "text/yml"; - private static final Logger logger = LoggerFactory.getLogger(NiFiRestConfigurationProvider.class); - private final JsonFactory jsonFactory = new JsonFactory(); - private final ConfigurationCache configurationCache; - private final HttpConnector httpConnector; - private final String templateNamePattern; - - public NiFiRestConfigurationProvider(ConfigurationCache configurationCache, String nifiUrl, String templateNamePattern) throws InvalidParameterException, GeneralSecurityException, IOException { - this(configurationCache, new HttpConnector(nifiUrl), templateNamePattern); - } - - public NiFiRestConfigurationProvider(ConfigurationCache configurationCache, HttpConnector httpConnector, String templateNamePattern) { - this.configurationCache = configurationCache; - this.httpConnector = httpConnector; - this.templateNamePattern = templateNamePattern; - } - - @Override - public List getContentTypes() { - return Collections.singletonList(CONTENT_TYPE); - } - - @Override - public Configuration getConfiguration(String contentType, Integer version, Map> parameters) throws ConfigurationProviderException { - if (!CONTENT_TYPE.equals(contentType)) { - throw new ConfigurationProviderException("Unsupported content type: " + contentType + " supported value is " + CONTENT_TYPE); - } - String filename = templateNamePattern; - for (Map.Entry> entry : parameters.entrySet()) { - if (entry.getValue().size() != 1) { - throw new InvalidParameterException("Multiple values for same parameter not supported in this provider."); - } - filename = filename.replaceAll(Pattern.quote("${" + entry.getKey() + "}"), entry.getValue().get(0)); - } - int index = filename.indexOf("${"); - while (index != -1) { - int endIndex = filename.indexOf("}", index); - if (endIndex == -1) { - break; - } - String variable = filename.substring(index + 2, endIndex); - if (!"version".equals(variable)) { - throw new InvalidParameterException("Found unsubstituted parameter " + variable); - } - index = endIndex + 1; - } - - String id = null; - if (version == null) { - String filenamePattern = Arrays.stream(filename.split(Pattern.quote("${version}"), -1)).map(Pattern::quote).collect(Collectors.joining("([0-9]+)")); - Pair maxIdAndVersion = getMaxIdAndVersion(filenamePattern); - id = maxIdAndVersion.getFirst(); - version = maxIdAndVersion.getSecond(); - } - filename = filename.replaceAll(Pattern.quote("${version}"), Integer.toString(version)); - WriteableConfiguration configuration = configurationCache.getCacheFileInfo(contentType, parameters).getConfiguration(version); - if (configuration.exists()) { - if (logger.isDebugEnabled()) { - logger.debug("Configuration " + configuration + " exists and can be served from configurationCache."); - } - } else { - if (logger.isDebugEnabled()) { - logger.debug("Configuration " + configuration + " doesn't exist, will need to download and convert template."); - } - if (id == null) { - try { - String tmpFilename = templateNamePattern; - for (Map.Entry> entry : parameters.entrySet()) { - if (entry.getValue().size() != 1) { - throw new InvalidParameterException("Multiple values for same parameter not supported in this provider."); - } - tmpFilename = tmpFilename.replaceAll(Pattern.quote("${" + entry.getKey() + "}"), entry.getValue().get(0)); - } - Pair>, Closeable> streamCloseablePair = getIdAndFilenameStream(); - try { - String finalFilename = filename; - id = streamCloseablePair.getFirst().filter(p -> finalFilename.equals(p.getSecond())).map(Pair::getFirst).findFirst() - .orElseThrow(() -> new InvalidParameterException("Unable to find template named " + finalFilename)); - } finally { - streamCloseablePair.getSecond().close(); - } - } catch (IOException|TemplatesIteratorException e) { - throw new ConfigurationProviderException("Unable to retrieve template list", e); - } - } - - HttpURLConnection urlConnection = httpConnector.get("/templates/" + id + "/download"); - - try (InputStream inputStream = urlConnection.getInputStream()){ - ConfigSchema configSchema = ConfigMain.transformTemplateToSchema(inputStream); - SchemaSaver.saveConfigSchema(configSchema, configuration.getOutputStream()); - } catch (IOException e) { - throw new ConfigurationProviderException("Unable to download template from url " + urlConnection.getURL(), e); - } catch (JAXBException e) { - throw new ConfigurationProviderException("Unable to convert template to yaml", e); - } finally { - urlConnection.disconnect(); - } - } - return configuration; - } - - private Pair>, Closeable> getIdAndFilenameStream() throws ConfigurationProviderException, IOException { - TemplatesIterator templatesIterator = new TemplatesIterator(httpConnector, jsonFactory); - return new Pair<>(StreamSupport.stream(Spliterators.spliteratorUnknownSize(templatesIterator, Spliterator.ORDERED), false), templatesIterator); - } - - private Pair>, Closeable> getIdAndVersionStream(String filenamePattern) throws ConfigurationProviderException, IOException { - Pattern filename = Pattern.compile(filenamePattern); - Pair>, Closeable> streamCloseablePair = getIdAndFilenameStream(); - return new Pair<>(streamCloseablePair.getFirst().map(p -> { - Matcher matcher = filename.matcher(p.getSecond()); - if (!matcher.matches()) { - return null; - } - return new Pair<>(p.getFirst(), Integer.parseInt(matcher.group(1))); - }).filter(Objects::nonNull), streamCloseablePair.getSecond()); - } - - private Pair getMaxIdAndVersion(String filenamePattern) throws ConfigurationProviderException { - try { - Pair>, Closeable> streamCloseablePair = getIdAndVersionStream(filenamePattern); - try { - return streamCloseablePair.getFirst().sorted(Comparator.comparing(p -> ((Pair) p).getSecond()).reversed()).findFirst() - .orElseThrow(() -> new ConfigurationProviderException("Didn't find any templates that matched " + filenamePattern)); - } finally { - streamCloseablePair.getSecond().close(); - } - } catch (IOException|TemplatesIteratorException e) { - throw new ConfigurationProviderException("Unable to retrieve template list", e); - } - } -} diff --git a/minifi/minifi-c2/minifi-c2-provider/minifi-c2-provider-nifi-rest/src/main/java/org/apache/nifi/minifi/c2/provider/nifi/rest/TemplatesIterator.java b/minifi/minifi-c2/minifi-c2-provider/minifi-c2-provider-nifi-rest/src/main/java/org/apache/nifi/minifi/c2/provider/nifi/rest/TemplatesIterator.java deleted file mode 100644 index 947a8538ba..0000000000 --- a/minifi/minifi-c2/minifi-c2-provider/minifi-c2-provider-nifi-rest/src/main/java/org/apache/nifi/minifi/c2/provider/nifi/rest/TemplatesIterator.java +++ /dev/null @@ -1,116 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.nifi.minifi.c2.provider.nifi.rest; - -import com.fasterxml.jackson.core.JsonFactory; -import com.fasterxml.jackson.core.JsonParser; -import com.fasterxml.jackson.core.JsonToken; -import org.apache.nifi.minifi.c2.api.ConfigurationProviderException; -import org.apache.nifi.minifi.c2.api.util.Pair; -import org.apache.nifi.minifi.c2.provider.util.HttpConnector; - -import java.io.Closeable; -import java.io.IOException; -import java.io.InputStream; -import java.net.HttpURLConnection; -import java.util.Iterator; -import java.util.NoSuchElementException; - -public class TemplatesIterator implements Iterator>, Closeable { - public static final String FLOW_TEMPLATES = "/flow/templates"; - - private final HttpURLConnection urlConnection; - private final InputStream inputStream; - private final JsonParser parser; - private Pair next; - - public TemplatesIterator(HttpConnector httpConnector, JsonFactory jsonFactory) throws ConfigurationProviderException, IOException { - urlConnection = httpConnector.get(FLOW_TEMPLATES); - inputStream = urlConnection.getInputStream(); - parser = jsonFactory.createParser(inputStream); - while (parser.nextToken() != JsonToken.END_OBJECT) { - if ("templates".equals(parser.getCurrentName())) { - break; - } - } - next = getNext(); - } - - private Pair getNext() throws IOException { - while (parser.nextToken() != JsonToken.END_ARRAY) { - if ("template".equals(parser.getCurrentName())) { - String id = null; - String name = null; - while (parser.nextToken() != JsonToken.END_OBJECT) { - String currentName = parser.getCurrentName(); - if ("id".equals(currentName)) { - parser.nextToken(); - id = parser.getText(); - } else if ("name".equals(currentName)) { - parser.nextToken(); - name = parser.getText(); - } - } - return new Pair<>(id, name); - } - } - return null; - } - - @Override - public boolean hasNext() { - return next != null; - } - - @Override - public Pair next() { - if (next == null) { - throw new NoSuchElementException(); - } - try { - return next; - } finally { - try { - next = getNext(); - } catch (IOException e) { - throw new TemplatesIteratorException(e); - } - } - } - - @Override - public void close() throws IOException { - if (parser != null) { - try { - parser.close(); - } catch (IOException e) { - //Ignore - } - } - if (inputStream != null) { - try { - inputStream.close(); - } catch (IOException e) { - //Ignore - } - } - if (urlConnection != null) { - urlConnection.disconnect(); - } - } -} diff --git a/minifi/minifi-c2/minifi-c2-provider/minifi-c2-provider-nifi-rest/src/test/java/org/apache/nifi/minifi/c2/provider/nifi/rest/NiFiRestConfigurationProviderTest.java b/minifi/minifi-c2/minifi-c2-provider/minifi-c2-provider-nifi-rest/src/test/java/org/apache/nifi/minifi/c2/provider/nifi/rest/NiFiRestConfigurationProviderTest.java deleted file mode 100644 index b37ca1f3c7..0000000000 --- a/minifi/minifi-c2/minifi-c2-provider/minifi-c2-provider-nifi-rest/src/test/java/org/apache/nifi/minifi/c2/provider/nifi/rest/NiFiRestConfigurationProviderTest.java +++ /dev/null @@ -1,66 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.nifi.minifi.c2.provider.nifi.rest; - -import org.apache.nifi.minifi.c2.api.cache.ConfigurationCache; -import org.apache.nifi.minifi.c2.provider.util.HttpConnector; -import org.junit.jupiter.api.AfterEach; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; - -import java.io.IOException; -import java.nio.file.Files; -import java.nio.file.Path; -import java.util.Collections; -import java.util.Comparator; - -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.mockito.Mockito.mock; - -public class NiFiRestConfigurationProviderTest { - private ConfigurationCache configConfigurationCache; - private HttpConnector httpConnector; - private NiFiRestConfigurationProvider niFiRestConfigurationProvider; - private Path cachePath; - - @BeforeEach - public void setup() throws IOException { - configConfigurationCache = mock(ConfigurationCache.class); - httpConnector = mock(HttpConnector.class); - niFiRestConfigurationProvider = new NiFiRestConfigurationProvider(configConfigurationCache, httpConnector, "${class}.v${version}"); - cachePath = Files.createTempDirectory(NiFiRestConfigurationProviderTest.class.getCanonicalName()); - } - - @AfterEach - public void teardown() throws IOException { - Files.walk(cachePath) - .sorted(Comparator.reverseOrder()) - .forEach(p -> { - try { - Files.deleteIfExists(p); - } catch (IOException e) { - p.toFile().deleteOnExit(); - } - }); - } - - @Test - public void testContentType() { - assertEquals(Collections.singletonList(NiFiRestConfigurationProvider.CONTENT_TYPE), niFiRestConfigurationProvider.getContentTypes()); - } -} diff --git a/minifi/minifi-c2/minifi-c2-provider/minifi-c2-provider-nifi-rest/src/test/java/org/apache/nifi/minifi/c2/provider/nifi/rest/TemplatesIteratorTest.java b/minifi/minifi-c2/minifi-c2-provider/minifi-c2-provider-nifi-rest/src/test/java/org/apache/nifi/minifi/c2/provider/nifi/rest/TemplatesIteratorTest.java deleted file mode 100644 index 9cc70d60bc..0000000000 --- a/minifi/minifi-c2/minifi-c2-provider/minifi-c2-provider-nifi-rest/src/test/java/org/apache/nifi/minifi/c2/provider/nifi/rest/TemplatesIteratorTest.java +++ /dev/null @@ -1,110 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.nifi.minifi.c2.provider.nifi.rest; - -import com.fasterxml.jackson.core.JsonFactory; -import org.apache.nifi.minifi.c2.api.ConfigurationProviderException; -import org.apache.nifi.minifi.c2.api.util.Pair; -import org.apache.nifi.minifi.c2.provider.util.HttpConnector; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; - -import java.io.IOException; -import java.net.HttpURLConnection; -import java.util.ArrayList; -import java.util.List; -import java.util.NoSuchElementException; - -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertFalse; -import static org.junit.jupiter.api.Assertions.assertThrows; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.verify; -import static org.mockito.Mockito.when; - -public class TemplatesIteratorTest { - private JsonFactory jsonFactory; - private HttpURLConnection httpURLConnection; - private HttpConnector httpConnector; - - @BeforeEach - public void setup() throws ConfigurationProviderException { - jsonFactory = new JsonFactory(); - httpURLConnection = mock(HttpURLConnection.class); - httpConnector = mock(HttpConnector.class); - when(httpConnector.get(TemplatesIterator.FLOW_TEMPLATES)).thenReturn(httpURLConnection); - } - - @Test - public void testIteratorNoSuchElementException() throws ConfigurationProviderException, IOException { - when(httpURLConnection.getInputStream()).thenReturn(TemplatesIteratorTest.class.getClassLoader().getResourceAsStream("noTemplates.json")); - - try (TemplatesIterator templatesIterator = new TemplatesIterator(httpConnector, jsonFactory)) { - assertFalse(templatesIterator.hasNext()); - assertThrows(NoSuchElementException.class, templatesIterator::next); - } finally { - verify(httpURLConnection).disconnect(); - } - } - - @Test - public void testIteratorNoTemplates() throws ConfigurationProviderException, IOException { - when(httpURLConnection.getInputStream()).thenReturn(TemplatesIteratorTest.class.getClassLoader().getResourceAsStream("noTemplates.json")); - List> idToNameList = new ArrayList<>(); - try (TemplatesIterator templatesIterator = new TemplatesIterator(httpConnector, jsonFactory)) { - templatesIterator.forEachRemaining(idToNameList::add); - } - assertEquals(0, idToNameList.size()); - - verify(httpURLConnection).disconnect(); - } - - @Test - public void testIteratorSingleTemplate() throws ConfigurationProviderException, IOException { - when(httpURLConnection.getInputStream()).thenReturn(TemplatesIteratorTest.class.getClassLoader().getResourceAsStream("oneTemplate.json")); - List> idToNameList = new ArrayList<>(); - try (TemplatesIterator templatesIterator = new TemplatesIterator(httpConnector, jsonFactory)) { - templatesIterator.forEachRemaining(idToNameList::add); - } - assertEquals(1, idToNameList.size()); - Pair idNamePair = idToNameList.get(0); - assertEquals("d05845ae-ceda-4c50-b7c2-037e42ddf1d3", idNamePair.getFirst()); - assertEquals("raspi3.v1", idNamePair.getSecond()); - - verify(httpURLConnection).disconnect(); - } - - @Test - public void testIteratorTwoTemplates() throws ConfigurationProviderException, IOException { - when(httpURLConnection.getInputStream()).thenReturn(TemplatesIteratorTest.class.getClassLoader().getResourceAsStream("twoTemplates.json")); - List> idToNameList = new ArrayList<>(); - try (TemplatesIterator templatesIterator = new TemplatesIterator(httpConnector, jsonFactory)) { - templatesIterator.forEachRemaining(idToNameList::add); - } - assertEquals(2, idToNameList.size()); - Pair idNamePair = idToNameList.get(0); - assertEquals("d05845ae-ceda-4c50-b7c2-037e42ddf1d3", idNamePair.getFirst()); - assertEquals("raspi3.v1", idNamePair.getSecond()); - - idNamePair = idToNameList.get(1); - assertEquals("9384b48d-85b4-478a-bf3e-64d113f8fbc5", idNamePair.getFirst()); - assertEquals("raspi3.v2", idNamePair.getSecond()); - - verify(httpURLConnection).disconnect(); - } -} diff --git a/minifi/minifi-c2/minifi-c2-provider/minifi-c2-provider-nifi-rest/src/test/resources/noTemplates.json b/minifi/minifi-c2/minifi-c2-provider/minifi-c2-provider-nifi-rest/src/test/resources/noTemplates.json deleted file mode 100644 index ff41507805..0000000000 --- a/minifi/minifi-c2/minifi-c2-provider/minifi-c2-provider-nifi-rest/src/test/resources/noTemplates.json +++ /dev/null @@ -1 +0,0 @@ -{"templates":[],"generated":"14:55:13 EST"} \ No newline at end of file diff --git a/minifi/minifi-c2/minifi-c2-provider/minifi-c2-provider-nifi-rest/src/test/resources/oneTemplate.json b/minifi/minifi-c2/minifi-c2-provider/minifi-c2-provider-nifi-rest/src/test/resources/oneTemplate.json deleted file mode 100644 index 0d981f13e4..0000000000 --- a/minifi/minifi-c2/minifi-c2-provider/minifi-c2-provider-nifi-rest/src/test/resources/oneTemplate.json +++ /dev/null @@ -1,21 +0,0 @@ -{ - "generated": "10:26:04 EST", - "templates": [ - { - "id": "d05845ae-ceda-4c50-b7c2-037e42ddf1d3", - "permissions": { - "canRead": true, - "canWrite": true - }, - "template": { - "description": "", - "encoding-version": "1.0", - "groupId": "8f8eda5e-015a-1000-a9c1-b7e4fe10ae83", - "id": "d05845ae-ceda-4c50-b7c2-037e42ddf1d3", - "name": "raspi3.v1", - "timestamp": "03/02/2017 10:26:01 EST", - "uri": "http://localhost:8081/nifi-api/templates/d05845ae-ceda-4c50-b7c2-037e42ddf1d3" - } - } - ] -} \ No newline at end of file diff --git a/minifi/minifi-c2/minifi-c2-provider/minifi-c2-provider-nifi-rest/src/test/resources/twoTemplates.json b/minifi/minifi-c2/minifi-c2-provider/minifi-c2-provider-nifi-rest/src/test/resources/twoTemplates.json deleted file mode 100644 index 9b4245a618..0000000000 --- a/minifi/minifi-c2/minifi-c2-provider/minifi-c2-provider-nifi-rest/src/test/resources/twoTemplates.json +++ /dev/null @@ -1,37 +0,0 @@ -{ - "generated": "14:53:15 EST", - "templates": [ - { - "id": "d05845ae-ceda-4c50-b7c2-037e42ddf1d3", - "permissions": { - "canRead": true, - "canWrite": true - }, - "template": { - "description": "", - "encoding-version": "1.0", - "groupId": "8f8eda5e-015a-1000-a9c1-b7e4fe10ae83", - "id": "d05845ae-ceda-4c50-b7c2-037e42ddf1d3", - "name": "raspi3.v1", - "timestamp": "03/02/2017 10:26:01 EST", - "uri": "http://localhost:8081/nifi-api/templates/d05845ae-ceda-4c50-b7c2-037e42ddf1d3" - } - }, - { - "id": "9384b48d-85b4-478a-bf3e-64d113f8fbc5", - "permissions": { - "canRead": true, - "canWrite": true - }, - "template": { - "description": "", - "encoding-version": "1.0", - "groupId": "8f8eda5e-015a-1000-a9c1-b7e4fe10ae83", - "id": "9384b48d-85b4-478a-bf3e-64d113f8fbc5", - "name": "raspi3.v2", - "timestamp": "03/02/2017 13:08:14 EST", - "uri": "http://localhost:8081/nifi-api/templates/9384b48d-85b4-478a-bf3e-64d113f8fbc5" - } - } - ] -} diff --git a/minifi/minifi-c2/minifi-c2-provider/pom.xml b/minifi/minifi-c2/minifi-c2-provider/pom.xml index 0cb544cfa0..cfa613072a 100644 --- a/minifi/minifi-c2/minifi-c2-provider/pom.xml +++ b/minifi/minifi-c2/minifi-c2-provider/pom.xml @@ -29,6 +29,5 @@ limitations under the License. minifi-c2-provider-util minifi-c2-provider-cache minifi-c2-provider-delegating - minifi-c2-provider-nifi-rest diff --git a/minifi/minifi-commons/minifi-commons-api/pom.xml b/minifi/minifi-commons/minifi-commons-api/pom.xml index 26196aad73..705c1e1704 100644 --- a/minifi/minifi-commons/minifi-commons-api/pom.xml +++ b/minifi/minifi-commons/minifi-commons-api/pom.xml @@ -16,7 +16,8 @@ ~ limitations under the License. --> - + 4.0.0 org.apache.nifi.minifi diff --git a/minifi/minifi-commons/minifi-commons-api/src/main/java/org/apache/nifi/minifi/commons/api/MiNiFiConstants.java b/minifi/minifi-commons/minifi-commons-api/src/main/java/org/apache/nifi/minifi/commons/api/MiNiFiConstants.java index 1bbac63869..1e41f3711c 100644 --- a/minifi/minifi-commons/minifi-commons-api/src/main/java/org/apache/nifi/minifi/commons/api/MiNiFiConstants.java +++ b/minifi/minifi-commons/minifi-commons-api/src/main/java/org/apache/nifi/minifi/commons/api/MiNiFiConstants.java @@ -19,5 +19,7 @@ package org.apache.nifi.minifi.commons.api; public interface MiNiFiConstants { String BOOTSTRAP_UPDATED_FILE_NAME = "bootstrap-updated.conf"; - String CONFIG_UPDATED_FILE_NAME = "config-updated.yml"; + + String BACKUP_EXTENSION = ".backup"; + String RAW_EXTENSION = ".raw"; } diff --git a/minifi/minifi-nar-bundles/minifi-framework-bundle/minifi-framework/minifi-framework-core/src/main/java/org/apache/nifi/minifi/MiNiFiProperties.java b/minifi/minifi-commons/minifi-commons-api/src/main/java/org/apache/nifi/minifi/commons/api/MiNiFiProperties.java similarity index 72% rename from minifi/minifi-nar-bundles/minifi-framework-bundle/minifi-framework/minifi-framework-core/src/main/java/org/apache/nifi/minifi/MiNiFiProperties.java rename to minifi/minifi-commons/minifi-commons-api/src/main/java/org/apache/nifi/minifi/commons/api/MiNiFiProperties.java index 12eb22a7f8..93bf1c16a2 100644 --- a/minifi/minifi-nar-bundles/minifi-framework-bundle/minifi-framework/minifi-framework-core/src/main/java/org/apache/nifi/minifi/MiNiFiProperties.java +++ b/minifi/minifi-commons/minifi-commons-api/src/main/java/org/apache/nifi/minifi/commons/api/MiNiFiProperties.java @@ -15,17 +15,19 @@ * limitations under the License. */ -package org.apache.nifi.minifi; +package org.apache.nifi.minifi.commons.api; -import static org.apache.nifi.minifi.MiNiFiProperties.ValidatorNames.BOOLEAN_VALIDATOR; -import static org.apache.nifi.minifi.MiNiFiProperties.ValidatorNames.LONG_VALIDATOR; -import static org.apache.nifi.minifi.MiNiFiProperties.ValidatorNames.NON_NEGATIVE_INTEGER_VALIDATOR; -import static org.apache.nifi.minifi.MiNiFiProperties.ValidatorNames.PORT_VALIDATOR; -import static org.apache.nifi.minifi.MiNiFiProperties.ValidatorNames.TIME_PERIOD_VALIDATOR; -import static org.apache.nifi.minifi.MiNiFiProperties.ValidatorNames.VALID; +import static java.util.stream.Collectors.toSet; +import static org.apache.nifi.minifi.commons.api.MiNiFiProperties.ValidatorNames.BOOLEAN_VALIDATOR; +import static org.apache.nifi.minifi.commons.api.MiNiFiProperties.ValidatorNames.LONG_VALIDATOR; +import static org.apache.nifi.minifi.commons.api.MiNiFiProperties.ValidatorNames.NON_NEGATIVE_INTEGER_VALIDATOR; +import static org.apache.nifi.minifi.commons.api.MiNiFiProperties.ValidatorNames.PORT_VALIDATOR; +import static org.apache.nifi.minifi.commons.api.MiNiFiProperties.ValidatorNames.TIME_PERIOD_VALIDATOR; +import static org.apache.nifi.minifi.commons.api.MiNiFiProperties.ValidatorNames.VALID; import java.util.Arrays; import java.util.LinkedHashMap; +import java.util.Set; import java.util.function.Function; import java.util.stream.Collectors; @@ -35,39 +37,6 @@ public enum MiNiFiProperties { LIB_DIR("lib.dir", "./lib", false, false, VALID), CONF_DIR("conf.dir", "./conf", false, false, VALID), GRACEFUL_SHUTDOWN_SECOND("graceful.shutdown.seconds", "20", false, true, NON_NEGATIVE_INTEGER_VALIDATOR), - NIFI_MINIFI_CONFIG("nifi.minifi.config", "./conf/config.yml", false, true, VALID), - NIFI_MINIFI_SECURITY_KEYSTORE("nifi.minifi.security.keystore", null, false, false, VALID), - NIFI_MINIFI_SECURITY_KEYSTORE_TYPE("nifi.minifi.security.keystoreType", null, false, false, VALID), - NIFI_MINIFI_SECURITY_KEYSTORE_PASSWD("nifi.minifi.security.keystorePasswd", null, true, false, VALID), - NIFI_MINIFI_SECURITY_KEY_PASSWD("nifi.minifi.security.keyPasswd", null, true, false, VALID), - NIFI_MINIFI_SECURITY_TRUSTSTORE("nifi.minifi.security.truststore", null, false, false, VALID), - NIFI_MINIFI_SECURITY_TRUSTSTORE_TYPE("nifi.minifi.security.truststoreType", null, false, false, VALID), - NIFI_MINIFI_SECURITY_TRUSTSTORE_PASSWD("nifi.minifi.security.truststorePasswd", null, true, false, VALID), - NIFI_MINIFI_SECURITY_SSL_PROTOCOL("nifi.minifi.security.ssl.protocol", null, false, false, VALID), - NIFI_MINIFI_SENSITIVE_PROPS_KEY("nifi.minifi.sensitive.props.key", null, true, false, VALID), - NIFI_MINIFI_SENSITIVE_PROPS_ALGORITHM("nifi.minifi.sensitive.props.algorithm", null, true, false, VALID), - NIFI_MINIFI_PROVENANCE_REPORTING_COMMENT("nifi.minifi.provenance.reporting.comment", null, false, true, VALID), - NIFI_MINIFI_PROVENANCE_REPORTING_SCHEDULING_STRATEGY("nifi.minifi.provenance.reporting.scheduling.strategy", null, false, true, VALID), - NIFI_MINIFI_PROVENANCE_REPORTING_SCHEDULING_PERIOD("nifi.minifi.provenance.reporting.scheduling.period", null, false, true, TIME_PERIOD_VALIDATOR), - NIFI_MINIFI_PROVENANCE_REPORTING_DESTINATION_URL("nifi.minifi.provenance.reporting.destination.url", null, false, true, VALID), - NIFI_MINIFI_PROVENANCE_REPORTING_INPUT_PORT_NAME("nifi.minifi.provenance.reporting.input.port.name", null, false, true, VALID), - NIFI_MINIFI_PROVENANCE_REPORTING_INSTANCE_URL("nifi.minifi.provenance.reporting.instance.url", null, false, true, VALID), - NIFI_MINIFI_PROVENANCE_REPORTING_BATCH_SIZE("nifi.minifi.provenance.reporting.batch.size", null, false, true, NON_NEGATIVE_INTEGER_VALIDATOR), - NIFI_MINIFI_PROVENANCE_REPORTING_COMMUNICATIONS_TIMEOUT("nifi.minifi.provenance.reporting.communications.timeout", null, false, true, TIME_PERIOD_VALIDATOR), - NIFI_MINIFI_FLOW_USE_PARENT_SSL("nifi.minifi.flow.use.parent.ssl", null, false, true, VALID), - NIFI_MINIFI_NOTIFIER_INGESTORS("nifi.minifi.notifier.ingestors", null, false, true, VALID), - NIFI_MINIFI_NOTIFIER_INGESTORS_FILE_CONFIG_PATH("nifi.minifi.notifier.ingestors.file.config.path", null, false, true, VALID), - NIFI_MINIFI_NOTIFIER_INGESTORS_FILE_POLLING_PERIOD_SECONDS("nifi.minifi.notifier.ingestors.file.polling.period.seconds", null, false, true, NON_NEGATIVE_INTEGER_VALIDATOR), - NIFI_MINIFI_NOTIFIER_INGESTORS_RECEIVE_HTTP_PORT("nifi.minifi.notifier.ingestors.receive.http.port", null, false, true, PORT_VALIDATOR), - NIFI_MINIFI_NOTIFIER_INGESTORS_PULL_HTTP_HOSTNAME("nifi.minifi.notifier.ingestors.pull.http.hostname", null, false, true, VALID), - NIFI_MINIFI_NOTIFIER_INGESTORS_PULL_HTTP_PORT("nifi.minifi.notifier.ingestors.pull.http.port", null, false, true, PORT_VALIDATOR), - NIFI_MINIFI_NOTIFIER_INGESTORS_PULL_HTTP_PATH("nifi.minifi.notifier.ingestors.pull.http.path", null, false, true, VALID), - NIFI_MINIFI_NOTIFIER_INGESTORS_PULL_HTTP_QUERY("nifi.minifi.notifier.ingestors.pull.http.query", null, false, true, VALID), - NIFI_MINIFI_NOTIFIER_INGESTORS_PULL_HTTP_PERIOD_MS("nifi.minifi.notifier.ingestors.pull.http.period.ms", null, false, true, NON_NEGATIVE_INTEGER_VALIDATOR), - NIFI_MINIFI_STATUS_REPORTER_COMPONENTS("nifi.minifi.status.reporter.components", null, false, true, VALID), - NIFI_MINIFI_STATUS_REPORTER_LOG_QUERY("nifi.minifi.status.reporter.log.query", null, false, true, VALID), - NIFI_MINIFI_STATUS_REPORTER_LOG_LEVEL("nifi.minifi.status.reporter.log.level", null, false, true, VALID), - NIFI_MINIFI_STATUS_REPORTER_LOG_PERIOD("nifi.minifi.status.reporter.log.period", null, false, true, VALID), JAVA_ARG_1("java.arg.1", null, false, true, VALID), JAVA_ARG_2("java.arg.2", null, false, true, VALID), JAVA_ARG_3("java.arg.3", null, false, true, VALID), @@ -82,7 +51,24 @@ public enum MiNiFiProperties { JAVA_ARG_12("java.arg.12", null, false, true, VALID), JAVA_ARG_13("java.arg.13", null, false, true, VALID), JAVA_ARG_14("java.arg.14", null, false, true, VALID), + NIFI_MINIFI_FLOW_CONFIG("nifi.minifi.flow.config", "./conf/flow.json.gz", false, false, VALID), + NIFI_MINIFI_FLOW_MAX_CONCURRENT_THREADS("nifi.minifi.flow.max.concurrent.threads", null, false, true, VALID), + NIFI_MINIFI_SECURITY_KEYSTORE("nifi.minifi.security.keystore", null, false, false, VALID), + NIFI_MINIFI_SECURITY_KEYSTORE_TYPE("nifi.minifi.security.keystoreType", null, false, false, VALID), + NIFI_MINIFI_SECURITY_KEYSTORE_PASSWD("nifi.minifi.security.keystorePasswd", null, true, false, VALID), + NIFI_MINIFI_SECURITY_KEY_PASSWD("nifi.minifi.security.keyPasswd", null, true, false, VALID), + NIFI_MINIFI_SECURITY_TRUSTSTORE("nifi.minifi.security.truststore", null, false, false, VALID), + NIFI_MINIFI_SECURITY_TRUSTSTORE_TYPE("nifi.minifi.security.truststoreType", null, false, false, VALID), + NIFI_MINIFI_SECURITY_TRUSTSTORE_PASSWD("nifi.minifi.security.truststorePasswd", null, true, false, VALID), + NIFI_MINIFI_SECURITY_SSL_PROTOCOL("nifi.minifi.security.ssl.protocol", null, false, false, VALID), + NIFI_MINIFI_FLOW_USE_PARENT_SSL("nifi.minifi.flow.use.parent.ssl", null, false, true, VALID), + NIFI_MINIFI_SENSITIVE_PROPS_KEY("nifi.minifi.sensitive.props.key", null, true, false, VALID), + NIFI_MINIFI_SENSITIVE_PROPS_ALGORITHM("nifi.minifi.sensitive.props.algorithm", null, true, false, VALID), C2_ENABLE("c2.enable", "false", false, true, BOOLEAN_VALIDATOR), + C2_AGENT_HEARTBEAT_PERIOD("c2.agent.heartbeat.period", "1000", false, true, LONG_VALIDATOR), + C2_AGENT_CLASS("c2.agent.class", "", false, true, VALID), + C2_AGENT_IDENTIFIER("c2.agent.identifier", null, false, true, VALID), + C2_FULL_HEARTBEAT("c2.full.heartbeat", "true", false, true, BOOLEAN_VALIDATOR), C2_REST_URL("c2.rest.url", "", false, true, VALID), C2_REST_URL_ACK("c2.rest.url.ack", "", false, true, VALID), C2_REST_PATH_BASE("c2.rest.path.base", "", false, true, VALID), @@ -93,13 +79,11 @@ public enum MiNiFiProperties { C2_REST_CALL_TIMEOUT("c2.rest.callTimeout", "10 sec", false, true, TIME_PERIOD_VALIDATOR), C2_MAX_IDLE_CONNECTIONS("c2.rest.maxIdleConnections", "5", false, true, NON_NEGATIVE_INTEGER_VALIDATOR), C2_KEEP_ALIVE_DURATION("c2.rest.keepAliveDuration", "5 min", false, true, TIME_PERIOD_VALIDATOR), - C2_AGENT_HEARTBEAT_PERIOD("c2.agent.heartbeat.period", "1000", false, true, LONG_VALIDATOR), - C2_AGENT_CLASS("c2.agent.class", "", false, true, VALID), + C2_REST_HTTP_HEADERS("c2.rest.http.headers", "Accept:application/json", false, true, VALID), C2_CONFIG_DIRECTORY("c2.config.directory", "./conf", false, true, VALID), C2_RUNTIME_MANIFEST_IDENTIFIER("c2.runtime.manifest.identifier", "", false, true, VALID), C2_RUNTIME_TYPE("c2.runtime.type", "", false, true, VALID), - C2_AGENT_IDENTIFIER("c2.agent.identifier", null, false, true, VALID), - C2_FULL_HEARTBEAT("c2.full.heartbeat", "true", false, true, BOOLEAN_VALIDATOR), + C2_ASSET_DIRECTORY("c2.asset.directory", "./asset", false, true, VALID), C2_SECURITY_TRUSTSTORE_LOCATION("c2.security.truststore.location", "", false, false, VALID), C2_SECURITY_TRUSTSTORE_PASSWORD("c2.security.truststore.password", "", true, false, VALID), C2_SECURITY_TRUSTSTORE_TYPE("c2.security.truststore.type", "JKS", false, false, VALID), @@ -107,11 +91,35 @@ public enum MiNiFiProperties { C2_SECURITY_KEYSTORE_PASSWORD("c2.security.keystore.password", "", true, false, VALID), C2_SECURITY_KEYSTORE_TYPE("c2.security.keystore.type", "JKS", false, false, VALID), C2_REQUEST_COMPRESSION("c2.request.compression", "none", false, true, VALID), - C2_ASSET_DIRECTORY("c2.asset.directory", "./asset", false, true, VALID); + NIFI_MINIFI_NOTIFIER_INGESTORS("nifi.minifi.notifier.ingestors", null, false, true, VALID), + NIFI_MINIFI_NOTIFIER_INGESTORS_FILE_CONFIG_PATH("nifi.minifi.notifier.ingestors.file.config.path", null, false, true, VALID), + NIFI_MINIFI_NOTIFIER_INGESTORS_FILE_POLLING_PERIOD_SECONDS("nifi.minifi.notifier.ingestors.file.polling.period.seconds", null, false, true, NON_NEGATIVE_INTEGER_VALIDATOR), + NIFI_MINIFI_NOTIFIER_INGESTORS_RECEIVE_HTTP_PORT("nifi.minifi.notifier.ingestors.receive.http.port", null, false, true, PORT_VALIDATOR), + NIFI_MINIFI_NOTIFIER_INGESTORS_PULL_HTTP_HOSTNAME("nifi.minifi.notifier.ingestors.pull.http.hostname", null, false, true, VALID), + NIFI_MINIFI_NOTIFIER_INGESTORS_PULL_HTTP_PORT("nifi.minifi.notifier.ingestors.pull.http.port", null, false, true, PORT_VALIDATOR), + NIFI_MINIFI_NOTIFIER_INGESTORS_PULL_HTTP_PATH("nifi.minifi.notifier.ingestors.pull.http.path", null, false, true, VALID), + NIFI_MINIFI_NOTIFIER_INGESTORS_PULL_HTTP_QUERY("nifi.minifi.notifier.ingestors.pull.http.query", null, false, true, VALID), + NIFI_MINIFI_NOTIFIER_INGESTORS_PULL_HTTP_PERIOD_MS("nifi.minifi.notifier.ingestors.pull.http.period.ms", null, false, true, NON_NEGATIVE_INTEGER_VALIDATOR), + NIFI_MINIFI_NOTIFIER_INGESTORS_PULL_HTTP_HEADERS("nifi.minifi.notifier.ingestors.pull.http.headers", null, false, true, NON_NEGATIVE_INTEGER_VALIDATOR), + NIFI_MINIFI_STATUS_REPORTER_COMPONENTS("nifi.minifi.status.reporter.components", null, false, true, VALID), + NIFI_MINIFI_STATUS_REPORTER_LOG_QUERY("nifi.minifi.status.reporter.log.query", null, false, true, VALID), + NIFI_MINIFI_STATUS_REPORTER_LOG_LEVEL("nifi.minifi.status.reporter.log.level", null, false, true, VALID), + NIFI_MINIFI_STATUS_REPORTER_LOG_PERIOD("nifi.minifi.status.reporter.log.period", null, false, true, VALID), + NIFI_MINIFI_PROVENANCE_REPORTING_COMMENT("nifi.minifi.provenance.reporting.comment", null, false, true, VALID), + NIFI_MINIFI_PROVENANCE_REPORTING_SCHEDULING_STRATEGY("nifi.minifi.provenance.reporting.scheduling.strategy", null, false, true, VALID), + NIFI_MINIFI_PROVENANCE_REPORTING_SCHEDULING_PERIOD("nifi.minifi.provenance.reporting.scheduling.period", null, false, true, TIME_PERIOD_VALIDATOR), + NIFI_MINIFI_PROVENANCE_REPORTING_DESTINATION_URL("nifi.minifi.provenance.reporting.destination.url", null, false, true, VALID), + NIFI_MINIFI_PROVENANCE_REPORTING_INPUT_PORT_NAME("nifi.minifi.provenance.reporting.input.port.name", null, false, true, VALID), + NIFI_MINIFI_PROVENANCE_REPORTING_INSTANCE_URL("nifi.minifi.provenance.reporting.instance.url", null, false, true, VALID), + NIFI_MINIFI_PROVENANCE_REPORTING_COMPRESS_EVENTS("nifi.minifi.provenance.reporting.compress.events", null, false, true, VALID), + NIFI_MINIFI_PROVENANCE_REPORTING_BATCH_SIZE("nifi.minifi.provenance.reporting.batch.size", null, false, true, NON_NEGATIVE_INTEGER_VALIDATOR), + NIFI_MINIFI_PROVENANCE_REPORTING_COMMUNICATIONS_TIMEOUT("nifi.minifi.provenance.reporting.communications.timeout", null, false, true, TIME_PERIOD_VALIDATOR); - public static final LinkedHashMap PROPERTIES_BY_KEY = Arrays.stream(MiNiFiProperties.values()) - .sorted() - .collect(Collectors.toMap(MiNiFiProperties::getKey, Function.identity(), (x, y) -> y, LinkedHashMap::new)); + // These are not "real" bootstrap properties. They are generated runtime in bootstrap and populated into also generated minifi.properties + public static final String MINIFI_BOOTSTRAP_FILE_PATH = "nifi.minifi.bootstrap.file"; + public static final String MINIFI_LOG_DIRECTORY = "nifi.minifi.log.directory"; + public static final String MINIFI_APP_LOG_FILE = "nifi.minifi.app.log.file"; + public static final String MINIFI_BOOTSTRAP_LOG_FILE = "nifi.minifi.bootstrap.log.file"; private final String key; private final String defaultValue; @@ -127,6 +135,43 @@ public enum MiNiFiProperties { this.validator = validator; } + public static LinkedHashMap sortedPropertiesByKey() { + return Arrays.stream(MiNiFiProperties.values()) + .sorted() + .collect(Collectors.toMap(MiNiFiProperties::getKey, Function.identity(), (x, y) -> y, LinkedHashMap::new)); + } + + public static Set securityPropertyKeys() { + return Set.of( + NIFI_MINIFI_SECURITY_KEYSTORE, + NIFI_MINIFI_SECURITY_KEYSTORE_TYPE, + NIFI_MINIFI_SECURITY_KEYSTORE_PASSWD, + NIFI_MINIFI_SECURITY_KEY_PASSWD, + NIFI_MINIFI_SECURITY_TRUSTSTORE, + NIFI_MINIFI_SECURITY_TRUSTSTORE_TYPE, + NIFI_MINIFI_SECURITY_TRUSTSTORE_PASSWD, + NIFI_MINIFI_SECURITY_SSL_PROTOCOL) + .stream() + .map(MiNiFiProperties::getKey) + .collect(toSet()); + } + + public static Set provenanceReportingPropertyKeys() { + return Set.of( + NIFI_MINIFI_PROVENANCE_REPORTING_COMMENT, + NIFI_MINIFI_PROVENANCE_REPORTING_SCHEDULING_STRATEGY, + NIFI_MINIFI_PROVENANCE_REPORTING_SCHEDULING_PERIOD, + NIFI_MINIFI_PROVENANCE_REPORTING_DESTINATION_URL, + NIFI_MINIFI_PROVENANCE_REPORTING_INPUT_PORT_NAME, + NIFI_MINIFI_PROVENANCE_REPORTING_INSTANCE_URL, + NIFI_MINIFI_PROVENANCE_REPORTING_COMPRESS_EVENTS, + NIFI_MINIFI_PROVENANCE_REPORTING_BATCH_SIZE, + NIFI_MINIFI_PROVENANCE_REPORTING_COMMUNICATIONS_TIMEOUT) + .stream() + .map(MiNiFiProperties::getKey) + .collect(toSet()); + } + public String getKey() { return key; } diff --git a/minifi/minifi-commons/minifi-commons-framework/.gitignore b/minifi/minifi-commons/minifi-commons-framework/.gitignore new file mode 100644 index 0000000000..5ff6309b71 --- /dev/null +++ b/minifi/minifi-commons/minifi-commons-framework/.gitignore @@ -0,0 +1,38 @@ +target/ +!.mvn/wrapper/maven-wrapper.jar +!**/src/main/**/target/ +!**/src/test/**/target/ + +### IntelliJ IDEA ### +.idea/modules.xml +.idea/jarRepositories.xml +.idea/compiler.xml +.idea/libraries/ +*.iws +*.iml +*.ipr + +### Eclipse ### +.apt_generated +.classpath +.factorypath +.project +.settings +.springBeans +.sts4-cache + +### NetBeans ### +/nbproject/private/ +/nbbuild/ +/dist/ +/nbdist/ +/.nb-gradle/ +build/ +!**/src/main/**/build/ +!**/src/test/**/build/ + +### VS Code ### +.vscode/ + +### Mac OS ### +.DS_Store \ No newline at end of file diff --git a/minifi/minifi-c2/minifi-c2-provider/minifi-c2-provider-nifi-rest/pom.xml b/minifi/minifi-commons/minifi-commons-framework/pom.xml similarity index 52% rename from minifi/minifi-c2/minifi-c2-provider/minifi-c2-provider-nifi-rest/pom.xml rename to minifi/minifi-commons/minifi-commons-framework/pom.xml index dc6f7b03e1..bc1e9f304e 100644 --- a/minifi/minifi-c2/minifi-c2-provider/minifi-c2-provider-nifi-rest/pom.xml +++ b/minifi/minifi-commons/minifi-commons-framework/pom.xml @@ -15,45 +15,39 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. --> - + 4.0.0 - minifi-c2-provider org.apache.nifi.minifi + minifi-commons 2.0.0-SNAPSHOT - minifi-c2-provider-nifi-rest - jar + + minifi-commons-framework org.apache.nifi.minifi - minifi-c2-api + minifi-commons-api + + + org.apache.nifi + nifi-framework-core-api + + + org.apache.nifi + nifi-api + + + org.apache.nifi + nifi-property-utils ${project.version} - org.apache.nifi.minifi - minifi-c2-provider-util - ${project.version} - - - com.fasterxml.jackson.core - jackson-core - - - org.apache.nifi.minifi - minifi-toolkit-configuration - ${project.version} - - - org.slf4j - slf4j-log4j12 - - - org.glassfish.jersey.media - jersey-media-json-jackson - - + com.fasterxml.jackson.module + jackson-module-jaxb-annotations @@ -64,12 +58,11 @@ limitations under the License. apache-rat-plugin - src/test/resources/noTemplates.json - src/test/resources/oneTemplate.json - src/test/resources/twoTemplates.json + **/org.mockito.plugins.MockMaker + **/default_flow.json - + \ No newline at end of file diff --git a/minifi/minifi-commons/minifi-commons-framework/src/main/java/org/apache/nifi/minifi/commons/service/FlowEnrichService.java b/minifi/minifi-commons/minifi-commons-framework/src/main/java/org/apache/nifi/minifi/commons/service/FlowEnrichService.java new file mode 100644 index 0000000000..3fd5e06474 --- /dev/null +++ b/minifi/minifi-commons/minifi-commons-framework/src/main/java/org/apache/nifi/minifi/commons/service/FlowEnrichService.java @@ -0,0 +1,290 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.nifi.minifi.commons.service; + +import static java.lang.Boolean.FALSE; +import static java.lang.Boolean.parseBoolean; +import static java.nio.charset.StandardCharsets.UTF_8; +import static java.util.Map.entry; +import static java.util.Optional.empty; +import static java.util.Optional.ofNullable; +import static java.util.UUID.randomUUID; +import static java.util.stream.Collectors.toMap; +import static org.apache.commons.lang3.StringUtils.EMPTY; + +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.core.JsonFactory; +import com.fasterxml.jackson.core.JsonGenerator; +import com.fasterxml.jackson.databind.DeserializationFeature; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.module.jaxb.JaxbAnnotationIntrospector; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import org.apache.commons.lang3.StringUtils; +import org.apache.nifi.controller.flow.VersionedDataflow; +import org.apache.nifi.controller.serialization.FlowSerializationException; +import org.apache.nifi.flow.Bundle; +import org.apache.nifi.flow.ComponentType; +import org.apache.nifi.flow.ControllerServiceAPI; +import org.apache.nifi.flow.ScheduledState; +import org.apache.nifi.flow.VersionedComponent; +import org.apache.nifi.flow.VersionedControllerService; +import org.apache.nifi.flow.VersionedProcessGroup; +import org.apache.nifi.flow.VersionedReportingTask; +import org.apache.nifi.logging.LogLevel; +import org.apache.nifi.minifi.commons.api.MiNiFiProperties; +import org.apache.nifi.properties.ReadableProperties; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class FlowEnrichService { + + static final String DEFAULT_SSL_CONTEXT_SERVICE_NAME = "SSL Context Service"; + + static final String COMMON_SSL_CONTEXT_SERVICE_NAME = "SSL-Context-Service"; + static final String COMMON_SSL_CONTEXT_SERVICE_ID = "generated-common-ssl-context"; + static final String SITE_TO_SITE_PROVENANCE_REPORTING_TASK_NAME = "Site-To-Site-Provenance-Reporting"; + static final String SITE_TO_SITE_PROVENANCE_REPORTING_TASK_ID = "generated-s2s-provenance-reporting-task"; + + private static final Logger LOG = LoggerFactory.getLogger(FlowEnrichService.class); + + private static final String NIFI_BUNDLE_GROUP = "org.apache.nifi"; + private static final String STANDARD_RESTRICTED_SSL_CONTEXT_SERVICE = "org.apache.nifi.ssl.StandardRestrictedSSLContextService"; + private static final String RESTRICTED_SSL_CONTEXT_SERVICE_API = "org.apache.nifi.ssl.RestrictedSSLContextService"; + private static final String SSL_CONTEXT_SERVICE_API = "org.apache.nifi.ssl.SSLContextService"; + private static final String SSL_CONTEXT_SERVICE_NAR = "nifi-ssl-context-service-nar"; + private static final String STANDARD_SERVICES_API_NAR_ARTIFACT = "nifi-standard-services-api-nar"; + private static final String SITE_TO_SITE_PROVENANCE_REPORTING_TASK = "org.apache.nifi.reporting.SiteToSiteProvenanceReportingTask"; + private static final String SITE_TO_SITE_REPORTING_NAR_ARTIFACT = "nifi-site-to-site-reporting-nar"; + private static final String PROVENANCE_REPORTING_TASK_PROTOCOL = "HTTP"; + private static final String PROVENANCE_REPORTING_TASK_BEGINNING_OF_STREAM = "beginning-of-stream"; + + private final ReadableProperties minifiProperties; + + public FlowEnrichService(ReadableProperties minifiProperties) { + this.minifiProperties = minifiProperties; + } + + public byte[] enrichFlow(byte[] flowCandidate) { + if (LOG.isDebugEnabled()) { + LOG.debug("Enriching flow with content: \n{}", new String(flowCandidate, UTF_8)); + } + + VersionedDataflow versionedDataflow = parseVersionedDataflow(flowCandidate); + + Optional maxConcurrentThreads = ofNullable(minifiProperties.getProperty(MiNiFiProperties.NIFI_MINIFI_FLOW_MAX_CONCURRENT_THREADS.getKey())) + .map(Integer::parseInt); + maxConcurrentThreads.ifPresent(versionedDataflow::setMaxTimerDrivenThreadCount); + + VersionedProcessGroup rootGroup = versionedDataflow.getRootGroup(); + if (rootGroup.getIdentifier() == null) { + rootGroup.setIdentifier(randomUUID().toString()); + } + if (rootGroup.getInstanceIdentifier() == null) { + rootGroup.setInstanceIdentifier(randomUUID().toString()); + } + + Optional commonSslControllerService = createCommonSslControllerService(); + commonSslControllerService + .ifPresent(sslControllerService -> { + List currentControllerServices = ofNullable(versionedDataflow.getControllerServices()).orElseGet(ArrayList::new); + currentControllerServices.add(sslControllerService); + versionedDataflow.setControllerServices(currentControllerServices); + }); + + commonSslControllerService + .filter(__ -> parseBoolean(minifiProperties.getProperty(MiNiFiProperties.NIFI_MINIFI_FLOW_USE_PARENT_SSL.getKey()))) + .map(VersionedComponent::getInstanceIdentifier) + .ifPresent(commonSslControllerServiceInstanceId -> overrideProcessorsSslControllerService(rootGroup, commonSslControllerServiceInstanceId)); + + createProvenanceReportingTask(commonSslControllerService.map(VersionedComponent::getInstanceIdentifier).orElse(EMPTY)) + .ifPresent(provenanceReportingTask -> { + List currentReportingTasks = ofNullable(versionedDataflow.getReportingTasks()).orElseGet(ArrayList::new); + currentReportingTasks.add(provenanceReportingTask); + versionedDataflow.setReportingTasks(currentReportingTasks); + }); + + byte[] enrichedFlow = toByteArray(versionedDataflow); + if (LOG.isDebugEnabled()) { + LOG.debug("Enriched flow with content: \n{}", new String(enrichedFlow, UTF_8)); + } + return enrichedFlow; + } + + private VersionedDataflow parseVersionedDataflow(byte[] flow) { + try { + ObjectMapper objectMapper = deserializationObjectMapper(); + return objectMapper.readValue(flow, VersionedDataflow.class); + } catch (final Exception e) { + throw new FlowSerializationException("Could not parse flow as a VersionedDataflow", e); + } + } + + private ObjectMapper deserializationObjectMapper() { + ObjectMapper objectMapper = new ObjectMapper(); + objectMapper.setAnnotationIntrospector(new JaxbAnnotationIntrospector(objectMapper.getTypeFactory())); + objectMapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false); + return objectMapper; + } + + private Optional createCommonSslControllerService() { + if (!parentSslEnabled()) { + LOG.debug("Parent SSL is disabled, skip creating parent SSL Controller Service"); + return empty(); + } + + LOG.debug("Parent SSL is enabled, creating parent SSL Controller Service"); + VersionedControllerService sslControllerService = new VersionedControllerService(); + sslControllerService.setIdentifier(randomUUID().toString()); + sslControllerService.setInstanceIdentifier(COMMON_SSL_CONTEXT_SERVICE_ID); + sslControllerService.setName(COMMON_SSL_CONTEXT_SERVICE_NAME); + sslControllerService.setComments(EMPTY); + sslControllerService.setType(STANDARD_RESTRICTED_SSL_CONTEXT_SERVICE); + sslControllerService.setScheduledState(ScheduledState.ENABLED); + sslControllerService.setBulletinLevel(LogLevel.WARN.name()); + sslControllerService.setComponentType(ComponentType.CONTROLLER_SERVICE); + sslControllerService.setBundle(createBundle(SSL_CONTEXT_SERVICE_NAR)); + sslControllerService.setProperties(sslControllerServiceProperties()); + sslControllerService.setControllerServiceApis(List.of( + controllerServiceAPI(SSL_CONTEXT_SERVICE_API, createBundle(STANDARD_SERVICES_API_NAR_ARTIFACT)), + controllerServiceAPI(RESTRICTED_SSL_CONTEXT_SERVICE_API, createBundle(STANDARD_SERVICES_API_NAR_ARTIFACT)) + )); + sslControllerService.setPropertyDescriptors(Map.of()); + return Optional.of(sslControllerService); + } + + private boolean parentSslEnabled() { + return MiNiFiProperties.securityPropertyKeys().stream() + .map(minifiProperties::getProperty) + .allMatch(StringUtils::isNotBlank); + } + + private Map sslControllerServiceProperties() { + return Map.of( + "Keystore Filename", minifiProperties.getProperty(MiNiFiProperties.NIFI_MINIFI_SECURITY_KEYSTORE.getKey()), + "Keystore Password", minifiProperties.getProperty(MiNiFiProperties.NIFI_MINIFI_SECURITY_KEYSTORE_PASSWD.getKey()), + "key-password", minifiProperties.getProperty(MiNiFiProperties.NIFI_MINIFI_SECURITY_KEY_PASSWD.getKey()), + "Keystore Type", minifiProperties.getProperty(MiNiFiProperties.NIFI_MINIFI_SECURITY_KEYSTORE_TYPE.getKey()), + "Truststore Filename", minifiProperties.getProperty(MiNiFiProperties.NIFI_MINIFI_SECURITY_TRUSTSTORE.getKey()), + "Truststore Password", minifiProperties.getProperty(MiNiFiProperties.NIFI_MINIFI_SECURITY_TRUSTSTORE_PASSWD.getKey()), + "Truststore Type", minifiProperties.getProperty(MiNiFiProperties.NIFI_MINIFI_SECURITY_TRUSTSTORE_TYPE.getKey()), + "SSL Protocol", minifiProperties.getProperty(MiNiFiProperties.NIFI_MINIFI_SECURITY_SSL_PROTOCOL.getKey()) + ); + } + + private ControllerServiceAPI controllerServiceAPI(String type, Bundle bundle) { + ControllerServiceAPI controllerServiceAPI = new ControllerServiceAPI(); + controllerServiceAPI.setType(type); + controllerServiceAPI.setBundle(bundle); + return controllerServiceAPI; + } + + private void overrideProcessorsSslControllerService(VersionedProcessGroup processGroup, String commonSslControllerServiceInstanceId) { + LOG.debug("Use parent SSL is enabled, overriding processors' SSL Controller service to {}", commonSslControllerServiceInstanceId); + processGroup.getProcessors() + .forEach(processor -> processor.getProperties() + .replace( + DEFAULT_SSL_CONTEXT_SERVICE_NAME, + processor.getProperties().get(DEFAULT_SSL_CONTEXT_SERVICE_NAME), + commonSslControllerServiceInstanceId)); + processGroup.getProcessGroups() + .forEach(childProcessGroup -> overrideProcessorsSslControllerService(childProcessGroup, commonSslControllerServiceInstanceId)); + } + + private Optional createProvenanceReportingTask(String sslControllerServiceIdentifier) { + if (!provenanceReportingEnabled()) { + LOG.debug("Provenance reporting task is disabled, skip creating provenance reporting task"); + return empty(); + } + LOG.debug("Provenance reporting task is enabled, creating provenance reporting task"); + + VersionedReportingTask reportingTask = new VersionedReportingTask(); + reportingTask.setIdentifier(randomUUID().toString()); + reportingTask.setInstanceIdentifier(SITE_TO_SITE_PROVENANCE_REPORTING_TASK_ID); + reportingTask.setName(SITE_TO_SITE_PROVENANCE_REPORTING_TASK_NAME); + reportingTask.setComments(minifiProperties.getProperty(MiNiFiProperties.NIFI_MINIFI_PROVENANCE_REPORTING_COMMENT.getKey())); + reportingTask.setType(SITE_TO_SITE_PROVENANCE_REPORTING_TASK); + reportingTask.setBundle(createBundle(SITE_TO_SITE_REPORTING_NAR_ARTIFACT)); + reportingTask.setScheduledState(ScheduledState.RUNNING); + reportingTask.setSchedulingStrategy(minifiProperties.getProperty(MiNiFiProperties.NIFI_MINIFI_PROVENANCE_REPORTING_SCHEDULING_STRATEGY.getKey())); + reportingTask.setSchedulingPeriod(minifiProperties.getProperty(MiNiFiProperties.NIFI_MINIFI_PROVENANCE_REPORTING_SCHEDULING_PERIOD.getKey())); + reportingTask.setComponentType(ComponentType.REPORTING_TASK); + reportingTask.setProperties(provenanceReportingTaskProperties(sslControllerServiceIdentifier)); + reportingTask.setPropertyDescriptors(Map.of()); + return Optional.of(reportingTask); + } + + private boolean provenanceReportingEnabled() { + return MiNiFiProperties.provenanceReportingPropertyKeys().stream() + .map(minifiProperties::getProperty) + .allMatch(StringUtils::isNotBlank); + } + + private Bundle createBundle(String artifact) { + Bundle bundle = new Bundle(); + bundle.setGroup(NIFI_BUNDLE_GROUP); + bundle.setArtifact(artifact); + bundle.setVersion(EMPTY); + return bundle; + } + + private Map provenanceReportingTaskProperties(String sslControllerServiceIdentifier) { + return List.of( + entry("Input Port Name", minifiProperties.getProperty(MiNiFiProperties.NIFI_MINIFI_PROVENANCE_REPORTING_INPUT_PORT_NAME.getKey())), + entry("s2s-transport-protocol", PROVENANCE_REPORTING_TASK_PROTOCOL), + entry("Platform", "nifi"), + entry("Destination URL", minifiProperties.getProperty(MiNiFiProperties.NIFI_MINIFI_PROVENANCE_REPORTING_DESTINATION_URL.getKey())), + entry("include-null-values", FALSE.toString()), + entry("Compress Events", minifiProperties.getProperty(MiNiFiProperties.NIFI_MINIFI_PROVENANCE_REPORTING_COMPRESS_EVENTS.getKey())), + entry("Batch Size", minifiProperties.getProperty(MiNiFiProperties.NIFI_MINIFI_PROVENANCE_REPORTING_BATCH_SIZE.getKey())), + entry("Communications Timeout", minifiProperties.getProperty(MiNiFiProperties.NIFI_MINIFI_PROVENANCE_REPORTING_COMMUNICATIONS_TIMEOUT.getKey())), + entry("start-position", PROVENANCE_REPORTING_TASK_BEGINNING_OF_STREAM), + entry("Instance URL", minifiProperties.getProperty(MiNiFiProperties.NIFI_MINIFI_PROVENANCE_REPORTING_INSTANCE_URL.getKey())), + entry("SSL Context Service", sslControllerServiceIdentifier)) + .stream() + .filter(entry -> StringUtils.isNotBlank(entry.getValue())) + .collect(toMap(Map.Entry::getKey, Map.Entry::getValue)); + } + + private byte[] toByteArray(VersionedDataflow versionedDataflow) { + try (ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream()) { + JsonFactory factory = new JsonFactory(); + JsonGenerator generator = factory.createGenerator(byteArrayOutputStream); + generator.setCodec(serializationObjectMapper()); + generator.writeObject(versionedDataflow); + generator.flush(); + byteArrayOutputStream.flush(); + return byteArrayOutputStream.toByteArray(); + } catch (IOException e) { + throw new RuntimeException("Unable to convert flow to byte array", e); + } + } + + private ObjectMapper serializationObjectMapper() { + ObjectMapper objectMapper = new ObjectMapper(); + objectMapper.setSerializationInclusion(JsonInclude.Include.NON_NULL); + objectMapper.setDefaultPropertyInclusion(JsonInclude.Value.construct(JsonInclude.Include.NON_NULL, JsonInclude.Include.NON_NULL)); + objectMapper.setAnnotationIntrospector(new JaxbAnnotationIntrospector(objectMapper.getTypeFactory())); + objectMapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false); + return objectMapper; + } +} diff --git a/minifi/minifi-commons/minifi-commons-framework/src/main/java/org/apache/nifi/minifi/commons/util/FlowUpdateUtils.java b/minifi/minifi-commons/minifi-commons-framework/src/main/java/org/apache/nifi/minifi/commons/util/FlowUpdateUtils.java new file mode 100644 index 0000000000..6a44cb48e1 --- /dev/null +++ b/minifi/minifi-commons/minifi-commons-framework/src/main/java/org/apache/nifi/minifi/commons/util/FlowUpdateUtils.java @@ -0,0 +1,88 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.nifi.minifi.commons.util; + +import static java.nio.charset.StandardCharsets.UTF_8; +import static java.nio.file.Files.copy; +import static java.nio.file.Files.deleteIfExists; +import static java.nio.file.Files.exists; +import static java.nio.file.Files.newOutputStream; +import static java.nio.file.StandardCopyOption.REPLACE_EXISTING; +import static java.nio.file.StandardOpenOption.CREATE; +import static java.nio.file.StandardOpenOption.TRUNCATE_EXISTING; +import static java.nio.file.StandardOpenOption.WRITE; + +import java.io.ByteArrayInputStream; +import java.io.IOException; +import java.io.OutputStream; +import java.io.UncheckedIOException; +import java.nio.file.Path; +import java.util.zip.GZIPOutputStream; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public abstract class FlowUpdateUtils { + + private static final Logger LOGGER = LoggerFactory.getLogger(FlowUpdateUtils.class); + + private FlowUpdateUtils() { + } + + public static void backup(Path current, Path backup) throws IOException { + if (current != null && exists(current)) { + copy(current, backup, REPLACE_EXISTING); + } else { + LOGGER.warn("Flow configuration files does not exist. No backup copy will be created"); + } + } + + public static void persist(byte[] flowConfig, Path flowPath, boolean compress) throws IOException { + LOGGER.debug("Persisting flow to path {} with content\n{}\n and compress {}", flowPath, new String(flowConfig, UTF_8), compress); + try (ByteArrayInputStream flowInputStream = new ByteArrayInputStream(flowConfig); + OutputStream fileOut = compress + ? new GZIPOutputStream(newOutputStream(flowPath, WRITE, CREATE, TRUNCATE_EXISTING)) + : newOutputStream(flowPath, WRITE, CREATE, TRUNCATE_EXISTING)) { + flowInputStream.transferTo(fileOut); + } + LOGGER.info("Updated configuration was written to: {}", flowPath); + } + + public static void revert(Path backupFlowConfigFile, Path currentFlowConfigFile) { + if (backupFlowConfigFile != null && exists(backupFlowConfigFile)) { + try { + copy(backupFlowConfigFile, currentFlowConfigFile, REPLACE_EXISTING); + } catch (IOException e) { + LOGGER.error("Revert to previous flow failed. Please stop MiNiFi and revert the flow manually"); + throw new UncheckedIOException("Failed to revert flow", e); + } + } else { + LOGGER.error("Backup flow configuration file does not exist"); + throw new RuntimeException("Backup flow configuration file does not exist"); + } + } + + public static void removeIfExists(Path flowConfigFile) { + if (flowConfigFile != null) { + try { + deleteIfExists(flowConfigFile); + } catch (IOException e) { + LOGGER.warn("Unable to remove flow configuration backup file", e); + } + } + } +} diff --git a/minifi/minifi-commons/minifi-commons-framework/src/test/java/org/apache/nifi/minifi/commons/service/FlowEnrichServiceTest.java b/minifi/minifi-commons/minifi-commons-framework/src/test/java/org/apache/nifi/minifi/commons/service/FlowEnrichServiceTest.java new file mode 100644 index 0000000000..c43ab70551 --- /dev/null +++ b/minifi/minifi-commons/minifi-commons-framework/src/test/java/org/apache/nifi/minifi/commons/service/FlowEnrichServiceTest.java @@ -0,0 +1,233 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.nifi.minifi.commons.service; + +import static java.nio.charset.StandardCharsets.UTF_8; +import static org.apache.nifi.minifi.commons.service.FlowEnrichService.COMMON_SSL_CONTEXT_SERVICE_NAME; +import static org.apache.nifi.minifi.commons.service.FlowEnrichService.DEFAULT_SSL_CONTEXT_SERVICE_NAME; +import static org.apache.nifi.minifi.commons.service.FlowEnrichService.SITE_TO_SITE_PROVENANCE_REPORTING_TASK_NAME; +import static org.junit.jupiter.api.Assertions.assertArrayEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.Mockito.mockStatic; + +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.DeserializationFeature; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.module.jaxb.JaxbAnnotationIntrospector; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.UUID; +import org.apache.commons.lang3.StringUtils; +import org.apache.nifi.controller.flow.VersionedDataflow; +import org.apache.nifi.flow.Bundle; +import org.apache.nifi.flow.VersionedConfigurableExtension; +import org.apache.nifi.flow.VersionedControllerService; +import org.apache.nifi.flow.VersionedProcessor; +import org.apache.nifi.flow.VersionedReportingTask; +import org.apache.nifi.minifi.commons.api.MiNiFiProperties; +import org.apache.nifi.properties.StandardReadableProperties; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; +import org.mockito.MockedStatic; + +public class FlowEnrichServiceTest { + + private static final Path DEFAULT_FLOW_JSON = Path.of("src/test/resources/default_flow.json"); + + @Test + public void testFlowIsLeftIntactIfEnrichingIsNotNecessary() { + // given + Map properties = Map.of(); + byte[] testFlowBytes = flowToString(loadDefaultFlow()).getBytes(UTF_8); + + // when + FlowEnrichService testFlowEnrichService = new FlowEnrichService(new StandardReadableProperties(properties)); + byte[] enrichedFlowBytes = testFlowEnrichService.enrichFlow(testFlowBytes); + + // then + assertArrayEquals(testFlowBytes, enrichedFlowBytes); + } + + @Test + public void testMissingRootGroupIdsAreFilledIn() { + // given + Map properties = Map.of(); + VersionedDataflow testFlow = loadDefaultFlow(); + testFlow.getRootGroup().setIdentifier(null); + testFlow.getRootGroup().setInstanceIdentifier(null); + byte[] testFlowBytes = flowToString(testFlow).getBytes(UTF_8); + UUID expectedIdentifier = UUID.randomUUID(); + + try (MockedStatic uuid = mockStatic(UUID.class)) { + uuid.when(UUID::randomUUID).thenReturn(expectedIdentifier); + + // when + FlowEnrichService testFlowEnrichService = new FlowEnrichService(new StandardReadableProperties(properties)); + byte[] enrichedFlowBytes = testFlowEnrichService.enrichFlow(testFlowBytes); + + // then + VersionedDataflow versionedDataflow = flowFromString(new String(enrichedFlowBytes, UTF_8)); + Assertions.assertEquals(expectedIdentifier.toString(), versionedDataflow.getRootGroup().getIdentifier()); + Assertions.assertEquals(expectedIdentifier.toString(), versionedDataflow.getRootGroup().getInstanceIdentifier()); + } + } + + @Test + public void testCommonSslControllerServiceIsAddedWithBundleVersionAndProcessorControllerServiceIsOverridden() { + // given + Map properties = securityProperties(true); + VersionedDataflow versionedDataflow = loadDefaultFlow(); + Bundle bundle = bundle("org.apache.nifi", "nifi-ssl-context-service-nar", StringUtils.EMPTY); + String originalSslControllerServiceId = "original_ssl_controller_service_id"; + versionedDataflow.getRootGroup() + .setProcessors(Set.of( + processor(bundle, "processor_1", originalSslControllerServiceId), + processor(bundle, "processor_2", originalSslControllerServiceId) + )); + byte[] testFlowBytes = flowToString(versionedDataflow).getBytes(UTF_8); + + // when + FlowEnrichService testFlowEnrichService = new FlowEnrichService(new StandardReadableProperties(properties)); + byte[] enrichedFlowBytes = testFlowEnrichService.enrichFlow(testFlowBytes); + + // then + VersionedDataflow enrichedFlow = flowFromString(new String(enrichedFlowBytes, UTF_8)); + Assertions.assertEquals(1, enrichedFlow.getControllerServices().size()); + VersionedControllerService sslControllerService = enrichedFlow.getControllerServices().get(0); + assertEquals(COMMON_SSL_CONTEXT_SERVICE_NAME, sslControllerService.getName()); + Assertions.assertEquals(StringUtils.EMPTY, sslControllerService.getBundle().getVersion()); + Set processors = enrichedFlow.getRootGroup().getProcessors(); + assertEquals(2, processors.size()); + assertTrue( + processors.stream() + .map(VersionedConfigurableExtension::getProperties) + .map(props -> props.get(DEFAULT_SSL_CONTEXT_SERVICE_NAME)) + .allMatch(controllerServiceName -> sslControllerService.getInstanceIdentifier().equals(controllerServiceName)) + ); + } + + @Test + public void testProvenanceReportingTaskIsAdded() { + // given + Map properties = Map.of( + MiNiFiProperties.NIFI_MINIFI_PROVENANCE_REPORTING_COMMENT.getKey(), "comment", + MiNiFiProperties.NIFI_MINIFI_PROVENANCE_REPORTING_SCHEDULING_STRATEGY.getKey(), "timer_driven", + MiNiFiProperties.NIFI_MINIFI_PROVENANCE_REPORTING_SCHEDULING_PERIOD.getKey(), "10 sec", + MiNiFiProperties.NIFI_MINIFI_PROVENANCE_REPORTING_DESTINATION_URL.getKey(), "http://host:port/destination", + MiNiFiProperties.NIFI_MINIFI_PROVENANCE_REPORTING_INPUT_PORT_NAME.getKey(), "input_port", + MiNiFiProperties.NIFI_MINIFI_PROVENANCE_REPORTING_INSTANCE_URL.getKey(), "http://host:port/input", + MiNiFiProperties.NIFI_MINIFI_PROVENANCE_REPORTING_COMPRESS_EVENTS.getKey(), "true", + MiNiFiProperties.NIFI_MINIFI_PROVENANCE_REPORTING_BATCH_SIZE.getKey(), "1000", + MiNiFiProperties.NIFI_MINIFI_PROVENANCE_REPORTING_COMMUNICATIONS_TIMEOUT.getKey(), "30 sec" + ); + byte[] testFlowBytes = flowToString(loadDefaultFlow()).getBytes(UTF_8); + + // when + FlowEnrichService testFlowEnrichService = new FlowEnrichService(new StandardReadableProperties(properties)); + byte[] enrichedFlowBytes = testFlowEnrichService.enrichFlow(testFlowBytes); + + // then + VersionedDataflow enrichedFlow = flowFromString(new String(enrichedFlowBytes, UTF_8)); + List reportingTasks = enrichedFlow.getReportingTasks(); + assertEquals(1, reportingTasks.size()); + VersionedReportingTask provenanceReportingTask = reportingTasks.get(0); + assertEquals(SITE_TO_SITE_PROVENANCE_REPORTING_TASK_NAME, provenanceReportingTask.getName()); + Assertions.assertEquals(properties.get(MiNiFiProperties.NIFI_MINIFI_PROVENANCE_REPORTING_COMMENT.getKey()), provenanceReportingTask.getComments()); + Assertions.assertEquals(properties.get(MiNiFiProperties.NIFI_MINIFI_PROVENANCE_REPORTING_SCHEDULING_STRATEGY.getKey()), provenanceReportingTask.getSchedulingStrategy()); + Assertions.assertEquals(properties.get(MiNiFiProperties.NIFI_MINIFI_PROVENANCE_REPORTING_SCHEDULING_PERIOD.getKey()), provenanceReportingTask.getSchedulingPeriod()); + Map provenanceReportingTaskProperties = provenanceReportingTask.getProperties(); + assertEquals(properties.get(MiNiFiProperties.NIFI_MINIFI_PROVENANCE_REPORTING_INPUT_PORT_NAME.getKey()), provenanceReportingTaskProperties.get("Input Port Name")); + assertEquals(properties.get(MiNiFiProperties.NIFI_MINIFI_PROVENANCE_REPORTING_DESTINATION_URL.getKey()), provenanceReportingTaskProperties.get("Destination URL")); + assertEquals(properties.get(MiNiFiProperties.NIFI_MINIFI_PROVENANCE_REPORTING_COMPRESS_EVENTS.getKey()), provenanceReportingTaskProperties.get("Compress Events")); + assertEquals(properties.get(MiNiFiProperties.NIFI_MINIFI_PROVENANCE_REPORTING_BATCH_SIZE.getKey()), provenanceReportingTaskProperties.get("Batch Size")); + assertEquals(properties.get(MiNiFiProperties.NIFI_MINIFI_PROVENANCE_REPORTING_COMMUNICATIONS_TIMEOUT.getKey()), + provenanceReportingTaskProperties.get("Communications Timeout")); + assertEquals(properties.get(MiNiFiProperties.NIFI_MINIFI_PROVENANCE_REPORTING_INSTANCE_URL.getKey()), provenanceReportingTaskProperties.get("Instance URL")); + } + + private VersionedDataflow loadDefaultFlow() { + try { + String flowString = Files.readString(DEFAULT_FLOW_JSON); + return flowFromString(flowString); + } catch (Exception e) { + throw new RuntimeException(e); + } + } + + private VersionedDataflow flowFromString(String flow) { + ObjectMapper objectMapper = new ObjectMapper(); + objectMapper.setAnnotationIntrospector(new JaxbAnnotationIntrospector(objectMapper.getTypeFactory())); + objectMapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false); + + try { + return objectMapper.readValue(flow, VersionedDataflow.class); + } catch (JsonProcessingException e) { + throw new RuntimeException(e); + } + } + + private String flowToString(VersionedDataflow versionedDataflow) { + ObjectMapper objectMapper = new ObjectMapper(); + objectMapper.setSerializationInclusion(JsonInclude.Include.NON_NULL); + objectMapper.setDefaultPropertyInclusion(JsonInclude.Value.construct(JsonInclude.Include.NON_NULL, JsonInclude.Include.NON_NULL)); + objectMapper.setAnnotationIntrospector(new JaxbAnnotationIntrospector(objectMapper.getTypeFactory())); + objectMapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false); + + try { + return objectMapper.writeValueAsString(versionedDataflow); + } catch (JsonProcessingException e) { + throw new RuntimeException(e); + } + } + + private Map securityProperties(Boolean useParentSslControllerService) { + return Map.of( + MiNiFiProperties.NIFI_MINIFI_SECURITY_KEYSTORE.getKey(), "path/to/keystore.jks", + MiNiFiProperties.NIFI_MINIFI_SECURITY_KEYSTORE_TYPE.getKey(), "jks", + MiNiFiProperties.NIFI_MINIFI_SECURITY_KEYSTORE_PASSWD.getKey(), "password123", + MiNiFiProperties.NIFI_MINIFI_SECURITY_KEY_PASSWD.getKey(), "password456", + MiNiFiProperties.NIFI_MINIFI_SECURITY_TRUSTSTORE.getKey(), "path/to/truststore.jks", + MiNiFiProperties.NIFI_MINIFI_SECURITY_TRUSTSTORE_TYPE.getKey(), "jks", + MiNiFiProperties.NIFI_MINIFI_SECURITY_TRUSTSTORE_PASSWD.getKey(), "password789", + MiNiFiProperties.NIFI_MINIFI_SECURITY_SSL_PROTOCOL.getKey(), "TLS1.2", + MiNiFiProperties.NIFI_MINIFI_FLOW_USE_PARENT_SSL.getKey(), useParentSslControllerService.toString() + ); + } + + private Bundle bundle(String group, String artifact, String version) { + Bundle bundle = new Bundle(); + bundle.setGroup(group); + bundle.setArtifact(artifact); + bundle.setVersion(version); + return bundle; + } + + private VersionedProcessor processor(Bundle bundle, String name, String originalSslControllerServiceId) { + VersionedProcessor versionedProcessor = new VersionedProcessor(); + versionedProcessor.setIdentifier(UUID.randomUUID().toString()); + versionedProcessor.setName(name); + versionedProcessor.setBundle(bundle); + versionedProcessor.setProperties(Map.of(DEFAULT_SSL_CONTEXT_SERVICE_NAME, originalSslControllerServiceId)); + return versionedProcessor; + } +} diff --git a/minifi/minifi-commons/minifi-commons-framework/src/test/resources/default_flow.json b/minifi/minifi-commons/minifi-commons-framework/src/test/resources/default_flow.json new file mode 100644 index 0000000000..f2dc9ba69c --- /dev/null +++ b/minifi/minifi-commons/minifi-commons-framework/src/test/resources/default_flow.json @@ -0,0 +1,40 @@ +{ + "encodingVersion": { + "majorVersion": 2, + "minorVersion": 0 + }, + "maxTimerDrivenThreadCount": 1, + "maxEventDrivenThreadCount": 1, + "registries": [], + "parameterContexts": [], + "parameterProviders": [], + "controllerServices": [], + "reportingTasks": [], + "templates": [], + "rootGroup": { + "identifier": "d6b20357-fba8-343d-b893-3eed5e75be39", + "instanceIdentifier": "e65d0286-b546-4a40-8e81-d495b4f70a47", + "name": "NiFi Flow", + "comments": "", + "position": { + "x": 0.0, + "y": 0.0 + }, + "processGroups": [], + "remoteProcessGroups": [], + "processors": [], + "inputPorts": [], + "outputPorts": [], + "connections": [], + "labels": [], + "funnels": [], + "controllerServices": [], + "variables": {}, + "defaultFlowFileExpiration": "0 sec", + "defaultBackPressureObjectThreshold": 10000, + "defaultBackPressureDataSizeThreshold": "1 GB", + "componentType": "PROCESS_GROUP", + "flowFileConcurrency": "UNBOUNDED", + "flowFileOutboundPolicy": "STREAM_WHEN_AVAILABLE" + } +} \ No newline at end of file diff --git a/minifi/minifi-commons/minifi-commons-framework/src/test/resources/mockito-extensions/org.mockito.plugins.MockMaker b/minifi/minifi-commons/minifi-commons-framework/src/test/resources/mockito-extensions/org.mockito.plugins.MockMaker new file mode 100644 index 0000000000..ca6ee9cea8 --- /dev/null +++ b/minifi/minifi-commons/minifi-commons-framework/src/test/resources/mockito-extensions/org.mockito.plugins.MockMaker @@ -0,0 +1 @@ +mock-maker-inline \ No newline at end of file diff --git a/minifi/minifi-commons/minifi-commons-schema/src/main/java/org/apache/nifi/minifi/commons/schema/common/BootstrapPropertyKeys.java b/minifi/minifi-commons/minifi-commons-schema/src/main/java/org/apache/nifi/minifi/commons/schema/common/BootstrapPropertyKeys.java deleted file mode 100644 index 126f72dbec..0000000000 --- a/minifi/minifi-commons/minifi-commons-schema/src/main/java/org/apache/nifi/minifi/commons/schema/common/BootstrapPropertyKeys.java +++ /dev/null @@ -1,123 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.nifi.minifi.commons.schema.common; - -import static org.apache.nifi.minifi.commons.schema.RemoteProcessGroupSchema.TIMEOUT_KEY; -import static org.apache.nifi.minifi.commons.schema.common.CommonPropertyKeys.COMMENT_KEY; -import static org.apache.nifi.minifi.commons.schema.common.CommonPropertyKeys.SCHEDULING_PERIOD_KEY; -import static org.apache.nifi.minifi.commons.schema.common.CommonPropertyKeys.SCHEDULING_STRATEGY_KEY; -import static org.apache.nifi.minifi.commons.schema.common.CommonPropertyKeys.USE_COMPRESSION_KEY; - -import java.util.Arrays; -import java.util.Collections; -import java.util.HashMap; -import java.util.HashSet; -import java.util.Map; -import java.util.Set; -import org.apache.nifi.minifi.commons.schema.ProvenanceReportingSchema; -import org.apache.nifi.minifi.commons.schema.SecurityPropertiesSchema; -import org.apache.nifi.minifi.commons.schema.SensitivePropsSchema; - -public class BootstrapPropertyKeys { - - public static final String STATUS_REPORTER_PROPERTY_PREFIX = "nifi.minifi.status.reporter"; - public static final String STATUS_REPORTER_COMPONENTS_KEY = STATUS_REPORTER_PROPERTY_PREFIX + ".components"; - - public static final String USE_PARENT_SSL = "nifi.minifi.flow.use.parent.ssl"; - - public static final String SECURITY_KEYSTORE_KEY = "nifi.minifi.security.keystore"; - public static final String SECURITY_KEYSTORE_TYPE_KEY = "nifi.minifi.security.keystoreType"; - public static final String SECURITY_KEYSTORE_PASSWORD_KEY = "nifi.minifi.security.keystorePasswd"; - public static final String SECURITY_KEY_PASSWORD_KEY = "nifi.minifi.security.keyPasswd"; - public static final String SECURITY_TRUSTSTORE_KEY = "nifi.minifi.security.truststore"; - public static final String SECURITY_TRUSTSTORE_TYPE_KEY = "nifi.minifi.security.truststoreType"; - public static final String SECURITY_TRUSTSTORE_PASSWORD_KEY = "nifi.minifi.security.truststorePasswd"; - public static final String SECURITY_SSL_PROTOCOL_KEY = "nifi.minifi.security.ssl.protocol"; - - public static final String SENSITIVE_PROPS_KEY_KEY = "nifi.minifi.sensitive.props.key"; - public static final String SENSITIVE_PROPS_ALGORITHM_KEY = "nifi.minifi.sensitive.props.algorithm"; - - public static final Set BOOTSTRAP_SECURITY_PROPERTY_KEYS = new HashSet<>( - Arrays.asList(SECURITY_KEYSTORE_KEY, - SECURITY_KEYSTORE_TYPE_KEY, - SECURITY_KEYSTORE_PASSWORD_KEY, - SECURITY_KEY_PASSWORD_KEY, - SECURITY_TRUSTSTORE_KEY, - SECURITY_TRUSTSTORE_TYPE_KEY, - SECURITY_TRUSTSTORE_PASSWORD_KEY, - SECURITY_SSL_PROTOCOL_KEY)); - - public static final Set BOOTSTRAP_SENSITIVE_PROPERTY_KEYS = new HashSet<>( - Arrays.asList( - SENSITIVE_PROPS_KEY_KEY, - SENSITIVE_PROPS_ALGORITHM_KEY)); - - public static final String PROVENANCE_REPORTING_COMMENT_KEY = "nifi.minifi.provenance.reporting.comment"; - public static final String PROVENANCE_REPORTING_SCHEDULING_STRATEGY_KEY = "nifi.minifi.provenance.reporting.scheduling.strategy"; - public static final String PROVENANCE_REPORTING_SCHEDULING_PERIOD_KEY = "nifi.minifi.provenance.reporting.scheduling.period"; - public static final String PROVENANCE_REPORTING_DESTINATION_URL_KEY = "nifi.minifi.provenance.reporting.destination.url"; - public static final String PROVENANCE_REPORTING_INPUT_PORT_NAME_KEY = "nifi.minifi.provenance.reporting.input.port.name"; - public static final String PROVENANCE_REPORTING_INSTANCE_URL_KEY = "nifi.minifi.provenance.reporting.instance.url"; - public static final String PROVENANCE_REPORTING_COMPRESS_EVENTS_KEY = "nifi.minifi.provenance.reporting.compress.events"; - public static final String PROVENANCE_REPORTING_BATCH_SIZE_KEY = "nifi.minifi.provenance.reporting.batch.size"; - public static final String PROVENANCE_REPORTING_COMMUNICATIONS_TIMEOUT_KEY = "nifi.minifi.provenance.reporting.communications.timeout"; - - public static final Set BOOTSTRAP_PROVENANCE_REPORTING_KEYS = new HashSet<>( - Arrays.asList(PROVENANCE_REPORTING_COMMENT_KEY, - PROVENANCE_REPORTING_SCHEDULING_STRATEGY_KEY, - PROVENANCE_REPORTING_SCHEDULING_PERIOD_KEY, - PROVENANCE_REPORTING_DESTINATION_URL_KEY, - PROVENANCE_REPORTING_INPUT_PORT_NAME_KEY, - PROVENANCE_REPORTING_INSTANCE_URL_KEY, - PROVENANCE_REPORTING_COMPRESS_EVENTS_KEY, - PROVENANCE_REPORTING_BATCH_SIZE_KEY, - PROVENANCE_REPORTING_COMMUNICATIONS_TIMEOUT_KEY - )); - - public static final Map BOOTSTRAP_KEYS_TO_YML_KEYS; - - static { - final Map mutableMap = new HashMap<>(); - mutableMap.put(SECURITY_KEYSTORE_KEY, SecurityPropertiesSchema.KEYSTORE_KEY); - mutableMap.put(SECURITY_KEYSTORE_TYPE_KEY, SecurityPropertiesSchema.KEYSTORE_TYPE_KEY); - mutableMap.put(SECURITY_KEYSTORE_PASSWORD_KEY, SecurityPropertiesSchema.KEYSTORE_PASSWORD_KEY); - mutableMap.put(SECURITY_KEY_PASSWORD_KEY, SecurityPropertiesSchema.KEY_PASSWORD_KEY); - - mutableMap.put(SECURITY_TRUSTSTORE_KEY, SecurityPropertiesSchema.TRUSTSTORE_KEY); - mutableMap.put(SECURITY_TRUSTSTORE_TYPE_KEY, SecurityPropertiesSchema.TRUSTSTORE_TYPE_KEY); - mutableMap.put(SECURITY_TRUSTSTORE_PASSWORD_KEY, SecurityPropertiesSchema.TRUSTSTORE_PASSWORD_KEY); - - mutableMap.put(SECURITY_SSL_PROTOCOL_KEY, SecurityPropertiesSchema.SSL_PROTOCOL_KEY); - - mutableMap.put(SENSITIVE_PROPS_KEY_KEY, SensitivePropsSchema.SENSITIVE_PROPS_KEY_KEY); - mutableMap.put(SENSITIVE_PROPS_ALGORITHM_KEY, SensitivePropsSchema.SENSITIVE_PROPS_ALGORITHM_KEY); - - mutableMap.put(PROVENANCE_REPORTING_COMMENT_KEY, COMMENT_KEY); - mutableMap.put(PROVENANCE_REPORTING_SCHEDULING_STRATEGY_KEY, SCHEDULING_STRATEGY_KEY); - mutableMap.put(PROVENANCE_REPORTING_SCHEDULING_PERIOD_KEY, SCHEDULING_PERIOD_KEY); - mutableMap.put(PROVENANCE_REPORTING_DESTINATION_URL_KEY, ProvenanceReportingSchema.DESTINATION_URL_KEY); - mutableMap.put(PROVENANCE_REPORTING_INPUT_PORT_NAME_KEY, ProvenanceReportingSchema.PORT_NAME_KEY); - mutableMap.put(PROVENANCE_REPORTING_INSTANCE_URL_KEY, ProvenanceReportingSchema.ORIGINATING_URL_KEY); - mutableMap.put(PROVENANCE_REPORTING_COMPRESS_EVENTS_KEY, USE_COMPRESSION_KEY); - mutableMap.put(PROVENANCE_REPORTING_BATCH_SIZE_KEY, ProvenanceReportingSchema.BATCH_SIZE_KEY); - mutableMap.put(PROVENANCE_REPORTING_COMMUNICATIONS_TIMEOUT_KEY, TIMEOUT_KEY); - - BOOTSTRAP_KEYS_TO_YML_KEYS = Collections.unmodifiableMap(mutableMap); - } - -} diff --git a/minifi/minifi-commons/pom.xml b/minifi/minifi-commons/pom.xml index 490934b169..33584db81c 100644 --- a/minifi/minifi-commons/pom.xml +++ b/minifi/minifi-commons/pom.xml @@ -26,8 +26,8 @@ limitations under the License. pom - minifi-commons-schema minifi-utils minifi-commons-api + minifi-commons-framework diff --git a/minifi/minifi-docker/dockerhub/Dockerfile b/minifi/minifi-docker/dockerhub/Dockerfile index 2b44702597..459a1fd9f9 100644 --- a/minifi/minifi-docker/dockerhub/Dockerfile +++ b/minifi/minifi-docker/dockerhub/Dockerfile @@ -16,7 +16,7 @@ # under the License. # -FROM eclipse-temurin:11-jre +FROM eclipse-temurin:17-jre LABEL maintainer="Apache NiFi " LABEL site="https://nifi.apache.org" diff --git a/minifi/minifi-docker/dockerhub/README.md b/minifi/minifi-docker/dockerhub/README.md index 703ec5d35e..aa45afc3d3 100644 --- a/minifi/minifi-docker/dockerhub/README.md +++ b/minifi/minifi-docker/dockerhub/README.md @@ -36,17 +36,17 @@ To build an image for a prior released version, one can override the `MINIFI_VER ## Running a container ### Supplying configuration to a container -The primary means by which a MiNiFi instance is configured is via the `config.yml` or the `bootstrap.conf`. +The primary means by which a MiNiFi instance is configured is via the `flow.json.raw` or the `bootstrap.conf`. This can be accomplished through: * the use of volumes, and * overlaying the base image #### Using volumes to provide configuration -The following example shows the usage of two volumes to provide both a `config.yml` and a `bootstrap.conf` to the container instance. This makes use of configuration files on the host and maps them to be used by the MiNiFi instance. This is helpful in scenarios where a single image is used for a variety of configurations. +The following example shows the usage of two volumes to provide both a `flow.json.raw` and a `bootstrap.conf` to the container instance. This makes use of configuration files on the host and maps them to be used by the MiNiFi instance. This is helpful in scenarios where a single image is used for a variety of configurations. docker run -d \ - -v ~/minifi-conf/config.yml:/opt/minifi/minifi-current/conf/config.yml \ + -v ~/minifi-conf/flow.json.raw:/opt/minifi/minifi-current/conf/flow.json.raw \ -v ~/minifi-conf/bootstrap.conf:/opt/minifi/minifi-current/conf/bootstrap.conf \ apache/nifi-minifi:latest @@ -55,7 +55,7 @@ Alternatively, it is possible to create a custom image inheriting from the publi FROM apache/nifi-minifi - ADD config.yml /opt/minifi/minifi-current/conf/config.yml + ADD flow.json.raw /opt/minifi/minifi-current/conf/flow.json.raw ADD bootstrap.conf /opt/minifi/minifi-current/conf/bootstrap.conf Building this `Dockerfile` will result in a custom image with the specified configuration files incorporated into the new image. This is best for applications where configuration is well defined and relatively static. diff --git a/minifi/minifi-docker/dockermaven/Dockerfile b/minifi/minifi-docker/dockermaven/Dockerfile index 2f6880fb42..a6366d2a08 100644 --- a/minifi/minifi-docker/dockermaven/Dockerfile +++ b/minifi/minifi-docker/dockermaven/Dockerfile @@ -17,7 +17,7 @@ # ARG IMAGE_NAME=eclipse-temurin -ARG IMAGE_TAG=11-jre +ARG IMAGE_TAG=17-jre FROM ${IMAGE_NAME}:${IMAGE_TAG} as builder LABEL stage=builder diff --git a/minifi/minifi-docker/pom.xml b/minifi/minifi-docker/pom.xml index b036bb228a..dce592116e 100644 --- a/minifi/minifi-docker/pom.xml +++ b/minifi/minifi-docker/pom.xml @@ -30,7 +30,7 @@ limitations under the License. ${project.version} eclipse-temurin - 11-jre + 17-jre diff --git a/minifi/minifi-docs/src/main/markdown/System_Admin_Guide.md b/minifi/minifi-docs/src/main/markdown/System_Admin_Guide.md index f2846075eb..69d5a77500 100644 --- a/minifi/minifi-docs/src/main/markdown/System_Admin_Guide.md +++ b/minifi/minifi-docs/src/main/markdown/System_Admin_Guide.md @@ -80,7 +80,7 @@ This documentation is for MiNiFi ${project.version}. # Automatic Warm-Redeploy -When many MiNiFi agents running on the edge, it may not be possible to manually stop, edit the *config.yml* and then restart every one every time their configuration needs to change. The Config Change Coordinator and its Ingestors were designed to automatically redeploy in response to a configuration update. +When many MiNiFi agents running on the edge, it may not be possible to manually stop, edit the *flow.json.raw* and then restart every one every time their configuration needs to change. The Config Change Coordinator and its Ingestors were designed to automatically redeploy in response to a configuration update. The Config Change Ingestors are the means by which the agent is notified of a potential new configuration. Currently there are three: @@ -128,10 +128,10 @@ This Config Change Ingestor sets up a light-weight Jetty HTTP(S) REST service in **Note:** The encoding is expected to be Unicode and the exact version specified by the BOM mark ('UTF-8','UTF-16BE' or 'UTF-16LE'). If there is no BOM mark, then UTF-8 is used. -Here is an example post request using `curl` hitting the local machine on port `8338` and it is executed with the config file *config.yml* in the directory the command is run from: +Here is an example post request using `curl` hitting the local machine on port `8338` and it is executed with the config file *flow.json.raw* in the directory the command is run from: ``` -curl --request POST --data-binary "@config.yml" http://localhost:8338/ +curl --request POST --data-binary "@flow.json.raw" http://localhost:8338/ ``` Below are the configuration options. There are no required options. If no properties are set then the server will bind to hostname `localhost` on a random open port, will only connect via HTTP and will use the `WholeConfigDifferentiator`. @@ -392,9 +392,7 @@ Bootstrap Config File: /Users/user/projects/nifi-minifi/minifi-assembly/target/m # Config File -The *config.yml* in the `conf` directory is the main configuration file for controlling how MiNiFi runs. This section provides an overview of the properties in this file. The file is a YAML file and follows the YAML format laid out [here](https://www.yaml.org/). - -Alternatively, the MiNiFi Toolkit Converter can aid in creating a *config.yml* from a generated template exported from a NiFi instance. This tool can be downloaded from https://nifi.apache.org/minifi/download.html under the "MiNiFi Toolkit Binaries" section. Information on the toolkit's usage is available at https://nifi.apache.org/minifi/minifi-toolkit.html. +The *flow.json.raw* in the `conf` directory is the main configuration file for controlling how MiNiFi runs. **Note:** Values for periods of time and data sizes must include the unit of measure, for example "10 sec" or "10 MB", not simply "10". diff --git a/minifi/minifi-docs/src/main/markdown/minifi-java-agent-quick-start.md b/minifi/minifi-docs/src/main/markdown/minifi-java-agent-quick-start.md index 2f43cfa1b7..7f689e834d 100644 --- a/minifi/minifi-docs/src/main/markdown/minifi-java-agent-quick-start.md +++ b/minifi/minifi-docs/src/main/markdown/minifi-java-agent-quick-start.md @@ -31,14 +31,12 @@ Apache NiFi MiNiFi is an Apache NiFi project, designed to collect data at its so # Before You Begin MiNiFi Java Agent is supported on the following operating systems: -* Red Hat Enterprise Linux / CentOS 6 (64-bit) * Red Hat Enterprise Linux / CentOS 7 (64-bit) -* Ubuntu Precise (12.04) (64-bit) -* Ubuntu Trusty (14.04) (64-bit) -* Ubuntu Xenial (16.04) (64-bit) +* Red Hat Enterprise Linux / CentOS 8 (64-bit) * Ubuntu Bionic (18.04) (64-bit) -* Debian 7 -* SUSE Linux Enterprise Server (SLES) 11 SP3 (64-bit) +* Ubuntu Focal Fossa (20.04) (64-bit) +* Debian 9 +* SUSE Linux Enterprise Server (SLES) 12 SP5 (64-bit) You can download the MiNiFi Java Agent and the MiNiFi Toolkit from the [MiNiFi download page](https://nifi.apache.org/minifi/download.html). @@ -97,21 +95,16 @@ You can use the MiNiFi Toolkit, located in your MiNiFi installation directory, a 1. Launch NiFi 2. Create a dataflow. -3. Convert your dataflow into a template. -4. Download your template as an .xml file. For more information on working with templates, see the [Templates](https://nifi.apache.org/docs/nifi-docs/html/user-guide.html#templates) section in the *User Guide*. -5. From the MiNiFi Toolkit, run the following command to turn your .xml file into a .yml file: -``` -config.sh transform input_file output_file -``` -6. Move your new .yml file to `minifi/conf`. -7. Rename your .yml file _config.yml_. +3. Export the dataflow in JSON format. +4. Move your new .json file to `minifi/conf`. +5. Rename your .json file _flow.json.raw_. **Note:** You can use one template at a time, per MiNiFi instance. -**Result:** Once you have your _config.yml_ file in the `minifi/conf` directory, launch that instance of MiNiFi and your dataflow begins automatically. +**Result:** Once you have your _flow.json.raw_ file in the `minifi/conf` directory, launch that instance of MiNiFi and your dataflow begins automatically. ### Utilizing a C2 Server via the c2 protocol -If you have a [C2 server](../../../../minifi-c2/README.md) running, you can expose the whole _config.yml_ for the agent to download. As the agent is heartbeating via the C2 protocol, changes in flow version will trigger automatic config updates. +If you have a [C2 server](../../../../minifi-c2/README.md) running, you can expose the whole _flow.json_ for the agent to download. As the agent is heartbeating via the C2 protocol, changes in flow version will trigger automatic config updates. 1. Launch C2 server 2. Configure MiNiFi for C2 capability @@ -137,8 +130,8 @@ c2.agent.class=agentClassName ### Manually To load a new dataflow for a MiNiFi instance to run: -1. Create a new _config.yml_ file with the new dataflow. -2. Replace the existing _config.yml_ in `minifi/conf` with the new file. +1. Create a new _flow.json.raw_ file with the new dataflow. +2. Replace the existing _flow.json.raw_ in `minifi/conf` with the new file. 3. Restart MiNiFi. ### Utilizing C2 protocol @@ -257,21 +250,22 @@ You can secure your MiNiFi dataflow using keystore or trust store SSL protocols, To run a MiNiFi dataflow securely: 1. Create your dataflow template as discussed above. -2. Move it to `minifi/conf` and rename _config.yml_. -3. Manually modify the Security Properties section of _config.yml_. For example: +2. Move it to `minifi/conf` and rename _flow.json.raw_. +3. Fill in the following properties in bootstrap.conf. The flow definition will automatically pick up the necessary properties ``` -Security Properties: -keystore: -keystore type: -keystore password: -key password: -truststore: -truststore type: -truststore password: -ssl protocol: TLS -Sensitive Props: -key: -algorithm: NIFI_PBKDF2_AES_GCM_256 +nifi.minifi.security.keystore= +nifi.minifi.security.keystoreType= +nifi.minifi.security.keystorePasswd= +nifi.minifi.security.keyPasswd= +nifi.minifi.security.truststore= +nifi.minifi.security.truststoreType= +nifi.minifi.security.truststorePasswd= +nifi.minifi.security.ssl.protocol= + +nifi.minifi.flow.use.parent.ssl=false + +nifi.minifi.sensitive.props.key= +nifi.minifi.sensitive.props.algorithm= ``` # Managing MiNiFi diff --git a/minifi/minifi-integration-tests/pom.xml b/minifi/minifi-integration-tests/pom.xml index 8ee88687d2..b09608309e 100644 --- a/minifi/minifi-integration-tests/pom.xml +++ b/minifi/minifi-integration-tests/pom.xml @@ -33,7 +33,7 @@ limitations under the License. org.apache.nifi.minifi - minifi-commons-schema + minifi-toolkit-schema ${project.version} test @@ -117,6 +117,7 @@ limitations under the License. **/expected.json + **/config.application.json.* diff --git a/minifi/minifi-integration-tests/src/test/java/org/apache/nifi/minifi/integration/c2/C2ProtocolIntegrationTest.java b/minifi/minifi-integration-tests/src/test/java/org/apache/nifi/minifi/integration/c2/C2ProtocolIntegrationTest.java index a8eaa692fa..ba1d191b3a 100644 --- a/minifi/minifi-integration-tests/src/test/java/org/apache/nifi/minifi/integration/c2/C2ProtocolIntegrationTest.java +++ b/minifi/minifi-integration-tests/src/test/java/org/apache/nifi/minifi/integration/c2/C2ProtocolIntegrationTest.java @@ -48,7 +48,7 @@ public class C2ProtocolIntegrationTest { private static final String AGENT_CLASS_1 = "raspi3"; private static final String AGENT_CLASS_2 = "raspi4"; private static final String SERVICE = "c2-authoritative"; - private static final String CONFIG_YAML = "config.text.yml.v2"; + private static final String FLOW_JSON = "config.application.json.v2"; private static Path certificatesDirectory; private static SSLContext trustSslContext; private static SSLSocketFactory healthCheckSocketFactory; @@ -74,9 +74,9 @@ public class C2ProtocolIntegrationTest { .getResource("docker-compose-c2-protocol.yml").getFile()).getParent(); certificatesDirectory = resourceDirectory.toAbsolutePath().resolve("certificates-c2-protocol"); authoritativeFiles = resourceDirectory.resolve("c2").resolve("protocol").resolve(SERVICE).resolve("files"); - minifiEdge1Version2 = authoritativeFiles.resolve("edge1").resolve(AGENT_CLASS_1).resolve(CONFIG_YAML); - minifiEdge2Version2 = authoritativeFiles.resolve("edge2").resolve(AGENT_CLASS_1).resolve(CONFIG_YAML); - minifiEdge3Version2 = authoritativeFiles.resolve("edge3").resolve(AGENT_CLASS_2).resolve(CONFIG_YAML); + minifiEdge1Version2 = authoritativeFiles.resolve("edge1").resolve(AGENT_CLASS_1).resolve(FLOW_JSON); + minifiEdge2Version2 = authoritativeFiles.resolve("edge2").resolve(AGENT_CLASS_1).resolve(FLOW_JSON); + minifiEdge3Version2 = authoritativeFiles.resolve("edge3").resolve(AGENT_CLASS_2).resolve(FLOW_JSON); if (Files.exists(minifiEdge1Version2)) { Files.delete(minifiEdge1Version2); diff --git a/minifi/minifi-integration-tests/src/test/java/org/apache/nifi/minifi/integration/c2/HierarchicalC2IntegrationTest.java b/minifi/minifi-integration-tests/src/test/java/org/apache/nifi/minifi/integration/c2/HierarchicalC2IntegrationTest.java index 23d7083b8c..f08cc2995a 100644 --- a/minifi/minifi-integration-tests/src/test/java/org/apache/nifi/minifi/integration/c2/HierarchicalC2IntegrationTest.java +++ b/minifi/minifi-integration-tests/src/test/java/org/apache/nifi/minifi/integration/c2/HierarchicalC2IntegrationTest.java @@ -68,9 +68,9 @@ public class HierarchicalC2IntegrationTest { .getResource("docker-compose-c2-hierarchical.yml").getFile()).getParent(); certificatesDirectory = resourceDirectory.toAbsolutePath().resolve("certificates-c2-hierarchical"); authoritativeFiles = resourceDirectory.resolve("c2").resolve("hierarchical").resolve("c2-authoritative").resolve("files"); - minifiEdge1Version2 = authoritativeFiles.resolve("edge1").resolve("raspi3").resolve("config.text.yml.v2"); - minifiEdge2Version2 = authoritativeFiles.resolve("edge2").resolve("raspi2").resolve("config.text.yml.v2"); - minifiEdge3Version2 = authoritativeFiles.resolve("edge3").resolve("raspi3").resolve("config.text.yml.v2"); + minifiEdge1Version2 = authoritativeFiles.resolve("edge1").resolve("raspi3").resolve("config.application.json.v2"); + minifiEdge2Version2 = authoritativeFiles.resolve("edge2").resolve("raspi2").resolve("config.application.json.v2"); + minifiEdge3Version2 = authoritativeFiles.resolve("edge3").resolve("raspi3").resolve("config.application.json.v2"); if (Files.exists(minifiEdge1Version2)) { Files.delete(minifiEdge1Version2); @@ -111,24 +111,24 @@ public class HierarchicalC2IntegrationTest { @Test public void testMiNiFiEdge1() throws Exception { LogUtil.verifyLogEntries("c2/hierarchical/minifi-edge1/expected.json", docker.containers().container("minifi-edge1")); - Path csvToJsonDir = resourceDirectory.resolve("standalone").resolve("v1").resolve("CsvToJson").resolve("yml"); - Files.copy(csvToJsonDir.resolve("CsvToJson.yml"), minifiEdge1Version2); - LogUtil.verifyLogEntries("standalone/v1/CsvToJson/yml/expected.json", docker.containers().container("minifi-edge1")); + Path csvToJsonDir = resourceDirectory.resolve("standalone").resolve("CsvToJson"); + Files.copy(csvToJsonDir.resolve("CsvToJson.json"), minifiEdge1Version2); + LogUtil.verifyLogEntries("standalone/CsvToJson/expected.json", docker.containers().container("minifi-edge1")); } @Test public void testMiNiFiEdge2() throws Exception { LogUtil.verifyLogEntries("c2/hierarchical/minifi-edge2/expected.json", docker.containers().container("minifi-edge2")); - Path csvToJsonDir = resourceDirectory.resolve("standalone").resolve("v1").resolve("CsvToJson").resolve("yml"); - Files.copy(csvToJsonDir.resolve("CsvToJson.yml"), minifiEdge2Version2); - LogUtil.verifyLogEntries("standalone/v1/CsvToJson/yml/expected.json", docker.containers().container("minifi-edge2")); + Path csvToJsonDir = resourceDirectory.resolve("standalone").resolve("CsvToJson"); + Files.copy(csvToJsonDir.resolve("CsvToJson.json"), minifiEdge2Version2); + LogUtil.verifyLogEntries("standalone/CsvToJson/expected.json", docker.containers().container("minifi-edge2")); } @Test public void testMiNiFiEdge3() throws Exception { LogUtil.verifyLogEntries("c2/hierarchical/minifi-edge3/expected.json", docker.containers().container("minifi-edge3")); - Path csvToJsonDir = resourceDirectory.resolve("standalone").resolve("v1").resolve("CsvToJson").resolve("yml"); - Files.copy(csvToJsonDir.resolve("CsvToJson.yml"), minifiEdge3Version2); - LogUtil.verifyLogEntries("standalone/v1/CsvToJson/yml/expected.json", docker.containers().container("minifi-edge3")); + Path csvToJsonDir = resourceDirectory.resolve("standalone").resolve("CsvToJson"); + Files.copy(csvToJsonDir.resolve("CsvToJson.json"), minifiEdge3Version2); + LogUtil.verifyLogEntries("standalone/CsvToJson/expected.json", docker.containers().container("minifi-edge3")); } } diff --git a/minifi/minifi-integration-tests/src/test/java/org/apache/nifi/minifi/integration/standalone/test/StandaloneYamlTest.java b/minifi/minifi-integration-tests/src/test/java/org/apache/nifi/minifi/integration/standalone/test/StandaloneJsonTest.java similarity index 57% rename from minifi/minifi-integration-tests/src/test/java/org/apache/nifi/minifi/integration/standalone/test/StandaloneYamlTest.java rename to minifi/minifi-integration-tests/src/test/java/org/apache/nifi/minifi/integration/standalone/test/StandaloneJsonTest.java index bd8d00d0d8..8ecfddb616 100644 --- a/minifi/minifi-integration-tests/src/test/java/org/apache/nifi/minifi/integration/standalone/test/StandaloneYamlTest.java +++ b/minifi/minifi-integration-tests/src/test/java/org/apache/nifi/minifi/integration/standalone/test/StandaloneJsonTest.java @@ -20,31 +20,25 @@ package org.apache.nifi.minifi.integration.standalone.test; import com.palantir.docker.compose.DockerComposeExtension; import com.palantir.docker.compose.connection.waiting.HealthChecks; -import org.apache.nifi.minifi.integration.util.LogUtil; -import org.junit.jupiter.api.AfterEach; -import org.junit.jupiter.params.ParameterizedTest; -import org.junit.jupiter.params.provider.Arguments; -import org.junit.jupiter.params.provider.MethodSource; - import java.io.BufferedReader; import java.io.BufferedWriter; import java.io.FileWriter; import java.io.InputStream; import java.io.InputStreamReader; import java.util.stream.Stream; +import org.apache.nifi.minifi.integration.util.LogUtil; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; -public class StandaloneYamlTest { +public class StandaloneJsonTest { static Stream verifyLogEntries() { - return Stream.of( - Arguments.of("v1", "CsvToJson"), - Arguments.of("v1", "DecompressionCircularFlow"), - Arguments.of("v1", "MiNiFiTailLogAttribute"), - Arguments.of("v1", "ReplaceTextExpressionLanguageCSVReformatting"), - Arguments.of("v2", "MultipleRelationships"), - Arguments.of("v2", "ProcessGroups"), - Arguments.of("v2", "StressTestFramework") - ); + return Stream.of("CsvToJson", "DecompressionCircularFlow", "MiNiFiTailLogAttribute", + "MiNiFiTailLogAttribute", "ReplaceTextExpressionLanguageCSVReformatting", + "MultipleRelationships", "ProcessGroups", "StressTestFramework") + .map(Arguments::of); } DockerComposeExtension docker; @@ -54,38 +48,38 @@ public class StandaloneYamlTest { docker.after(); } - public void setDocker(final String version, final String name) throws Exception { - String dockerComposeYmlFile = "target/test-classes/docker-compose-" + version + "-" + name + "Test-yml.yml"; - try (InputStream inputStream = StandaloneYamlTest.class.getClassLoader().getResourceAsStream("docker-compose-v1-standalone.yml"); + @ParameterizedTest(name = "{index}: Name: {0}") + @MethodSource + public void verifyLogEntries(String name) throws Exception { + setDocker(name); + LogUtil.verifyLogEntries(getExpectedJson(name), docker.containers().container("minifi")); + } + + private void setDocker(String name) throws Exception { + String dockerComposeYmlFile = "target/test-classes/docker-compose-" + name + "Test.yml"; + try (InputStream inputStream = StandaloneJsonTest.class.getClassLoader().getResourceAsStream("docker-compose-standalone.yml"); InputStreamReader inputStreamReader = new InputStreamReader(inputStream); BufferedReader bufferedReader = new BufferedReader(inputStreamReader); FileWriter fileWriter = new FileWriter(dockerComposeYmlFile); BufferedWriter bufferedWriter = new BufferedWriter(fileWriter)) { String line; while ((line = bufferedReader.readLine()) != null) { - bufferedWriter.write(line.replace("REPLACED_WITH_CONFIG_FILE", getConfigYml(version, name))); + bufferedWriter.write(line.replace("REPLACED_WITH_CONFIG_FILE", getFlowConfigPath(name))); bufferedWriter.newLine(); } } docker = DockerComposeExtension.builder() - .file(dockerComposeYmlFile) - .waitingForService("minifi", HealthChecks.toRespond2xxOverHttp(8000, dockerPort -> "http://" + dockerPort.getIp() + ":" + dockerPort.getExternalPort())) - .build(); + .file(dockerComposeYmlFile) + .waitingForService("minifi", HealthChecks.toRespond2xxOverHttp(8000, dockerPort -> "http://" + dockerPort.getIp() + ":" + dockerPort.getExternalPort())) + .build(); docker.before(); } - protected String getConfigYml(final String version, final String name) { - return "./standalone/" + version + "/" + name + "/yml/" + name + ".yml"; + private String getFlowConfigPath(String name) { + return "./standalone/" + name + "/" + name + ".json"; } - protected String getExpectedJson(final String version, final String name) { - return "standalone/" + version + "/" + name + "/yml/expected.json"; - } - - @ParameterizedTest(name = "{index}: Schema Version: {0} Name: {1}") - @MethodSource - public void verifyLogEntries(final String version, final String name) throws Exception { - setDocker(version, name); - LogUtil.verifyLogEntries(getExpectedJson(version, name), docker.containers().container("minifi")); + private String getExpectedJson(String name) { + return "standalone/" + name + "/expected.json"; } } \ No newline at end of file diff --git a/minifi/minifi-integration-tests/src/test/java/org/apache/nifi/minifi/integration/standalone/test/StandaloneXmlTest.java b/minifi/minifi-integration-tests/src/test/java/org/apache/nifi/minifi/integration/standalone/test/StandaloneXmlTest.java deleted file mode 100644 index 34f5452c31..0000000000 --- a/minifi/minifi-integration-tests/src/test/java/org/apache/nifi/minifi/integration/standalone/test/StandaloneXmlTest.java +++ /dev/null @@ -1,51 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.nifi.minifi.integration.standalone.test; - -import org.apache.nifi.minifi.commons.schema.ConfigSchema; -import org.apache.nifi.minifi.commons.schema.serialization.SchemaSaver; -import org.apache.nifi.minifi.toolkit.configuration.ConfigMain; - -import java.io.InputStream; -import java.io.OutputStream; -import java.nio.file.Files; -import java.nio.file.Paths; - -public class StandaloneXmlTest extends StandaloneYamlTest { - public void setDocker(String version, String name) throws Exception { - ConfigSchema configSchema; - try (InputStream inputStream = StandaloneXmlTest.class.getClassLoader().getResourceAsStream("./standalone/" + version + "/" + name + "/xml/" + name + ".xml")) { - configSchema = ConfigMain.transformTemplateToSchema(inputStream); - } - try (OutputStream outputStream = Files.newOutputStream(Paths.get(StandaloneXmlTest.class.getClassLoader().getResource("docker-compose-v1-standalone.yml").getFile()) - .getParent().toAbsolutePath().resolve(getConfigYml(version, name)))) { - SchemaSaver.saveConfigSchema(configSchema, outputStream); - } - super.setDocker(version, name); - } - - @Override - protected String getConfigYml(final String version, final String name) { - return "./standalone/" + version + "/" + name + "/xml/" + name + ".yml"; - } - - @Override - protected String getExpectedJson(final String version, final String name) { - return "standalone/" + version + "/" + name + "/xml/expected.json"; - } -} diff --git a/minifi/minifi-integration-tests/src/test/resources/bootstrap.conf b/minifi/minifi-integration-tests/src/test/resources/bootstrap.conf index ac152b873d..da17a1df13 100644 --- a/minifi/minifi-integration-tests/src/test/resources/bootstrap.conf +++ b/minifi/minifi-integration-tests/src/test/resources/bootstrap.conf @@ -22,34 +22,119 @@ java=java run.as= # Configure where MiNiFi's lib and conf directories live +# When running as a Windows service set full paths instead of relative paths lib.dir=./lib conf.dir=./conf # How long to wait after telling MiNiFi to shutdown before explicitly killing the Process graceful.shutdown.seconds=20 -# The location for the configuration file -nifi.minifi.config=./conf/config.yml +# Disable JSR 199 so that we can use JSP's without running a JDK +java.arg.1=-Dorg.apache.jasper.compiler.disablejsr199=true -# Notifiers to use for the associated agent, comma separated list of class names -#nifi.minifi.notifier.ingestors=org.apache.nifi.minifi.bootstrap.configuration.ingestors.FileChangeIngestor -#nifi.minifi.notifier.ingestors=org.apache.nifi.minifi.bootstrap.configuration.ingestors.RestChangeIngestor -#nifi.minifi.notifier.ingestors=org.apache.nifi.minifi.bootstrap.configuration.ingestors.PullHttpChangeIngestor +# JVM memory settings +java.arg.2=-Xms256m +java.arg.3=-Xmx256m -# File change notifier configuration +# allowRestrictedHeaders is required for Cluster/Node communications to work properly +java.arg.5=-Dsun.net.http.allowRestrictedHeaders=true +java.arg.6=-Djava.protocol.handler.pkgs=sun.net.www.protocol +# Sets the provider of SecureRandom to /dev/urandom to prevent blocking on VMs +java.arg.7=-Djava.security.egd=file:/dev/urandom + +# Garbage Collector Algorithm +java.arg.13=-XX:+UseG1GC + +#Set headless mode by default +java.arg.14=-Djava.awt.headless=true + +# Enable Remote Debugging +#java.arg.debug=-agentlib:jdwp=transport=dt_socket,server=y,suspend=n,address=8000 + + +### MiNiFi Flow Config File +nifi.minifi.flow.config=./conf/flow.json.gz + + +### MiNiFi Security Properties +nifi.minifi.security.keystore= +nifi.minifi.security.keystoreType= +nifi.minifi.security.keystorePasswd= +nifi.minifi.security.keyPasswd= +nifi.minifi.security.truststore= +nifi.minifi.security.truststoreType= +nifi.minifi.security.truststorePasswd= +nifi.minifi.security.ssl.protocol= + +# Ignore all processor SSL controller services and use parent minifi SSL instead +nifi.minifi.flow.use.parent.ssl=false + + +### MiNiFi Sensitive Properties Settings +nifi.minifi.sensitive.props.key= +nifi.minifi.sensitive.props.algorithm= + + +### MiNiFi Command & Control (C2) Configuration +# Enabling C2 Uncomment each of the following options +#c2.enable=true +## heartbeat in milliseconds +#c2.agent.heartbeat.period=5000 +## define parameters about your agent +#c2.agent.class= +# Optional. Defaults to a hardware based unique identifier +#c2.agent.identifier= +# If set to false heartbeat won't contain the manifest. Defaults to true. +#c2.full.heartbeat=false +## define protocol parameters +# DEPRECATED: c2.rest.url and c2.rest.url.ack are deprecated in favor of c2.rest.path.* properties and are target to be removed in future release +# The absolute url of the C2 server's heartbeat endpoint, eg.: http://localhost/c2-server/api/heartbeat +#c2.rest.url= +# The absolute url of the C2 server's acknowledge endpoint, eg.: http://localhost/c2-server/api/acknowledge +#c2.rest.url.ack= +# C2 Rest Path Properties +# The base path of the C2 server's REST API, eg.: http://localhost/c2-server/api +#c2.rest.path.base= +# Relative url of the C2 server's heartbeat endpoint, eg.: /heartbeat +#c2.rest.path.heartbeat= +# Relative url of the C2 server's acknowledge endpoint, eg.: /acknowledge +#c2.rest.path.acknowledge= +## c2 timeouts +#c2.rest.connectionTimeout=5 sec +#c2.rest.readTimeout=5 sec +#c2.rest.callTimeout=10 sec +#c2.config.directory=./conf +#c2.runtime.manifest.identifier=minifi +#c2.runtime.type=minifi-java +# Directory for storing assets downloaded via C2 update/asset command +#c2.asset.directory=./asset +## Define TLS security properties for C2 communications +#c2.security.truststore.location= +#c2.security.truststore.password= +#c2.security.truststore.type=JKS +#c2.security.keystore.location= +#c2.security.keystore.password= +#c2.security.keystore.type=JKS +#c2.request.compression=none + + +### MiNiFi Notifiers For Flow Updates +# If C2 is enabled these should be disabled +# Notifiers to use for the associated agent, comma separated list of class names, choose from the options below +#nifi.minifi.notifier.ingestors= + +# File change notifier configuration - org.apache.nifi.minifi.bootstrap.configuration.ingestors.FileChangeIngestor # Path of the file to monitor for changes. When these occur, the FileChangeNotifier, if configured, will begin the configuration reloading process #nifi.minifi.notifier.ingestors.file.config.path= # How frequently the file specified by 'nifi.minifi.notifier.file.config.path' should be evaluated for changes. #nifi.minifi.notifier.ingestors.file.polling.period.seconds=5 -# Rest change notifier configuration - +# Rest change notifier configuration - org.apache.nifi.minifi.bootstrap.configuration.ingestors.RestChangeIngestor # Port on which the Jetty server will bind to, keep commented for a random open port #nifi.minifi.notifier.ingestors.receive.http.port=8338 -#Pull HTTP change notifier configuration - +#Pull HTTP change notifier configuration - org.apache.nifi.minifi.bootstrap.configuration.ingestors.PullHttpChangeIngestor # Hostname on which to pull configurations from #nifi.minifi.notifier.ingestors.pull.http.hostname=localhost # Port on which to pull configurations from @@ -60,12 +145,14 @@ nifi.minifi.config=./conf/config.yml #nifi.minifi.notifier.ingestors.pull.http.query=class=raspi3 # Period on which to pull configurations from, defaults to 5 minutes if commented out #nifi.minifi.notifier.ingestors.pull.http.period.ms=300000 +# Comma separated list of HTTP headers, eg: Accept:application/json,Accept:application/yaml +#nifi.minifi.notifier.ingestors.pull.http.headers= + +### MiNiFi Status Reporter # Periodic Status Reporters to use for the associated agent, comma separated list of class names #nifi.minifi.status.reporter.components=org.apache.nifi.minifi.bootstrap.status.reporters.StatusLogger - # Periodic Status Logger configuration - # The FlowStatus query to submit to the MiNiFi instance #nifi.minifi.status.reporter.log.query=instance:health,bulletins # The log level at which the status will be logged @@ -73,25 +160,45 @@ nifi.minifi.config=./conf/config.yml # The period (in milliseconds) at which to log the status #nifi.minifi.status.reporter.log.period=60000 -# Disable JSR 199 so that we can use JSP's without running a JDK -java.arg.1=-Dorg.apache.jasper.compiler.disablejsr199=true -# JVM memory settings -java.arg.2=-Xms256m -java.arg.3=-Xmx256m +### MiNiFi Provenance Reporting Properties +#nifi.minifi.provenance.reporting.comment= +#nifi.minifi.provenance.reporting.scheduling.strategy= +#nifi.minifi.provenance.reporting.scheduling.period= +#nifi.minifi.provenance.reporting.destination.url= +#nifi.minifi.provenance.reporting.input.port.name= +#nifi.minifi.provenance.reporting.instance.url= +#nifi.minifi.provenance.reporting.batch.size= +#nifi.minifi.provenance.reporting.communications.timeout= +#nifi.minifi.provenance.reporting.compress.events= -# Enable Remote Debugging -#java.arg.debug=-agentlib:jdwp=transport=dt_socket,server=y,suspend=n,address=8000 -# allowRestrictedHeaders is required for Cluster/Node communications to work properly -java.arg.5=-Dsun.net.http.allowRestrictedHeaders=true -java.arg.6=-Djava.protocol.handler.pkgs=sun.net.www.protocol - -# The G1GC is still considered experimental but has proven to be very advantageous in providing great -# performance without significant "stop-the-world" delays. -#java.arg.13=-XX:+UseG1GC - -#Set headless mode by default -java.arg.14=-Djava.awt.headless=true - -java.arg.15=-Djava.security.egd=file:/dev/./urandom \ No newline at end of file +### NiFi Framework Properties +# Core Properties +nifi.flowcontroller.graceful.shutdown.period=10 sec +nifi.flowservice.writedelay.interval=500 ms +nifi.administrative.yield.duration=30 sec +nifi.variable.registry.properties= +nifi.bored.yield.duration=10 millis +# FlowFile Repository +nifi.flowfile.repository.implementation=org.apache.nifi.controller.repository.WriteAheadFlowFileRepository +nifi.flowfile.repository.checkpoint.interval=20 secs +nifi.flowfile.repository.always.sync=false +# Content Repository +nifi.content.repository.implementation=org.apache.nifi.controller.repository.FileSystemRepository +nifi.content.claim.max.appendable.size=50 KB +nifi.content.repository.archive.max.retention.period=7 days +nifi.content.repository.archive.max.usage.percentage=50% +nifi.content.repository.archive.enabled=false +# Provenance Repository +nifi.provenance.repository.implementation=org.apache.nifi.provenance.NoOpProvenanceRepository +nifi.provenance.repository.rollover.time= +nifi.provenance.repository.index.shard.size= +nifi.provenance.repository.max.storage.size= +nifi.provenance.repository.max.storage.time= +# Components Status Repository +nifi.components.status.repository.implementation=org.apache.nifi.controller.status.history.VolatileComponentStatusRepository +nifi.components.status.snapshot.frequency=1 min +# Swap Manager +nifi.swap.manager.implementation=org.apache.nifi.controller.FileSystemSwapManager +nifi.queue.swap.threshold=20000 diff --git a/minifi/minifi-integration-tests/src/test/resources/c2/hierarchical/c2-authoritative/conf/minifi-c2-context.xml b/minifi/minifi-integration-tests/src/test/resources/c2/hierarchical/c2-authoritative/conf/minifi-c2-context.xml index 7e2c6a868c..13604ea0a2 100644 --- a/minifi/minifi-integration-tests/src/test/resources/c2/hierarchical/c2-authoritative/conf/minifi-c2-context.xml +++ b/minifi/minifi-integration-tests/src/test/resources/c2/hierarchical/c2-authoritative/conf/minifi-c2-context.xml @@ -32,7 +32,7 @@ - text/yml + application/json diff --git a/minifi/minifi-integration-tests/src/test/resources/c2/hierarchical/c2-authoritative/files/edge1/raspi3/config.application.json.v1 b/minifi/minifi-integration-tests/src/test/resources/c2/hierarchical/c2-authoritative/files/edge1/raspi3/config.application.json.v1 new file mode 100644 index 0000000000..cbab52041c --- /dev/null +++ b/minifi/minifi-integration-tests/src/test/resources/c2/hierarchical/c2-authoritative/files/edge1/raspi3/config.application.json.v1 @@ -0,0 +1,40 @@ +{ + "encodingVersion": { + "majorVersion": 2, + "minorVersion": 0 + }, + "maxTimerDrivenThreadCount": 10, + "maxEventDrivenThreadCount": 1, + "registries": [], + "parameterContexts": [], + "parameterProviders": [], + "controllerServices": [], + "reportingTasks": [], + "templates": [], + "rootGroup": { + "identifier": "c1b4e586-2011-3f81-a11e-8d669f084d1c", + "instanceIdentifier": "29db3dbc-0188-1000-7025-4cab8b52d278", + "name": "Edge 1 raspi3.v1", + "comments": "", + "position": { + "x": 0.0, + "y": 0.0 + }, + "processGroups": [], + "remoteProcessGroups": [], + "processors": [], + "inputPorts": [], + "outputPorts": [], + "connections": [], + "labels": [], + "funnels": [], + "controllerServices": [], + "variables": {}, + "defaultFlowFileExpiration": "0 sec", + "defaultBackPressureObjectThreshold": 10000, + "defaultBackPressureDataSizeThreshold": "1 GB", + "componentType": "PROCESS_GROUP", + "flowFileConcurrency": "UNBOUNDED", + "flowFileOutboundPolicy": "STREAM_WHEN_AVAILABLE" + } +} \ No newline at end of file diff --git a/minifi/minifi-integration-tests/src/test/resources/c2/hierarchical/c2-authoritative/files/edge1/raspi3/config.text.yml.v1 b/minifi/minifi-integration-tests/src/test/resources/c2/hierarchical/c2-authoritative/files/edge1/raspi3/config.text.yml.v1 deleted file mode 100644 index 459b7e8864..0000000000 --- a/minifi/minifi-integration-tests/src/test/resources/c2/hierarchical/c2-authoritative/files/edge1/raspi3/config.text.yml.v1 +++ /dev/null @@ -1,63 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the \"License\"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an \"AS IS\" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -MiNiFi Config Version: 3 -Flow Controller: - name: Edge 1 raspi3.v1 - comment: '' -Core Properties: - flow controller graceful shutdown period: 10 sec - flow service write delay interval: 500 ms - administrative yield duration: 30 sec - bored yield duration: 10 millis - max concurrent threads: 1 -FlowFile Repository: - partitions: 256 - checkpoint interval: 2 mins - always sync: false - Swap: - threshold: 20000 - in period: 5 sec - in threads: 1 - out period: 5 sec - out threads: 4 -Content Repository: - content claim max appendable size: 10 MB - content claim max flow files: 100 - always sync: false -Provenance Repository: - provenance rollover time: 1 min -Component Status Repository: - buffer size: 1440 - snapshot frequency: 1 min -Security Properties: - keystore: '' - keystore type: '' - keystore password: '' - key password: '' - truststore: '' - truststore type: '' - truststore password: '' - ssl protocol: '' - Sensitive Props: - key: '' - algorithm: NIFI_PBKDF2_AES_GCM_256 - provider: BC -Processors: [] -Process Groups: [] -Funnels: [] -Connections: [] -Remote Process Groups: [] -NiFi Properties Overrides: {} diff --git a/minifi/minifi-integration-tests/src/test/resources/c2/hierarchical/c2-authoritative/files/edge2/raspi2/config.application.json.v1 b/minifi/minifi-integration-tests/src/test/resources/c2/hierarchical/c2-authoritative/files/edge2/raspi2/config.application.json.v1 new file mode 100644 index 0000000000..6a15b9ec94 --- /dev/null +++ b/minifi/minifi-integration-tests/src/test/resources/c2/hierarchical/c2-authoritative/files/edge2/raspi2/config.application.json.v1 @@ -0,0 +1,40 @@ +{ + "encodingVersion": { + "majorVersion": 2, + "minorVersion": 0 + }, + "maxTimerDrivenThreadCount": 10, + "maxEventDrivenThreadCount": 1, + "registries": [], + "parameterContexts": [], + "parameterProviders": [], + "controllerServices": [], + "reportingTasks": [], + "templates": [], + "rootGroup": { + "identifier": "c1b4e586-2011-3f81-a11e-8d669f084d1c", + "instanceIdentifier": "29db3dbc-0188-1000-7025-4cab8b52d278", + "name": "Edge 2 raspi2.v1", + "comments": "", + "position": { + "x": 0.0, + "y": 0.0 + }, + "processGroups": [], + "remoteProcessGroups": [], + "processors": [], + "inputPorts": [], + "outputPorts": [], + "connections": [], + "labels": [], + "funnels": [], + "controllerServices": [], + "variables": {}, + "defaultFlowFileExpiration": "0 sec", + "defaultBackPressureObjectThreshold": 10000, + "defaultBackPressureDataSizeThreshold": "1 GB", + "componentType": "PROCESS_GROUP", + "flowFileConcurrency": "UNBOUNDED", + "flowFileOutboundPolicy": "STREAM_WHEN_AVAILABLE" + } +} \ No newline at end of file diff --git a/minifi/minifi-integration-tests/src/test/resources/c2/hierarchical/c2-authoritative/files/edge2/raspi2/config.text.yml.v1 b/minifi/minifi-integration-tests/src/test/resources/c2/hierarchical/c2-authoritative/files/edge2/raspi2/config.text.yml.v1 deleted file mode 100644 index 772f37994e..0000000000 --- a/minifi/minifi-integration-tests/src/test/resources/c2/hierarchical/c2-authoritative/files/edge2/raspi2/config.text.yml.v1 +++ /dev/null @@ -1,63 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the \"License\"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an \"AS IS\" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -MiNiFi Config Version: 3 -Flow Controller: - name: Edge 2 raspi2.v1 - comment: '' -Core Properties: - flow controller graceful shutdown period: 10 sec - flow service write delay interval: 500 ms - administrative yield duration: 30 sec - bored yield duration: 10 millis - max concurrent threads: 1 -FlowFile Repository: - partitions: 256 - checkpoint interval: 2 mins - always sync: false - Swap: - threshold: 20000 - in period: 5 sec - in threads: 1 - out period: 5 sec - out threads: 4 -Content Repository: - content claim max appendable size: 10 MB - content claim max flow files: 100 - always sync: false -Provenance Repository: - provenance rollover time: 1 min -Component Status Repository: - buffer size: 1440 - snapshot frequency: 1 min -Security Properties: - keystore: '' - keystore type: '' - keystore password: '' - key password: '' - truststore: '' - truststore type: '' - truststore password: '' - ssl protocol: '' - Sensitive Props: - key: '' - algorithm: NIFI_PBKDF2_AES_GCM_256 - provider: BC -Processors: [] -Process Groups: [] -Funnels: [] -Connections: [] -Remote Process Groups: [] -NiFi Properties Overrides: {} diff --git a/minifi/minifi-integration-tests/src/test/resources/c2/hierarchical/c2-authoritative/files/edge3/raspi3/config.application.json.v1 b/minifi/minifi-integration-tests/src/test/resources/c2/hierarchical/c2-authoritative/files/edge3/raspi3/config.application.json.v1 new file mode 100644 index 0000000000..13d5c97edc --- /dev/null +++ b/minifi/minifi-integration-tests/src/test/resources/c2/hierarchical/c2-authoritative/files/edge3/raspi3/config.application.json.v1 @@ -0,0 +1,40 @@ +{ + "encodingVersion": { + "majorVersion": 2, + "minorVersion": 0 + }, + "maxTimerDrivenThreadCount": 10, + "maxEventDrivenThreadCount": 1, + "registries": [], + "parameterContexts": [], + "parameterProviders": [], + "controllerServices": [], + "reportingTasks": [], + "templates": [], + "rootGroup": { + "identifier": "c1b4e586-2011-3f81-a11e-8d669f084d1c", + "instanceIdentifier": "29db3dbc-0188-1000-7025-4cab8b52d278", + "name": "Edge 3 raspi3.v1", + "comments": "", + "position": { + "x": 0.0, + "y": 0.0 + }, + "processGroups": [], + "remoteProcessGroups": [], + "processors": [], + "inputPorts": [], + "outputPorts": [], + "connections": [], + "labels": [], + "funnels": [], + "controllerServices": [], + "variables": {}, + "defaultFlowFileExpiration": "0 sec", + "defaultBackPressureObjectThreshold": 10000, + "defaultBackPressureDataSizeThreshold": "1 GB", + "componentType": "PROCESS_GROUP", + "flowFileConcurrency": "UNBOUNDED", + "flowFileOutboundPolicy": "STREAM_WHEN_AVAILABLE" + } +} \ No newline at end of file diff --git a/minifi/minifi-integration-tests/src/test/resources/c2/hierarchical/c2-authoritative/files/edge3/raspi3/config.text.yml.v1 b/minifi/minifi-integration-tests/src/test/resources/c2/hierarchical/c2-authoritative/files/edge3/raspi3/config.text.yml.v1 deleted file mode 100644 index 9a9ea12388..0000000000 --- a/minifi/minifi-integration-tests/src/test/resources/c2/hierarchical/c2-authoritative/files/edge3/raspi3/config.text.yml.v1 +++ /dev/null @@ -1,63 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the \"License\"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an \"AS IS\" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -MiNiFi Config Version: 3 -Flow Controller: - name: Edge 3 raspi3.v1 - comment: '' -Core Properties: - flow controller graceful shutdown period: 10 sec - flow service write delay interval: 500 ms - administrative yield duration: 30 sec - bored yield duration: 10 millis - max concurrent threads: 1 -FlowFile Repository: - partitions: 256 - checkpoint interval: 2 mins - always sync: false - Swap: - threshold: 20000 - in period: 5 sec - in threads: 1 - out period: 5 sec - out threads: 4 -Content Repository: - content claim max appendable size: 10 MB - content claim max flow files: 100 - always sync: false -Provenance Repository: - provenance rollover time: 1 min -Component Status Repository: - buffer size: 1440 - snapshot frequency: 1 min -Security Properties: - keystore: '' - keystore type: '' - keystore password: '' - key password: '' - truststore: '' - truststore type: '' - truststore password: '' - ssl protocol: '' - Sensitive Props: - key: '' - algorithm: NIFI_PBKDF2_AES_GCM_256 - provider: BC -Processors: [] -Process Groups: [] -Funnels: [] -Connections: [] -Remote Process Groups: [] -NiFi Properties Overrides: {} diff --git a/minifi/minifi-integration-tests/src/test/resources/c2/hierarchical/minifi-edge1/bootstrap.conf b/minifi/minifi-integration-tests/src/test/resources/c2/hierarchical/minifi-edge1/bootstrap.conf index c5cfe5f9e8..a1701e76a7 100644 --- a/minifi/minifi-integration-tests/src/test/resources/c2/hierarchical/minifi-edge1/bootstrap.conf +++ b/minifi/minifi-integration-tests/src/test/resources/c2/hierarchical/minifi-edge1/bootstrap.conf @@ -22,34 +22,120 @@ java=java run.as= # Configure where MiNiFi's lib and conf directories live +# When running as a Windows service set full paths instead of relative paths lib.dir=./lib conf.dir=./conf # How long to wait after telling MiNiFi to shutdown before explicitly killing the Process graceful.shutdown.seconds=20 -# The location for the configuration file -nifi.minifi.config=./conf/config.yml +# Disable JSR 199 so that we can use JSP's without running a JDK +java.arg.1=-Dorg.apache.jasper.compiler.disablejsr199=true -# Notifiers to use for the associated agent, comma separated list of class names -#nifi.minifi.notifier.ingestors=org.apache.nifi.minifi.bootstrap.configuration.ingestors.FileChangeIngestor -#nifi.minifi.notifier.ingestors=org.apache.nifi.minifi.bootstrap.configuration.ingestors.RestChangeIngestor +# JVM memory settings +java.arg.2=-Xms256m +java.arg.3=-Xmx256m + +# allowRestrictedHeaders is required for Cluster/Node communications to work properly +java.arg.5=-Dsun.net.http.allowRestrictedHeaders=true +java.arg.6=-Djava.protocol.handler.pkgs=sun.net.www.protocol + +# Sets the provider of SecureRandom to /dev/urandom to prevent blocking on VMs +java.arg.7=-Djava.security.egd=file:/dev/urandom + +# Garbage Collector Algorithm +java.arg.13=-XX:+UseG1GC + +#Set headless mode by default +java.arg.14=-Djava.awt.headless=true + +# Enable Remote Debugging +#java.arg.debug=-agentlib:jdwp=transport=dt_socket,server=y,suspend=n,address=8000 + + +### MiNiFi Flow Config File +nifi.minifi.flow.config=./conf/flow.json.gz + + +### MiNiFi Security Properties +nifi.minifi.security.keystore= +nifi.minifi.security.keystoreType= +nifi.minifi.security.keystorePasswd= +nifi.minifi.security.keyPasswd= +nifi.minifi.security.truststore= +nifi.minifi.security.truststoreType= +nifi.minifi.security.truststorePasswd= +nifi.minifi.security.ssl.protocol= + +# Ignore all processor SSL controller services and use parent minifi SSL instead +nifi.minifi.flow.use.parent.ssl=false + + +### MiNiFi Sensitive Properties Settings +nifi.minifi.sensitive.props.key= +nifi.minifi.sensitive.props.algorithm= + + +### MiNiFi Command & Control (C2) Configuration +# Enabling C2 Uncomment each of the following options +#c2.enable=true +## heartbeat in milliseconds +#c2.agent.heartbeat.period=5000 +## define parameters about your agent +#c2.agent.class= +# Optional. Defaults to a hardware based unique identifier +#c2.agent.identifier= +# If set to false heartbeat won't contain the manifest. Defaults to true. +#c2.full.heartbeat=false +## define protocol parameters +# DEPRECATED: c2.rest.url and c2.rest.url.ack are deprecated in favor of c2.rest.path.* properties and are target to be removed in future release +# The absolute url of the C2 server's heartbeat endpoint, eg.: http://localhost/c2-server/api/heartbeat +#c2.rest.url= +# The absolute url of the C2 server's acknowledge endpoint, eg.: http://localhost/c2-server/api/acknowledge +#c2.rest.url.ack= +# C2 Rest Path Properties +# The base path of the C2 server's REST API, eg.: http://localhost/c2-server/api +#c2.rest.path.base= +# Relative url of the C2 server's heartbeat endpoint, eg.: /heartbeat +#c2.rest.path.heartbeat= +# Relative url of the C2 server's acknowledge endpoint, eg.: /acknowledge +#c2.rest.path.acknowledge= +## c2 timeouts +#c2.rest.connectionTimeout=5 sec +#c2.rest.readTimeout=5 sec +#c2.rest.callTimeout=10 sec +#c2.config.directory=./conf +#c2.runtime.manifest.identifier=minifi +#c2.runtime.type=minifi-java +# Directory for storing assets downloaded via C2 update/asset command +#c2.asset.directory=./asset +## Define TLS security properties for C2 communications +#c2.security.truststore.location= +#c2.security.truststore.password= +#c2.security.truststore.type=JKS +#c2.security.keystore.location= +#c2.security.keystore.password= +#c2.security.keystore.type=JKS +#c2.request.compression=none +c2.rest.http.headers=Accept:application/json + + +### MiNiFi Notifiers For Flow Updates +# If C2 is enabled these should be disabled +# Notifiers to use for the associated agent, comma separated list of class names, choose from the options below nifi.minifi.notifier.ingestors=org.apache.nifi.minifi.bootstrap.configuration.ingestors.PullHttpChangeIngestor -# File change notifier configuration - +# File change notifier configuration - org.apache.nifi.minifi.bootstrap.configuration.ingestors.FileChangeIngestor # Path of the file to monitor for changes. When these occur, the FileChangeNotifier, if configured, will begin the configuration reloading process #nifi.minifi.notifier.ingestors.file.config.path= # How frequently the file specified by 'nifi.minifi.notifier.file.config.path' should be evaluated for changes. #nifi.minifi.notifier.ingestors.file.polling.period.seconds=5 -# Rest change notifier configuration - +# Rest change notifier configuration - org.apache.nifi.minifi.bootstrap.configuration.ingestors.RestChangeIngestor # Port on which the Jetty server will bind to, keep commented for a random open port #nifi.minifi.notifier.ingestors.receive.http.port=8338 -#Pull HTTP change notifier configuration - +#Pull HTTP change notifier configuration - org.apache.nifi.minifi.bootstrap.configuration.ingestors.PullHttpChangeIngestor # Hostname on which to pull configurations from nifi.minifi.notifier.ingestors.pull.http.hostname=c2-authoritative # Port on which to pull configurations from @@ -60,7 +146,9 @@ nifi.minifi.notifier.ingestors.pull.http.path=/c2/config nifi.minifi.notifier.ingestors.pull.http.query=net=edge1&class=raspi3 # Period on which to pull configurations from, defaults to 5 minutes if commented out nifi.minifi.notifier.ingestors.pull.http.period.ms=3000 - +# Comma separated list of HTTP headers, eg: Accept:application/json,Accept:application/yaml +nifi.minifi.notifier.ingestors.pull.http.headers=Accept:application/json +# Pull Ingestor Security Properties nifi.minifi.notifier.ingestors.pull.http.keystore.location=./conf/keystore.jks nifi.minifi.notifier.ingestors.pull.http.keystore.type=JKS nifi.minifi.notifier.ingestors.pull.http.keystore.password=badKeystorePass @@ -68,11 +156,11 @@ nifi.minifi.notifier.ingestors.pull.http.truststore.location=./conf/truststore.j nifi.minifi.notifier.ingestors.pull.http.truststore.type=JKS nifi.minifi.notifier.ingestors.pull.http.truststore.password=badTrustPass + +### MiNiFi Status Reporter # Periodic Status Reporters to use for the associated agent, comma separated list of class names #nifi.minifi.status.reporter.components=org.apache.nifi.minifi.bootstrap.status.reporters.StatusLogger - # Periodic Status Logger configuration - # The FlowStatus query to submit to the MiNiFi instance #nifi.minifi.status.reporter.log.query=instance:health,bulletins # The log level at which the status will be logged @@ -80,25 +168,45 @@ nifi.minifi.notifier.ingestors.pull.http.truststore.password=badTrustPass # The period (in milliseconds) at which to log the status #nifi.minifi.status.reporter.log.period=60000 -# Disable JSR 199 so that we can use JSP's without running a JDK -java.arg.1=-Dorg.apache.jasper.compiler.disablejsr199=true -# JVM memory settings -java.arg.2=-Xms256m -java.arg.3=-Xmx256m +### MiNiFi Provenance Reporting Properties +#nifi.minifi.provenance.reporting.comment= +#nifi.minifi.provenance.reporting.scheduling.strategy= +#nifi.minifi.provenance.reporting.scheduling.period= +#nifi.minifi.provenance.reporting.destination.url= +#nifi.minifi.provenance.reporting.input.port.name= +#nifi.minifi.provenance.reporting.instance.url= +#nifi.minifi.provenance.reporting.batch.size= +#nifi.minifi.provenance.reporting.communications.timeout= +#nifi.minifi.provenance.reporting.compress.events= -# Enable Remote Debugging -#java.arg.debug=-agentlib:jdwp=transport=dt_socket,server=y,suspend=n,address=8000 -# allowRestrictedHeaders is required for Cluster/Node communications to work properly -java.arg.5=-Dsun.net.http.allowRestrictedHeaders=true -java.arg.6=-Djava.protocol.handler.pkgs=sun.net.www.protocol - -# The G1GC is still considered experimental but has proven to be very advantageous in providing great -# performance without significant "stop-the-world" delays. -#java.arg.13=-XX:+UseG1GC - -#Set headless mode by default -java.arg.14=-Djava.awt.headless=true - -java.arg.15=-Djava.security.egd=file:/dev/./urandom \ No newline at end of file +### NiFi Framework Properties +# Core Properties +nifi.flowcontroller.graceful.shutdown.period=10 sec +nifi.flowservice.writedelay.interval=500 ms +nifi.administrative.yield.duration=30 sec +nifi.variable.registry.properties= +nifi.bored.yield.duration=10 millis +# FlowFile Repository +nifi.flowfile.repository.implementation=org.apache.nifi.controller.repository.WriteAheadFlowFileRepository +nifi.flowfile.repository.checkpoint.interval=20 secs +nifi.flowfile.repository.always.sync=false +# Content Repository +nifi.content.repository.implementation=org.apache.nifi.controller.repository.FileSystemRepository +nifi.content.claim.max.appendable.size=50 KB +nifi.content.repository.archive.max.retention.period=7 days +nifi.content.repository.archive.max.usage.percentage=50% +nifi.content.repository.archive.enabled=false +# Provenance Repository +nifi.provenance.repository.implementation=org.apache.nifi.provenance.NoOpProvenanceRepository +nifi.provenance.repository.rollover.time= +nifi.provenance.repository.index.shard.size= +nifi.provenance.repository.max.storage.size= +nifi.provenance.repository.max.storage.time= +# Components Status Repository +nifi.components.status.repository.implementation=org.apache.nifi.controller.status.history.VolatileComponentStatusRepository +nifi.components.status.snapshot.frequency=1 min +# Swap Manager +nifi.swap.manager.implementation=org.apache.nifi.controller.FileSystemSwapManager +nifi.queue.swap.threshold=20000 diff --git a/minifi/minifi-integration-tests/src/test/resources/c2/hierarchical/minifi-edge1/expected.json b/minifi/minifi-integration-tests/src/test/resources/c2/hierarchical/minifi-edge1/expected.json index 244a68e8c0..0737f438d7 100644 --- a/minifi/minifi-integration-tests/src/test/resources/c2/hierarchical/minifi-edge1/expected.json +++ b/minifi/minifi-integration-tests/src/test/resources/c2/hierarchical/minifi-edge1/expected.json @@ -3,6 +3,6 @@ "pattern": "ConfigurationChangeCoordinator Notifying Listeners of a change" }, { - "pattern": "MiNiFi has finished reloading successfully and swap.yml file exists. Deleting old configuration" + "pattern": "MiNiFi has finished reloading successfully and applied the new flow configuration" } ] \ No newline at end of file diff --git a/minifi/minifi-integration-tests/src/test/resources/c2/hierarchical/minifi-edge2/bootstrap.conf b/minifi/minifi-integration-tests/src/test/resources/c2/hierarchical/minifi-edge2/bootstrap.conf index 1bcad1596d..1dec7e78da 100644 --- a/minifi/minifi-integration-tests/src/test/resources/c2/hierarchical/minifi-edge2/bootstrap.conf +++ b/minifi/minifi-integration-tests/src/test/resources/c2/hierarchical/minifi-edge2/bootstrap.conf @@ -22,34 +22,120 @@ java=java run.as= # Configure where MiNiFi's lib and conf directories live +# When running as a Windows service set full paths instead of relative paths lib.dir=./lib conf.dir=./conf # How long to wait after telling MiNiFi to shutdown before explicitly killing the Process graceful.shutdown.seconds=20 -# The location for the configuration file -nifi.minifi.config=./conf/config.yml +# Disable JSR 199 so that we can use JSP's without running a JDK +java.arg.1=-Dorg.apache.jasper.compiler.disablejsr199=true -# Notifiers to use for the associated agent, comma separated list of class names -#nifi.minifi.notifier.ingestors=org.apache.nifi.minifi.bootstrap.configuration.ingestors.FileChangeIngestor -#nifi.minifi.notifier.ingestors=org.apache.nifi.minifi.bootstrap.configuration.ingestors.RestChangeIngestor +# JVM memory settings +java.arg.2=-Xms256m +java.arg.3=-Xmx256m + +# allowRestrictedHeaders is required for Cluster/Node communications to work properly +java.arg.5=-Dsun.net.http.allowRestrictedHeaders=true +java.arg.6=-Djava.protocol.handler.pkgs=sun.net.www.protocol + +# Sets the provider of SecureRandom to /dev/urandom to prevent blocking on VMs +java.arg.7=-Djava.security.egd=file:/dev/urandom + +# Garbage Collector Algorithm +java.arg.13=-XX:+UseG1GC + +#Set headless mode by default +java.arg.14=-Djava.awt.headless=true + +# Enable Remote Debugging +#java.arg.debug=-agentlib:jdwp=transport=dt_socket,server=y,suspend=n,address=8000 + + +### MiNiFi Flow Config File +nifi.minifi.flow.config=./conf/flow.json.gz + + +### MiNiFi Security Properties +nifi.minifi.security.keystore= +nifi.minifi.security.keystoreType= +nifi.minifi.security.keystorePasswd= +nifi.minifi.security.keyPasswd= +nifi.minifi.security.truststore= +nifi.minifi.security.truststoreType= +nifi.minifi.security.truststorePasswd= +nifi.minifi.security.ssl.protocol= + +# Ignore all processor SSL controller services and use parent minifi SSL instead +nifi.minifi.flow.use.parent.ssl=false + + +### MiNiFi Sensitive Properties Settings +nifi.minifi.sensitive.props.key= +nifi.minifi.sensitive.props.algorithm= + + +### MiNiFi Command & Control (C2) Configuration +# Enabling C2 Uncomment each of the following options +#c2.enable=true +## heartbeat in milliseconds +#c2.agent.heartbeat.period=5000 +## define parameters about your agent +#c2.agent.class= +# Optional. Defaults to a hardware based unique identifier +#c2.agent.identifier= +# If set to false heartbeat won't contain the manifest. Defaults to true. +#c2.full.heartbeat=false +## define protocol parameters +# DEPRECATED: c2.rest.url and c2.rest.url.ack are deprecated in favor of c2.rest.path.* properties and are target to be removed in future release +# The absolute url of the C2 server's heartbeat endpoint, eg.: http://localhost/c2-server/api/heartbeat +#c2.rest.url= +# The absolute url of the C2 server's acknowledge endpoint, eg.: http://localhost/c2-server/api/acknowledge +#c2.rest.url.ack= +# C2 Rest Path Properties +# The base path of the C2 server's REST API, eg.: http://localhost/c2-server/api +#c2.rest.path.base= +# Relative url of the C2 server's heartbeat endpoint, eg.: /heartbeat +#c2.rest.path.heartbeat= +# Relative url of the C2 server's acknowledge endpoint, eg.: /acknowledge +#c2.rest.path.acknowledge= +## c2 timeouts +#c2.rest.connectionTimeout=5 sec +#c2.rest.readTimeout=5 sec +#c2.rest.callTimeout=10 sec +#c2.config.directory=./conf +#c2.runtime.manifest.identifier=minifi +#c2.runtime.type=minifi-java +# Directory for storing assets downloaded via C2 update/asset command +#c2.asset.directory=./asset +## Define TLS security properties for C2 communications +#c2.security.truststore.location= +#c2.security.truststore.password= +#c2.security.truststore.type=JKS +#c2.security.keystore.location= +#c2.security.keystore.password= +#c2.security.keystore.type=JKS +#c2.request.compression=none +c2.rest.http.headers=Accept:application/json + + +### MiNiFi Notifiers For Flow Updates +# If C2 is enabled these should be disabled +# Notifiers to use for the associated agent, comma separated list of class names, choose from the options below nifi.minifi.notifier.ingestors=org.apache.nifi.minifi.bootstrap.configuration.ingestors.PullHttpChangeIngestor -# File change notifier configuration - +# File change notifier configuration - org.apache.nifi.minifi.bootstrap.configuration.ingestors.FileChangeIngestor # Path of the file to monitor for changes. When these occur, the FileChangeNotifier, if configured, will begin the configuration reloading process #nifi.minifi.notifier.ingestors.file.config.path= # How frequently the file specified by 'nifi.minifi.notifier.file.config.path' should be evaluated for changes. #nifi.minifi.notifier.ingestors.file.polling.period.seconds=5 -# Rest change notifier configuration - +# Rest change notifier configuration - org.apache.nifi.minifi.bootstrap.configuration.ingestors.RestChangeIngestor # Port on which the Jetty server will bind to, keep commented for a random open port #nifi.minifi.notifier.ingestors.receive.http.port=8338 -#Pull HTTP change notifier configuration - +#Pull HTTP change notifier configuration - org.apache.nifi.minifi.bootstrap.configuration.ingestors.PullHttpChangeIngestor # Hostname on which to pull configurations from nifi.minifi.notifier.ingestors.pull.http.hostname=c2-edge2 # Port on which to pull configurations from @@ -60,12 +146,14 @@ nifi.minifi.notifier.ingestors.pull.http.path=/c2/config nifi.minifi.notifier.ingestors.pull.http.query=net=edge2&class=raspi2 # Period on which to pull configurations from, defaults to 5 minutes if commented out nifi.minifi.notifier.ingestors.pull.http.period.ms=3000 +# Comma separated list of HTTP headers, eg: Accept:application/json,Accept:application/yaml +nifi.minifi.notifier.ingestors.pull.http.headers=Accept:application/json + +### MiNiFi Status Reporter # Periodic Status Reporters to use for the associated agent, comma separated list of class names #nifi.minifi.status.reporter.components=org.apache.nifi.minifi.bootstrap.status.reporters.StatusLogger - # Periodic Status Logger configuration - # The FlowStatus query to submit to the MiNiFi instance #nifi.minifi.status.reporter.log.query=instance:health,bulletins # The log level at which the status will be logged @@ -73,25 +161,45 @@ nifi.minifi.notifier.ingestors.pull.http.period.ms=3000 # The period (in milliseconds) at which to log the status #nifi.minifi.status.reporter.log.period=60000 -# Disable JSR 199 so that we can use JSP's without running a JDK -java.arg.1=-Dorg.apache.jasper.compiler.disablejsr199=true -# JVM memory settings -java.arg.2=-Xms256m -java.arg.3=-Xmx256m +### MiNiFi Provenance Reporting Properties +#nifi.minifi.provenance.reporting.comment= +#nifi.minifi.provenance.reporting.scheduling.strategy= +#nifi.minifi.provenance.reporting.scheduling.period= +#nifi.minifi.provenance.reporting.destination.url= +#nifi.minifi.provenance.reporting.input.port.name= +#nifi.minifi.provenance.reporting.instance.url= +#nifi.minifi.provenance.reporting.batch.size= +#nifi.minifi.provenance.reporting.communications.timeout= +#nifi.minifi.provenance.reporting.compress.events= -# Enable Remote Debugging -#java.arg.debug=-agentlib:jdwp=transport=dt_socket,server=y,suspend=n,address=8000 -# allowRestrictedHeaders is required for Cluster/Node communications to work properly -java.arg.5=-Dsun.net.http.allowRestrictedHeaders=true -java.arg.6=-Djava.protocol.handler.pkgs=sun.net.www.protocol - -# The G1GC is still considered experimental but has proven to be very advantageous in providing great -# performance without significant "stop-the-world" delays. -#java.arg.13=-XX:+UseG1GC - -#Set headless mode by default -java.arg.14=-Djava.awt.headless=true - -java.arg.15=-Djava.security.egd=file:/dev/./urandom \ No newline at end of file +### NiFi Framework Properties +# Core Properties +nifi.flowcontroller.graceful.shutdown.period=10 sec +nifi.flowservice.writedelay.interval=500 ms +nifi.administrative.yield.duration=30 sec +nifi.variable.registry.properties= +nifi.bored.yield.duration=10 millis +# FlowFile Repository +nifi.flowfile.repository.implementation=org.apache.nifi.controller.repository.WriteAheadFlowFileRepository +nifi.flowfile.repository.checkpoint.interval=20 secs +nifi.flowfile.repository.always.sync=false +# Content Repository +nifi.content.repository.implementation=org.apache.nifi.controller.repository.FileSystemRepository +nifi.content.claim.max.appendable.size=50 KB +nifi.content.repository.archive.max.retention.period=7 days +nifi.content.repository.archive.max.usage.percentage=50% +nifi.content.repository.archive.enabled=false +# Provenance Repository +nifi.provenance.repository.implementation=org.apache.nifi.provenance.NoOpProvenanceRepository +nifi.provenance.repository.rollover.time= +nifi.provenance.repository.index.shard.size= +nifi.provenance.repository.max.storage.size= +nifi.provenance.repository.max.storage.time= +# Components Status Repository +nifi.components.status.repository.implementation=org.apache.nifi.controller.status.history.VolatileComponentStatusRepository +nifi.components.status.snapshot.frequency=1 min +# Swap Manager +nifi.swap.manager.implementation=org.apache.nifi.controller.FileSystemSwapManager +nifi.queue.swap.threshold=20000 diff --git a/minifi/minifi-integration-tests/src/test/resources/c2/hierarchical/minifi-edge2/expected.json b/minifi/minifi-integration-tests/src/test/resources/c2/hierarchical/minifi-edge2/expected.json index 244a68e8c0..0737f438d7 100644 --- a/minifi/minifi-integration-tests/src/test/resources/c2/hierarchical/minifi-edge2/expected.json +++ b/minifi/minifi-integration-tests/src/test/resources/c2/hierarchical/minifi-edge2/expected.json @@ -3,6 +3,6 @@ "pattern": "ConfigurationChangeCoordinator Notifying Listeners of a change" }, { - "pattern": "MiNiFi has finished reloading successfully and swap.yml file exists. Deleting old configuration" + "pattern": "MiNiFi has finished reloading successfully and applied the new flow configuration" } ] \ No newline at end of file diff --git a/minifi/minifi-integration-tests/src/test/resources/c2/hierarchical/minifi-edge3/bootstrap.conf b/minifi/minifi-integration-tests/src/test/resources/c2/hierarchical/minifi-edge3/bootstrap.conf index c0de827e3c..7ac172baf4 100644 --- a/minifi/minifi-integration-tests/src/test/resources/c2/hierarchical/minifi-edge3/bootstrap.conf +++ b/minifi/minifi-integration-tests/src/test/resources/c2/hierarchical/minifi-edge3/bootstrap.conf @@ -22,34 +22,120 @@ java=java run.as= # Configure where MiNiFi's lib and conf directories live +# When running as a Windows service set full paths instead of relative paths lib.dir=./lib conf.dir=./conf # How long to wait after telling MiNiFi to shutdown before explicitly killing the Process graceful.shutdown.seconds=20 -# The location for the configuration file -nifi.minifi.config=./conf/config.yml +# Disable JSR 199 so that we can use JSP's without running a JDK +java.arg.1=-Dorg.apache.jasper.compiler.disablejsr199=true -# Notifiers to use for the associated agent, comma separated list of class names -#nifi.minifi.notifier.ingestors=org.apache.nifi.minifi.bootstrap.configuration.ingestors.FileChangeIngestor -#nifi.minifi.notifier.ingestors=org.apache.nifi.minifi.bootstrap.configuration.ingestors.RestChangeIngestor +# JVM memory settings +java.arg.2=-Xms256m +java.arg.3=-Xmx256m + +# allowRestrictedHeaders is required for Cluster/Node communications to work properly +java.arg.5=-Dsun.net.http.allowRestrictedHeaders=true +java.arg.6=-Djava.protocol.handler.pkgs=sun.net.www.protocol + +# Sets the provider of SecureRandom to /dev/urandom to prevent blocking on VMs +java.arg.7=-Djava.security.egd=file:/dev/urandom + +# Garbage Collector Algorithm +java.arg.13=-XX:+UseG1GC + +#Set headless mode by default +java.arg.14=-Djava.awt.headless=true + +# Enable Remote Debugging +#java.arg.debug=-agentlib:jdwp=transport=dt_socket,server=y,suspend=n,address=8000 + + +### MiNiFi Flow Config File +nifi.minifi.flow.config=./conf/flow.json.gz + + +### MiNiFi Security Properties +nifi.minifi.security.keystore= +nifi.minifi.security.keystoreType= +nifi.minifi.security.keystorePasswd= +nifi.minifi.security.keyPasswd= +nifi.minifi.security.truststore= +nifi.minifi.security.truststoreType= +nifi.minifi.security.truststorePasswd= +nifi.minifi.security.ssl.protocol= + +# Ignore all processor SSL controller services and use parent minifi SSL instead +nifi.minifi.flow.use.parent.ssl=false + + +### MiNiFi Sensitive Properties Settings +nifi.minifi.sensitive.props.key= +nifi.minifi.sensitive.props.algorithm= + + +### MiNiFi Command & Control (C2) Configuration +# Enabling C2 Uncomment each of the following options +#c2.enable=true +## heartbeat in milliseconds +#c2.agent.heartbeat.period=5000 +## define parameters about your agent +#c2.agent.class= +# Optional. Defaults to a hardware based unique identifier +#c2.agent.identifier= +# If set to false heartbeat won't contain the manifest. Defaults to true. +#c2.full.heartbeat=false +## define protocol parameters +# DEPRECATED: c2.rest.url and c2.rest.url.ack are deprecated in favor of c2.rest.path.* properties and are target to be removed in future release +# The absolute url of the C2 server's heartbeat endpoint, eg.: http://localhost/c2-server/api/heartbeat +#c2.rest.url= +# The absolute url of the C2 server's acknowledge endpoint, eg.: http://localhost/c2-server/api/acknowledge +#c2.rest.url.ack= +# C2 Rest Path Properties +# The base path of the C2 server's REST API, eg.: http://localhost/c2-server/api +#c2.rest.path.base= +# Relative url of the C2 server's heartbeat endpoint, eg.: /heartbeat +#c2.rest.path.heartbeat= +# Relative url of the C2 server's acknowledge endpoint, eg.: /acknowledge +#c2.rest.path.acknowledge= +## c2 timeouts +#c2.rest.connectionTimeout=5 sec +#c2.rest.readTimeout=5 sec +#c2.rest.callTimeout=10 sec +#c2.config.directory=./conf +#c2.runtime.manifest.identifier=minifi +#c2.runtime.type=minifi-java +# Directory for storing assets downloaded via C2 update/asset command +#c2.asset.directory=./asset +## Define TLS security properties for C2 communications +#c2.security.truststore.location= +#c2.security.truststore.password= +#c2.security.truststore.type=JKS +#c2.security.keystore.location= +#c2.security.keystore.password= +#c2.security.keystore.type=JKS +#c2.request.compression=none +c2.rest.http.headers=Accept:application/json + + +### MiNiFi Notifiers For Flow Updates +# If C2 is enabled these should be disabled +# Notifiers to use for the associated agent, comma separated list of class names, choose from the options below nifi.minifi.notifier.ingestors=org.apache.nifi.minifi.bootstrap.configuration.ingestors.PullHttpChangeIngestor -# File change notifier configuration - +# File change notifier configuration - org.apache.nifi.minifi.bootstrap.configuration.ingestors.FileChangeIngestor # Path of the file to monitor for changes. When these occur, the FileChangeNotifier, if configured, will begin the configuration reloading process #nifi.minifi.notifier.ingestors.file.config.path= # How frequently the file specified by 'nifi.minifi.notifier.file.config.path' should be evaluated for changes. #nifi.minifi.notifier.ingestors.file.polling.period.seconds=5 -# Rest change notifier configuration - +# Rest change notifier configuration - org.apache.nifi.minifi.bootstrap.configuration.ingestors.RestChangeIngestor # Port on which the Jetty server will bind to, keep commented for a random open port #nifi.minifi.notifier.ingestors.receive.http.port=8338 -#Pull HTTP change notifier configuration - +#Pull HTTP change notifier configuration - org.apache.nifi.minifi.bootstrap.configuration.ingestors.PullHttpChangeIngestor # Hostname on which to pull configurations from nifi.minifi.notifier.ingestors.pull.http.hostname=c2-authoritative # Port on which to pull configurations from @@ -60,6 +146,8 @@ nifi.minifi.notifier.ingestors.pull.http.path=/c2/config nifi.minifi.notifier.ingestors.pull.http.query=net=edge3&class=raspi3 # Period on which to pull configurations from, defaults to 5 minutes if commented out nifi.minifi.notifier.ingestors.pull.http.period.ms=3000 +# Comma separated list of HTTP headers, eg: Accept:application/json,Accept:application/yaml +nifi.minifi.notifier.ingestors.pull.http.headers=Accept:application/json nifi.minifi.notifier.ingestors.pull.http.proxy.hostname=squid-edge3 nifi.minifi.notifier.ingestors.pull.http.proxy.port=3128 @@ -71,11 +159,11 @@ nifi.minifi.notifier.ingestors.pull.http.truststore.location=./conf/truststore.j nifi.minifi.notifier.ingestors.pull.http.truststore.type=JKS nifi.minifi.notifier.ingestors.pull.http.truststore.password=badTrustPass + +### MiNiFi Status Reporter # Periodic Status Reporters to use for the associated agent, comma separated list of class names #nifi.minifi.status.reporter.components=org.apache.nifi.minifi.bootstrap.status.reporters.StatusLogger - # Periodic Status Logger configuration - # The FlowStatus query to submit to the MiNiFi instance #nifi.minifi.status.reporter.log.query=instance:health,bulletins # The log level at which the status will be logged @@ -83,25 +171,45 @@ nifi.minifi.notifier.ingestors.pull.http.truststore.password=badTrustPass # The period (in milliseconds) at which to log the status #nifi.minifi.status.reporter.log.period=60000 -# Disable JSR 199 so that we can use JSP's without running a JDK -java.arg.1=-Dorg.apache.jasper.compiler.disablejsr199=true -# JVM memory settings -java.arg.2=-Xms256m -java.arg.3=-Xmx256m +### MiNiFi Provenance Reporting Properties +#nifi.minifi.provenance.reporting.comment= +#nifi.minifi.provenance.reporting.scheduling.strategy= +#nifi.minifi.provenance.reporting.scheduling.period= +#nifi.minifi.provenance.reporting.destination.url= +#nifi.minifi.provenance.reporting.input.port.name= +#nifi.minifi.provenance.reporting.instance.url= +#nifi.minifi.provenance.reporting.batch.size= +#nifi.minifi.provenance.reporting.communications.timeout= +#nifi.minifi.provenance.reporting.compress.events= -# Enable Remote Debugging -#java.arg.debug=-agentlib:jdwp=transport=dt_socket,server=y,suspend=n,address=8000 -# allowRestrictedHeaders is required for Cluster/Node communications to work properly -java.arg.5=-Dsun.net.http.allowRestrictedHeaders=true -java.arg.6=-Djava.protocol.handler.pkgs=sun.net.www.protocol - -# The G1GC is still considered experimental but has proven to be very advantageous in providing great -# performance without significant "stop-the-world" delays. -#java.arg.13=-XX:+UseG1GC - -#Set headless mode by default -java.arg.14=-Djava.awt.headless=true - -java.arg.15=-Djava.security.egd=file:/dev/./urandom \ No newline at end of file +### NiFi Framework Properties +# Core Properties +nifi.flowcontroller.graceful.shutdown.period=10 sec +nifi.flowservice.writedelay.interval=500 ms +nifi.administrative.yield.duration=30 sec +nifi.variable.registry.properties= +nifi.bored.yield.duration=10 millis +# FlowFile Repository +nifi.flowfile.repository.implementation=org.apache.nifi.controller.repository.WriteAheadFlowFileRepository +nifi.flowfile.repository.checkpoint.interval=20 secs +nifi.flowfile.repository.always.sync=false +# Content Repository +nifi.content.repository.implementation=org.apache.nifi.controller.repository.FileSystemRepository +nifi.content.claim.max.appendable.size=50 KB +nifi.content.repository.archive.max.retention.period=7 days +nifi.content.repository.archive.max.usage.percentage=50% +nifi.content.repository.archive.enabled=false +# Provenance Repository +nifi.provenance.repository.implementation=org.apache.nifi.provenance.NoOpProvenanceRepository +nifi.provenance.repository.rollover.time= +nifi.provenance.repository.index.shard.size= +nifi.provenance.repository.max.storage.size= +nifi.provenance.repository.max.storage.time= +# Components Status Repository +nifi.components.status.repository.implementation=org.apache.nifi.controller.status.history.VolatileComponentStatusRepository +nifi.components.status.snapshot.frequency=1 min +# Swap Manager +nifi.swap.manager.implementation=org.apache.nifi.controller.FileSystemSwapManager +nifi.queue.swap.threshold=20000 diff --git a/minifi/minifi-integration-tests/src/test/resources/c2/hierarchical/minifi-edge3/expected.json b/minifi/minifi-integration-tests/src/test/resources/c2/hierarchical/minifi-edge3/expected.json index 244a68e8c0..0737f438d7 100644 --- a/minifi/minifi-integration-tests/src/test/resources/c2/hierarchical/minifi-edge3/expected.json +++ b/minifi/minifi-integration-tests/src/test/resources/c2/hierarchical/minifi-edge3/expected.json @@ -3,6 +3,6 @@ "pattern": "ConfigurationChangeCoordinator Notifying Listeners of a change" }, { - "pattern": "MiNiFi has finished reloading successfully and swap.yml file exists. Deleting old configuration" + "pattern": "MiNiFi has finished reloading successfully and applied the new flow configuration" } ] \ No newline at end of file diff --git a/minifi/minifi-integration-tests/src/test/resources/c2/protocol/c2-authoritative/conf/minifi-c2-context.xml b/minifi/minifi-integration-tests/src/test/resources/c2/protocol/c2-authoritative/conf/minifi-c2-context.xml index 44ef049fc7..a4ca4b8128 100644 --- a/minifi/minifi-integration-tests/src/test/resources/c2/protocol/c2-authoritative/conf/minifi-c2-context.xml +++ b/minifi/minifi-integration-tests/src/test/resources/c2/protocol/c2-authoritative/conf/minifi-c2-context.xml @@ -32,7 +32,7 @@ - text/yml + application/json diff --git a/minifi/minifi-integration-tests/src/test/resources/c2/protocol/c2-authoritative/files/raspi3/config.application.json.v1 b/minifi/minifi-integration-tests/src/test/resources/c2/protocol/c2-authoritative/files/raspi3/config.application.json.v1 new file mode 100644 index 0000000000..d32926b277 --- /dev/null +++ b/minifi/minifi-integration-tests/src/test/resources/c2/protocol/c2-authoritative/files/raspi3/config.application.json.v1 @@ -0,0 +1,163 @@ +{ + "encodingVersion": { + "majorVersion": 2, + "minorVersion": 0 + }, + "maxTimerDrivenThreadCount": 1, + "maxEventDrivenThreadCount": 1, + "registries": [], + "parameterContexts": [], + "parameterProviders": [], + "controllerServices": [], + "reportingTasks": [], + "templates": [], + "rootGroup": { + "identifier": "f4cd72f7-b235-394b-928a-ffb0e7cf2db6", + "instanceIdentifier": "Root-Group", + "name": "Edge raspi3 v1.0", + "comments": "", + "position": { + "x": 0.0, + "y": 0.0 + }, + "processGroups": [], + "remoteProcessGroups": [], + "processors": [ + { + "identifier": "cc7aa68d-7c45-3fd6-8eb7-297de95b0caa", + "instanceIdentifier": "186322b1-4778-40e4-ba9d-a0a511d762f7", + "name": "LogAttribute", + "comments": "", + "position": { + "x": 0.0, + "y": 0.0 + }, + "type": "org.apache.nifi.processors.standard.LogAttribute", + "bundle": { + "group": "org.apache.nifi.minifi", + "artifact": "minifi-standard-nar", + "version": "2.0.0-SNAPSHOT" + }, + "properties": { + "character-set": "UTF-8", + "Log FlowFile Properties": "true", + "Log Level": "info", + "attributes-to-log-regex": ".*", + "Output Format": "Line per Attribute", + "Log Payload": "true" + }, + "propertyDescriptors": {}, + "style": {}, + "schedulingPeriod": "1000 ms", + "schedulingStrategy": "TIMER_DRIVEN", + "executionNode": "ALL", + "penaltyDuration": "30000 ms", + "yieldDuration": "1000 ms", + "bulletinLevel": "WARN", + "runDurationMillis": 0, + "concurrentlySchedulableTaskCount": 1, + "autoTerminatedRelationships": [ + "success" + ], + "scheduledState": "RUNNING", + "retryCount": 10, + "retriedRelationships": [], + "backoffMechanism": "PENALIZE_FLOWFILE", + "maxBackoffPeriod": "10 mins", + "componentType": "PROCESSOR", + "groupIdentifier": "f4cd72f7-b235-394b-928a-ffb0e7cf2db6" + }, + { + "identifier": "8a22101a-ed02-3ebc-a01d-cab50ba1ba0e", + "instanceIdentifier": "c6a06b10-0af2-423b-9927-0bb8e6e04bc3", + "name": "GenerateTestText", + "comments": "", + "position": { + "x": 0.0, + "y": 0.0 + }, + "type": "org.apache.nifi.processors.standard.GenerateFlowFile", + "bundle": { + "group": "org.apache.nifi.minifi", + "artifact": "minifi-standard-nar", + "version": "2.0.0-SNAPSHOT" + }, + "properties": { + "character-set": "UTF-8", + "File Size": "0B", + "generate-ff-custom-text": "__testTextRaspi3__", + "Batch Size": "1", + "Unique FlowFiles": "false", + "Data Format": "Text" + }, + "propertyDescriptors": {}, + "style": {}, + "schedulingPeriod": "1000 ms", + "schedulingStrategy": "TIMER_DRIVEN", + "executionNode": "ALL", + "penaltyDuration": "30000 ms", + "yieldDuration": "1000 ms", + "bulletinLevel": "WARN", + "runDurationMillis": 0, + "concurrentlySchedulableTaskCount": 1, + "autoTerminatedRelationships": [], + "scheduledState": "RUNNING", + "retryCount": 10, + "retriedRelationships": [], + "backoffMechanism": "PENALIZE_FLOWFILE", + "maxBackoffPeriod": "10 mins", + "componentType": "PROCESSOR", + "groupIdentifier": "f4cd72f7-b235-394b-928a-ffb0e7cf2db6" + } + ], + "inputPorts": [], + "outputPorts": [], + "connections": [ + { + "identifier": "52446a69-fbc8-367a-b391-96571e5f5b4b", + "instanceIdentifier": "641b0183-8653-4988-9a74-bcf9780b1397", + "name": "GenerateTestText/success/LogAttribute", + "source": { + "id": "8a22101a-ed02-3ebc-a01d-cab50ba1ba0e", + "type": "PROCESSOR", + "groupId": "f4cd72f7-b235-394b-928a-ffb0e7cf2db6", + "name": "GenerateTestText", + "comments": "", + "instanceIdentifier": "c6a06b10-0af2-423b-9927-0bb8e6e04bc3" + }, + "destination": { + "id": "cc7aa68d-7c45-3fd6-8eb7-297de95b0caa", + "type": "PROCESSOR", + "groupId": "f4cd72f7-b235-394b-928a-ffb0e7cf2db6", + "name": "LogAttribute", + "comments": "", + "instanceIdentifier": "186322b1-4778-40e4-ba9d-a0a511d762f7" + }, + "labelIndex": 1, + "zIndex": 0, + "selectedRelationships": [ + "success" + ], + "backPressureObjectThreshold": 10000, + "backPressureDataSizeThreshold": "10 MB", + "flowFileExpiration": "0 seconds", + "prioritizers": [], + "bends": [], + "loadBalanceStrategy": "DO_NOT_LOAD_BALANCE", + "loadBalanceCompression": "DO_NOT_COMPRESS", + "componentType": "CONNECTION", + "groupIdentifier": "f4cd72f7-b235-394b-928a-ffb0e7cf2db6" + } + ], + "labels": [], + "funnels": [], + "controllerServices": [], + "variables": {}, + "defaultFlowFileExpiration": "0 sec", + "defaultBackPressureObjectThreshold": 10000, + "defaultBackPressureDataSizeThreshold": "1 GB", + "componentType": "PROCESS_GROUP", + "flowFileConcurrency": "UNBOUNDED", + "flowFileOutboundPolicy": "STREAM_WHEN_AVAILABLE" + } +} \ No newline at end of file diff --git a/minifi/minifi-integration-tests/src/test/resources/c2/protocol/c2-authoritative/files/raspi3/config.text.yml.v1 b/minifi/minifi-integration-tests/src/test/resources/c2/protocol/c2-authoritative/files/raspi3/config.text.yml.v1 deleted file mode 100644 index 1d5bdf9c4f..0000000000 --- a/minifi/minifi-integration-tests/src/test/resources/c2/protocol/c2-authoritative/files/raspi3/config.text.yml.v1 +++ /dev/null @@ -1,129 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the \"License\"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an \"AS IS\" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -MiNiFi Config Version: 3 -Flow Controller: - name: Edge raspi3 v1.0 - comment: '' -Core Properties: - flow controller graceful shutdown period: 10 sec - flow service write delay interval: 500 ms - administrative yield duration: 30 sec - bored yield duration: 10 millis - max concurrent threads: 1 - variable registry properties: '' -FlowFile Repository: - implementation: org.apache.nifi.controller.repository.WriteAheadFlowFileRepository - partitions: 256 - checkpoint interval: 2 mins - always sync: false - Swap: - threshold: 20000 - in period: 5 sec - in threads: 1 - out period: 5 sec - out threads: 4 -Content Repository: - implementation: org.apache.nifi.controller.repository.FileSystemRepository - content claim max appendable size: 10 MB - content claim max flow files: 100 - content repository archive enabled: false - content repository archive max retention period: 12 hours - content repository archive max usage percentage: 50% - always sync: false -Provenance Repository: - provenance rollover time: 1 min - implementation: org.apache.nifi.provenance.WriteAheadProvenanceRepository - provenance index shard size: 500 MB - provenance max storage size: 1 GB - provenance max storage time: 24 hours - provenance buffer size: 10000 -Component Status Repository: - buffer size: 1440 - snapshot frequency: 1 min -Security Properties: - keystore: '' - keystore type: '' - keystore password: '' - key password: '' - truststore: '' - truststore type: '' - truststore password: '' - ssl protocol: '' - Sensitive Props: - key: '' - algorithm: NIFI_PBKDF2_AES_GCM_256 -Processors: -- id: c6a06b10-0af2-423b-9927-0bb8e6e04bc3 - name: GenerateTestText - class: org.apache.nifi.processors.standard.GenerateFlowFile - max concurrent tasks: 1 - scheduling strategy: TIMER_DRIVEN - scheduling period: 1000 ms - penalization period: 30000 ms - yield period: 1000 ms - run duration nanos: 0 - auto-terminated relationships list: [ - ] - Properties: - Batch Size: '1' - Data Format: Text - File Size: 0B - Unique FlowFiles: 'false' - character-set: UTF-8 - generate-ff-custom-text: __testTextRaspi3__ -- id: 186322b1-4778-40e4-ba9d-a0a511d762f7 - name: LogAttribute - class: org.apache.nifi.processors.standard.LogAttribute - max concurrent tasks: 1 - scheduling strategy: TIMER_DRIVEN - scheduling period: 1000 ms - penalization period: 30000 ms - yield period: 1000 ms - run duration nanos: 0 - auto-terminated relationships list: - - success - Properties: - Log FlowFile Properties: 'true' - Log Level: info - Log Payload: 'true' - Output Format: Line per Attribute - attributes-to-log-regex: .* - character-set: UTF-8 -Controller Services: [ - ] -Process Groups: [ - ] -Input Ports: [ - ] -Output Ports: [ - ] -Funnels: [ - ] -Connections: -- id: 641b0183-8653-4988-9a74-bcf9780b1397 - name: GenerateTestText/success/LogAttribute - source id: c6a06b10-0af2-423b-9927-0bb8e6e04bc3 - source relationship names: - - success - destination id: 186322b1-4778-40e4-ba9d-a0a511d762f7 - max work queue size: 10000 - max work queue data size: 10 MB - flowfile expiration: 0 seconds - queue prioritizer class: '' -Remote Process Groups: [ - ] -NiFi Properties Overrides: { - } diff --git a/minifi/minifi-integration-tests/src/test/resources/c2/protocol/c2-authoritative/files/raspi4/config.application.json.v1 b/minifi/minifi-integration-tests/src/test/resources/c2/protocol/c2-authoritative/files/raspi4/config.application.json.v1 new file mode 100644 index 0000000000..c71110e28b --- /dev/null +++ b/minifi/minifi-integration-tests/src/test/resources/c2/protocol/c2-authoritative/files/raspi4/config.application.json.v1 @@ -0,0 +1,163 @@ +{ + "encodingVersion": { + "majorVersion": 2, + "minorVersion": 0 + }, + "maxTimerDrivenThreadCount": 1, + "maxEventDrivenThreadCount": 1, + "registries": [], + "parameterContexts": [], + "parameterProviders": [], + "controllerServices": [], + "reportingTasks": [], + "templates": [], + "rootGroup": { + "identifier": "f4cd72f7-b235-394b-928a-ffb0e7cf2db6", + "instanceIdentifier": "Root-Group", + "name": "Edge raspi4 v1.0", + "comments": "", + "position": { + "x": 0.0, + "y": 0.0 + }, + "processGroups": [], + "remoteProcessGroups": [], + "processors": [ + { + "identifier": "02d5a4ee-c6a0-3525-bf3e-f786c21261ef", + "instanceIdentifier": "6ef15904-e69e-425b-b4a9-427c367220a3", + "name": "GenerateFlowFile", + "comments": "", + "position": { + "x": 0.0, + "y": 0.0 + }, + "type": "org.apache.nifi.processors.standard.GenerateFlowFile", + "bundle": { + "group": "org.apache.nifi.minifi", + "artifact": "minifi-standard-nar", + "version": "2.0.0-SNAPSHOT" + }, + "properties": { + "character-set": "UTF-8", + "File Size": "0B", + "generate-ff-custom-text": "__testTextRaspi4__", + "Batch Size": "1", + "Unique FlowFiles": "false", + "Data Format": "Text" + }, + "propertyDescriptors": {}, + "style": {}, + "schedulingPeriod": "1000 ms", + "schedulingStrategy": "TIMER_DRIVEN", + "executionNode": "ALL", + "penaltyDuration": "30000 ms", + "yieldDuration": "1000 ms", + "bulletinLevel": "WARN", + "runDurationMillis": 0, + "concurrentlySchedulableTaskCount": 1, + "autoTerminatedRelationships": [], + "scheduledState": "RUNNING", + "retryCount": 10, + "retriedRelationships": [], + "backoffMechanism": "PENALIZE_FLOWFILE", + "maxBackoffPeriod": "10 mins", + "componentType": "PROCESSOR", + "groupIdentifier": "f4cd72f7-b235-394b-928a-ffb0e7cf2db6" + }, + { + "identifier": "0c683d93-d332-342a-ad3d-cfcbfb6f06d2", + "instanceIdentifier": "26f9038d-2cd9-4df3-a174-c48dda90fce7", + "name": "LogAttribute", + "comments": "", + "position": { + "x": 0.0, + "y": 0.0 + }, + "type": "org.apache.nifi.processors.standard.LogAttribute", + "bundle": { + "group": "org.apache.nifi.minifi", + "artifact": "minifi-standard-nar", + "version": "2.0.0-SNAPSHOT" + }, + "properties": { + "character-set": "UTF-8", + "Log FlowFile Properties": "true", + "Log Level": "info", + "attributes-to-log-regex": ".*", + "Output Format": "Line per Attribute", + "Log Payload": "true" + }, + "propertyDescriptors": {}, + "style": {}, + "schedulingPeriod": "0 ms", + "schedulingStrategy": "TIMER_DRIVEN", + "executionNode": "ALL", + "penaltyDuration": "30000 ms", + "yieldDuration": "1000 ms", + "bulletinLevel": "WARN", + "runDurationMillis": 0, + "concurrentlySchedulableTaskCount": 1, + "autoTerminatedRelationships": [ + "success" + ], + "scheduledState": "RUNNING", + "retryCount": 10, + "retriedRelationships": [], + "backoffMechanism": "PENALIZE_FLOWFILE", + "maxBackoffPeriod": "10 mins", + "componentType": "PROCESSOR", + "groupIdentifier": "f4cd72f7-b235-394b-928a-ffb0e7cf2db6" + } + ], + "inputPorts": [], + "outputPorts": [], + "connections": [ + { + "identifier": "437bccab-5d00-395b-9d47-a1deac7f56a9", + "instanceIdentifier": "68ebc161-1b82-472b-a2f6-ee4173033f60", + "name": "GenerateFlowFile/success/LogAttribute", + "source": { + "id": "02d5a4ee-c6a0-3525-bf3e-f786c21261ef", + "type": "PROCESSOR", + "groupId": "f4cd72f7-b235-394b-928a-ffb0e7cf2db6", + "name": "GenerateFlowFile", + "comments": "", + "instanceIdentifier": "6ef15904-e69e-425b-b4a9-427c367220a3" + }, + "destination": { + "id": "0c683d93-d332-342a-ad3d-cfcbfb6f06d2", + "type": "PROCESSOR", + "groupId": "f4cd72f7-b235-394b-928a-ffb0e7cf2db6", + "name": "LogAttribute", + "comments": "", + "instanceIdentifier": "26f9038d-2cd9-4df3-a174-c48dda90fce7" + }, + "labelIndex": 1, + "zIndex": 0, + "selectedRelationships": [ + "success" + ], + "backPressureObjectThreshold": 10000, + "backPressureDataSizeThreshold": "10 MB", + "flowFileExpiration": "0 seconds", + "prioritizers": [], + "bends": [], + "loadBalanceStrategy": "DO_NOT_LOAD_BALANCE", + "loadBalanceCompression": "DO_NOT_COMPRESS", + "componentType": "CONNECTION", + "groupIdentifier": "f4cd72f7-b235-394b-928a-ffb0e7cf2db6" + } + ], + "labels": [], + "funnels": [], + "controllerServices": [], + "variables": {}, + "defaultFlowFileExpiration": "0 sec", + "defaultBackPressureObjectThreshold": 10000, + "defaultBackPressureDataSizeThreshold": "1 GB", + "componentType": "PROCESS_GROUP", + "flowFileConcurrency": "UNBOUNDED", + "flowFileOutboundPolicy": "STREAM_WHEN_AVAILABLE" + } +} \ No newline at end of file diff --git a/minifi/minifi-integration-tests/src/test/resources/c2/protocol/c2-authoritative/files/raspi4/config.text.yml.v1 b/minifi/minifi-integration-tests/src/test/resources/c2/protocol/c2-authoritative/files/raspi4/config.text.yml.v1 deleted file mode 100644 index fc528ae7a2..0000000000 --- a/minifi/minifi-integration-tests/src/test/resources/c2/protocol/c2-authoritative/files/raspi4/config.text.yml.v1 +++ /dev/null @@ -1,129 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the \"License\"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an \"AS IS\" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -MiNiFi Config Version: 3 -Flow Controller: - name: Edge raspi4 v1.0 - comment: '' -Core Properties: - flow controller graceful shutdown period: 10 sec - flow service write delay interval: 500 ms - administrative yield duration: 30 sec - bored yield duration: 10 millis - max concurrent threads: 1 - variable registry properties: '' -FlowFile Repository: - implementation: org.apache.nifi.controller.repository.WriteAheadFlowFileRepository - partitions: 256 - checkpoint interval: 2 mins - always sync: false - Swap: - threshold: 20000 - in period: 5 sec - in threads: 1 - out period: 5 sec - out threads: 4 -Content Repository: - implementation: org.apache.nifi.controller.repository.FileSystemRepository - content claim max appendable size: 10 MB - content claim max flow files: 100 - content repository archive enabled: false - content repository archive max retention period: 12 hours - content repository archive max usage percentage: 50% - always sync: false -Provenance Repository: - provenance rollover time: 1 min - implementation: org.apache.nifi.provenance.WriteAheadProvenanceRepository - provenance index shard size: 500 MB - provenance max storage size: 1 GB - provenance max storage time: 24 hours - provenance buffer size: 10000 -Component Status Repository: - buffer size: 1440 - snapshot frequency: 1 min -Security Properties: - keystore: '' - keystore type: '' - keystore password: '' - key password: '' - truststore: '' - truststore type: '' - truststore password: '' - ssl protocol: '' - Sensitive Props: - key: '' - algorithm: NIFI_PBKDF2_AES_GCM_256 -Processors: -- id: 6ef15904-e69e-425b-b4a9-427c367220a3 - name: GenerateFlowFile - class: org.apache.nifi.processors.standard.GenerateFlowFile - max concurrent tasks: 1 - scheduling strategy: TIMER_DRIVEN - scheduling period: 1000 ms - penalization period: 30000 ms - yield period: 1000 ms - run duration nanos: 0 - auto-terminated relationships list: [ - ] - Properties: - Batch Size: '1' - Data Format: Text - File Size: 0B - Unique FlowFiles: 'false' - character-set: UTF-8 - generate-ff-custom-text: __testTextRaspi4__ -- id: 26f9038d-2cd9-4df3-a174-c48dda90fce7 - name: LogAttribute - class: org.apache.nifi.processors.standard.LogAttribute - max concurrent tasks: 1 - scheduling strategy: TIMER_DRIVEN - scheduling period: 0 ms - penalization period: 30000 ms - yield period: 1000 ms - run duration nanos: 0 - auto-terminated relationships list: - - success - Properties: - Log FlowFile Properties: 'true' - Log Level: info - Log Payload: 'true' - Output Format: Line per Attribute - attributes-to-log-regex: .* - character-set: UTF-8 -Controller Services: [ - ] -Process Groups: [ - ] -Input Ports: [ - ] -Output Ports: [ - ] -Funnels: [ - ] -Connections: -- id: 68ebc161-1b82-472b-a2f6-ee4173033f60 - name: GenerateFlowFile/success/LogAttribute - source id: 6ef15904-e69e-425b-b4a9-427c367220a3 - source relationship names: - - success - destination id: 26f9038d-2cd9-4df3-a174-c48dda90fce7 - max work queue size: 10000 - max work queue data size: 10 MB - flowfile expiration: 0 seconds - queue prioritizer class: '' -Remote Process Groups: [ - ] -NiFi Properties Overrides: { - } diff --git a/minifi/minifi-integration-tests/src/test/resources/c2/protocol/minifi-edge1/bootstrap.conf b/minifi/minifi-integration-tests/src/test/resources/c2/protocol/minifi-edge1/bootstrap.conf index 0c5240cb28..75a688a717 100644 --- a/minifi/minifi-integration-tests/src/test/resources/c2/protocol/minifi-edge1/bootstrap.conf +++ b/minifi/minifi-integration-tests/src/test/resources/c2/protocol/minifi-edge1/bootstrap.conf @@ -15,13 +15,75 @@ # limitations under the License. # -# C2 Properties +# Java command to use when running MiNiFi +java=java + +# Username to use when running MiNiFi. This value will be ignored on Windows. +run.as= + +# Configure where MiNiFi's lib and conf directories live +# When running as a Windows service set full paths instead of relative paths +lib.dir=./lib +conf.dir=./conf + +# How long to wait after telling MiNiFi to shutdown before explicitly killing the Process +graceful.shutdown.seconds=20 + +# Disable JSR 199 so that we can use JSP's without running a JDK +java.arg.1=-Dorg.apache.jasper.compiler.disablejsr199=true + +# JVM memory settings +java.arg.2=-Xms256m +java.arg.3=-Xmx256m + +# allowRestrictedHeaders is required for Cluster/Node communications to work properly +java.arg.5=-Dsun.net.http.allowRestrictedHeaders=true +java.arg.6=-Djava.protocol.handler.pkgs=sun.net.www.protocol + +# Sets the provider of SecureRandom to /dev/urandom to prevent blocking on VMs +java.arg.7=-Djava.security.egd=file:/dev/urandom + +# Garbage Collector Algorithm +java.arg.13=-XX:+UseG1GC + +#Set headless mode by default +java.arg.14=-Djava.awt.headless=true + +# Enable Remote Debugging +#java.arg.debug=-agentlib:jdwp=transport=dt_socket,server=y,suspend=n,address=8000 + + +### MiNiFi Flow Config File +nifi.minifi.flow.config=./conf/flow.json.gz + + +### MiNiFi Security Properties +nifi.minifi.security.keystore= +nifi.minifi.security.keystoreType= +nifi.minifi.security.keystorePasswd= +nifi.minifi.security.keyPasswd= +nifi.minifi.security.truststore= +nifi.minifi.security.truststoreType= +nifi.minifi.security.truststorePasswd= +nifi.minifi.security.ssl.protocol= + +# Ignore all processor SSL controller services and use parent minifi SSL instead +nifi.minifi.flow.use.parent.ssl=false + + +### MiNiFi Sensitive Properties Settings +nifi.minifi.sensitive.props.key= +nifi.minifi.sensitive.props.algorithm= + + +### MiNiFi Command & Control (C2) Configuration c2.enable=true c2.rest.url=https://c2-authoritative:10443/c2/config/heartbeat c2.rest.url.ack=https://c2-authoritative:10443/c2/config/acknowledge c2.rest.connectionTimeout=5 sec c2.rest.readTimeout=5 sec c2.rest.callTimeout=10 sec +c2.rest.http.headers=Accept:application/json c2.agent.heartbeat.period=2000 c2.agent.class=raspi3 c2.config.directory=./conf @@ -34,64 +96,40 @@ c2.security.keystore.location=./conf/keystore.jks c2.security.keystore.password=badKeystorePass c2.security.keystore.type=JKS -# Java command to use when running MiNiFi -java=java - -# Username to use when running MiNiFi. This value will be ignored on Windows. -run.as= - -# Configure where MiNiFi's lib and conf directories live -lib.dir=./lib -conf.dir=./conf - -# How long to wait after telling MiNiFi to shutdown before explicitly killing the Process -graceful.shutdown.seconds=20 - -# The location for the configuration file -nifi.minifi.config=./conf/config.yml - -# Notifiers to use for the associated agent, comma separated list of class names -nifi.minifi.notifier.ingestors=org.apache.nifi.minifi.bootstrap.configuration.ingestors.FileChangeIngestor -#nifi.minifi.notifier.ingestors=org.apache.nifi.minifi.bootstrap.configuration.ingestors.RestChangeIngestor -#nifi.minifi.notifier.ingestors=org.apache.nifi.minifi.bootstrap.configuration.ingestors.PullHttpChangeIngestor - -# File change notifier configuration +### MiNiFi Notifiers For Flow Updates +# If C2 is enabled these should be disabled +# Notifiers to use for the associated agent, comma separated list of class names, choose from the options below +#nifi.minifi.notifier.ingestors= +# File change notifier configuration - org.apache.nifi.minifi.bootstrap.configuration.ingestors.FileChangeIngestor # Path of the file to monitor for changes. When these occur, the FileChangeNotifier, if configured, will begin the configuration reloading process -nifi.minifi.notifier.ingestors.file.config.path=./conf/config-new.yml +#nifi.minifi.notifier.ingestors.file.config.path= # How frequently the file specified by 'nifi.minifi.notifier.file.config.path' should be evaluated for changes. -nifi.minifi.notifier.ingestors.file.polling.period.seconds=2 - -# Rest change notifier configuration +#nifi.minifi.notifier.ingestors.file.polling.period.seconds=5 +# Rest change notifier configuration - org.apache.nifi.minifi.bootstrap.configuration.ingestors.RestChangeIngestor # Port on which the Jetty server will bind to, keep commented for a random open port #nifi.minifi.notifier.ingestors.receive.http.port=8338 -#Pull HTTP change notifier configuration - +#Pull HTTP change notifier configuration - org.apache.nifi.minifi.bootstrap.configuration.ingestors.PullHttpChangeIngestor # Hostname on which to pull configurations from -# nifi.minifi.notifier.ingestors.pull.http.hostname=c2-authoritative +#nifi.minifi.notifier.ingestors.pull.http.hostname=localhost # Port on which to pull configurations from -# nifi.minifi.notifier.ingestors.pull.http.port=10443 +#nifi.minifi.notifier.ingestors.pull.http.port=4567 # Path to pull configurations from -# nifi.minifi.notifier.ingestors.pull.http.path=/c2/config +#nifi.minifi.notifier.ingestors.pull.http.path=/c2/config # Query string to pull configurations with -# nifi.minifi.notifier.ingestors.pull.http.query=net=edge1&class=raspi3 +#nifi.minifi.notifier.ingestors.pull.http.query=class=raspi3 # Period on which to pull configurations from, defaults to 5 minutes if commented out -# nifi.minifi.notifier.ingestors.pull.http.period.ms=3000 +#nifi.minifi.notifier.ingestors.pull.http.period.ms=300000 +# Comma separated list of HTTP headers, eg: Accept:application/json,Accept:application/yaml +#nifi.minifi.notifier.ingestors.pull.http.headers= -# nifi.minifi.notifier.ingestors.pull.http.keystore.location=./conf/keystore.jks -# nifi.minifi.notifier.ingestors.pull.http.keystore.type=JKS -# nifi.minifi.notifier.ingestors.pull.http.keystore.password=badKeystorePass -# nifi.minifi.notifier.ingestors.pull.http.truststore.location=./conf/truststore.jks -# nifi.minifi.notifier.ingestors.pull.http.truststore.type=JKS -# nifi.minifi.notifier.ingestors.pull.http.truststore.password=badTrustPass +### MiNiFi Status Reporter # Periodic Status Reporters to use for the associated agent, comma separated list of class names #nifi.minifi.status.reporter.components=org.apache.nifi.minifi.bootstrap.status.reporters.StatusLogger - # Periodic Status Logger configuration - # The FlowStatus query to submit to the MiNiFi instance #nifi.minifi.status.reporter.log.query=instance:health,bulletins # The log level at which the status will be logged @@ -99,25 +137,45 @@ nifi.minifi.notifier.ingestors.file.polling.period.seconds=2 # The period (in milliseconds) at which to log the status #nifi.minifi.status.reporter.log.period=60000 -# Disable JSR 199 so that we can use JSP's without running a JDK -java.arg.1=-Dorg.apache.jasper.compiler.disablejsr199=true -# JVM memory settings -java.arg.2=-Xms256m -java.arg.3=-Xmx256m +### MiNiFi Provenance Reporting Properties +#nifi.minifi.provenance.reporting.comment= +#nifi.minifi.provenance.reporting.scheduling.strategy= +#nifi.minifi.provenance.reporting.scheduling.period= +#nifi.minifi.provenance.reporting.destination.url= +#nifi.minifi.provenance.reporting.input.port.name= +#nifi.minifi.provenance.reporting.instance.url= +#nifi.minifi.provenance.reporting.batch.size= +#nifi.minifi.provenance.reporting.communications.timeout= +#nifi.minifi.provenance.reporting.compress.events= -# Enable Remote Debugging -# java.arg.debug=-agentlib:jdwp=transport=dt_socket,server=y,suspend=y,address=8000 -# allowRestrictedHeaders is required for Cluster/Node communications to work properly -java.arg.5=-Dsun.net.http.allowRestrictedHeaders=true -java.arg.6=-Djava.protocol.handler.pkgs=sun.net.www.protocol - -# The G1GC is still considered experimental but has proven to be very advantageous in providing great -# performance without significant "stop-the-world" delays. -#java.arg.13=-XX:+UseG1GC - -#Set headless mode by default -java.arg.14=-Djava.awt.headless=true - -java.arg.15=-Djava.security.egd=file:/dev/./urandom \ No newline at end of file +### NiFi Framework Properties +# Core Properties +nifi.flowcontroller.graceful.shutdown.period=10 sec +nifi.flowservice.writedelay.interval=500 ms +nifi.administrative.yield.duration=30 sec +nifi.variable.registry.properties= +nifi.bored.yield.duration=10 millis +# FlowFile Repository +nifi.flowfile.repository.implementation=org.apache.nifi.controller.repository.WriteAheadFlowFileRepository +nifi.flowfile.repository.checkpoint.interval=20 secs +nifi.flowfile.repository.always.sync=false +# Content Repository +nifi.content.repository.implementation=org.apache.nifi.controller.repository.FileSystemRepository +nifi.content.claim.max.appendable.size=50 KB +nifi.content.repository.archive.max.retention.period=7 days +nifi.content.repository.archive.max.usage.percentage=50% +nifi.content.repository.archive.enabled=false +# Provenance Repository +nifi.provenance.repository.implementation=org.apache.nifi.provenance.NoOpProvenanceRepository +nifi.provenance.repository.rollover.time= +nifi.provenance.repository.index.shard.size= +nifi.provenance.repository.max.storage.size= +nifi.provenance.repository.max.storage.time= +# Components Status Repository +nifi.components.status.repository.implementation=org.apache.nifi.controller.status.history.VolatileComponentStatusRepository +nifi.components.status.snapshot.frequency=1 min +# Swap Manager +nifi.swap.manager.implementation=org.apache.nifi.controller.FileSystemSwapManager +nifi.queue.swap.threshold=20000 diff --git a/minifi/minifi-integration-tests/src/test/resources/c2/protocol/minifi-edge2/bootstrap.conf b/minifi/minifi-integration-tests/src/test/resources/c2/protocol/minifi-edge2/bootstrap.conf index 0c5240cb28..ea648c8783 100644 --- a/minifi/minifi-integration-tests/src/test/resources/c2/protocol/minifi-edge2/bootstrap.conf +++ b/minifi/minifi-integration-tests/src/test/resources/c2/protocol/minifi-edge2/bootstrap.conf @@ -15,13 +15,75 @@ # limitations under the License. # -# C2 Properties +# Java command to use when running MiNiFi +java=java + +# Username to use when running MiNiFi. This value will be ignored on Windows. +run.as= + +# Configure where MiNiFi's lib and conf directories live +# When running as a Windows service set full paths instead of relative paths +lib.dir=./lib +conf.dir=./conf + +# How long to wait after telling MiNiFi to shutdown before explicitly killing the Process +graceful.shutdown.seconds=20 + +# Disable JSR 199 so that we can use JSP's without running a JDK +java.arg.1=-Dorg.apache.jasper.compiler.disablejsr199=true + +# JVM memory settings +java.arg.2=-Xms256m +java.arg.3=-Xmx256m + +# allowRestrictedHeaders is required for Cluster/Node communications to work properly +java.arg.5=-Dsun.net.http.allowRestrictedHeaders=true +java.arg.6=-Djava.protocol.handler.pkgs=sun.net.www.protocol + +# Sets the provider of SecureRandom to /dev/urandom to prevent blocking on VMs +java.arg.7=-Djava.security.egd=file:/dev/urandom + +# Garbage Collector Algorithm +java.arg.13=-XX:+UseG1GC + +#Set headless mode by default +java.arg.14=-Djava.awt.headless=true + +# Enable Remote Debugging +#java.arg.debug=-agentlib:jdwp=transport=dt_socket,server=y,suspend=n,address=8000 + + +### MiNiFi Flow Config File +nifi.minifi.flow.config=./conf/flow.json.gz + + +### MiNiFi Security Properties +nifi.minifi.security.keystore= +nifi.minifi.security.keystoreType= +nifi.minifi.security.keystorePasswd= +nifi.minifi.security.keyPasswd= +nifi.minifi.security.truststore= +nifi.minifi.security.truststoreType= +nifi.minifi.security.truststorePasswd= +nifi.minifi.security.ssl.protocol= + +# Ignore all processor SSL controller services and use parent minifi SSL instead +nifi.minifi.flow.use.parent.ssl=false + + +### MiNiFi Sensitive Properties Settings +nifi.minifi.sensitive.props.key= +nifi.minifi.sensitive.props.algorithm= + + +### MiNiFi Command & Control (C2) Configuration c2.enable=true c2.rest.url=https://c2-authoritative:10443/c2/config/heartbeat c2.rest.url.ack=https://c2-authoritative:10443/c2/config/acknowledge c2.rest.connectionTimeout=5 sec c2.rest.readTimeout=5 sec c2.rest.callTimeout=10 sec +c2.rest.http.headers=Accept:application/json c2.agent.heartbeat.period=2000 c2.agent.class=raspi3 c2.config.directory=./conf @@ -34,64 +96,41 @@ c2.security.keystore.location=./conf/keystore.jks c2.security.keystore.password=badKeystorePass c2.security.keystore.type=JKS -# Java command to use when running MiNiFi -java=java -# Username to use when running MiNiFi. This value will be ignored on Windows. -run.as= - -# Configure where MiNiFi's lib and conf directories live -lib.dir=./lib -conf.dir=./conf - -# How long to wait after telling MiNiFi to shutdown before explicitly killing the Process -graceful.shutdown.seconds=20 - -# The location for the configuration file -nifi.minifi.config=./conf/config.yml - -# Notifiers to use for the associated agent, comma separated list of class names -nifi.minifi.notifier.ingestors=org.apache.nifi.minifi.bootstrap.configuration.ingestors.FileChangeIngestor -#nifi.minifi.notifier.ingestors=org.apache.nifi.minifi.bootstrap.configuration.ingestors.RestChangeIngestor -#nifi.minifi.notifier.ingestors=org.apache.nifi.minifi.bootstrap.configuration.ingestors.PullHttpChangeIngestor - -# File change notifier configuration +### MiNiFi Notifiers For Flow Updates +# If C2 is enabled these should be disabled +# Notifiers to use for the associated agent, comma separated list of class names, choose from the options below +#nifi.minifi.notifier.ingestors= +# File change notifier configuration - org.apache.nifi.minifi.bootstrap.configuration.ingestors.FileChangeIngestor # Path of the file to monitor for changes. When these occur, the FileChangeNotifier, if configured, will begin the configuration reloading process -nifi.minifi.notifier.ingestors.file.config.path=./conf/config-new.yml +#nifi.minifi.notifier.ingestors.file.config.path= # How frequently the file specified by 'nifi.minifi.notifier.file.config.path' should be evaluated for changes. -nifi.minifi.notifier.ingestors.file.polling.period.seconds=2 - -# Rest change notifier configuration +#nifi.minifi.notifier.ingestors.file.polling.period.seconds=5 +# Rest change notifier configuration - org.apache.nifi.minifi.bootstrap.configuration.ingestors.RestChangeIngestor # Port on which the Jetty server will bind to, keep commented for a random open port #nifi.minifi.notifier.ingestors.receive.http.port=8338 -#Pull HTTP change notifier configuration - +#Pull HTTP change notifier configuration - org.apache.nifi.minifi.bootstrap.configuration.ingestors.PullHttpChangeIngestor # Hostname on which to pull configurations from -# nifi.minifi.notifier.ingestors.pull.http.hostname=c2-authoritative +#nifi.minifi.notifier.ingestors.pull.http.hostname=localhost # Port on which to pull configurations from -# nifi.minifi.notifier.ingestors.pull.http.port=10443 +#nifi.minifi.notifier.ingestors.pull.http.port=4567 # Path to pull configurations from -# nifi.minifi.notifier.ingestors.pull.http.path=/c2/config +#nifi.minifi.notifier.ingestors.pull.http.path=/c2/config # Query string to pull configurations with -# nifi.minifi.notifier.ingestors.pull.http.query=net=edge1&class=raspi3 +#nifi.minifi.notifier.ingestors.pull.http.query=class=raspi3 # Period on which to pull configurations from, defaults to 5 minutes if commented out -# nifi.minifi.notifier.ingestors.pull.http.period.ms=3000 +#nifi.minifi.notifier.ingestors.pull.http.period.ms=300000 +# Comma separated list of HTTP headers, eg: Accept:application/json,Accept:application/yaml +#nifi.minifi.notifier.ingestors.pull.http.headers= -# nifi.minifi.notifier.ingestors.pull.http.keystore.location=./conf/keystore.jks -# nifi.minifi.notifier.ingestors.pull.http.keystore.type=JKS -# nifi.minifi.notifier.ingestors.pull.http.keystore.password=badKeystorePass -# nifi.minifi.notifier.ingestors.pull.http.truststore.location=./conf/truststore.jks -# nifi.minifi.notifier.ingestors.pull.http.truststore.type=JKS -# nifi.minifi.notifier.ingestors.pull.http.truststore.password=badTrustPass +### MiNiFi Status Reporter # Periodic Status Reporters to use for the associated agent, comma separated list of class names #nifi.minifi.status.reporter.components=org.apache.nifi.minifi.bootstrap.status.reporters.StatusLogger - # Periodic Status Logger configuration - # The FlowStatus query to submit to the MiNiFi instance #nifi.minifi.status.reporter.log.query=instance:health,bulletins # The log level at which the status will be logged @@ -99,25 +138,45 @@ nifi.minifi.notifier.ingestors.file.polling.period.seconds=2 # The period (in milliseconds) at which to log the status #nifi.minifi.status.reporter.log.period=60000 -# Disable JSR 199 so that we can use JSP's without running a JDK -java.arg.1=-Dorg.apache.jasper.compiler.disablejsr199=true -# JVM memory settings -java.arg.2=-Xms256m -java.arg.3=-Xmx256m +### MiNiFi Provenance Reporting Properties +#nifi.minifi.provenance.reporting.comment= +#nifi.minifi.provenance.reporting.scheduling.strategy= +#nifi.minifi.provenance.reporting.scheduling.period= +#nifi.minifi.provenance.reporting.destination.url= +#nifi.minifi.provenance.reporting.input.port.name= +#nifi.minifi.provenance.reporting.instance.url= +#nifi.minifi.provenance.reporting.batch.size= +#nifi.minifi.provenance.reporting.communications.timeout= +#nifi.minifi.provenance.reporting.compress.events= -# Enable Remote Debugging -# java.arg.debug=-agentlib:jdwp=transport=dt_socket,server=y,suspend=y,address=8000 -# allowRestrictedHeaders is required for Cluster/Node communications to work properly -java.arg.5=-Dsun.net.http.allowRestrictedHeaders=true -java.arg.6=-Djava.protocol.handler.pkgs=sun.net.www.protocol - -# The G1GC is still considered experimental but has proven to be very advantageous in providing great -# performance without significant "stop-the-world" delays. -#java.arg.13=-XX:+UseG1GC - -#Set headless mode by default -java.arg.14=-Djava.awt.headless=true - -java.arg.15=-Djava.security.egd=file:/dev/./urandom \ No newline at end of file +### NiFi Framework Properties +# Core Properties +nifi.flowcontroller.graceful.shutdown.period=10 sec +nifi.flowservice.writedelay.interval=500 ms +nifi.administrative.yield.duration=30 sec +nifi.variable.registry.properties= +nifi.bored.yield.duration=10 millis +# FlowFile Repository +nifi.flowfile.repository.implementation=org.apache.nifi.controller.repository.WriteAheadFlowFileRepository +nifi.flowfile.repository.checkpoint.interval=20 secs +nifi.flowfile.repository.always.sync=false +# Content Repository +nifi.content.repository.implementation=org.apache.nifi.controller.repository.FileSystemRepository +nifi.content.claim.max.appendable.size=50 KB +nifi.content.repository.archive.max.retention.period=7 days +nifi.content.repository.archive.max.usage.percentage=50% +nifi.content.repository.archive.enabled=false +# Provenance Repository +nifi.provenance.repository.implementation=org.apache.nifi.provenance.NoOpProvenanceRepository +nifi.provenance.repository.rollover.time= +nifi.provenance.repository.index.shard.size= +nifi.provenance.repository.max.storage.size= +nifi.provenance.repository.max.storage.time= +# Components Status Repository +nifi.components.status.repository.implementation=org.apache.nifi.controller.status.history.VolatileComponentStatusRepository +nifi.components.status.snapshot.frequency=1 min +# Swap Manager +nifi.swap.manager.implementation=org.apache.nifi.controller.FileSystemSwapManager +nifi.queue.swap.threshold=20000 diff --git a/minifi/minifi-integration-tests/src/test/resources/c2/protocol/minifi-edge3/bootstrap.conf b/minifi/minifi-integration-tests/src/test/resources/c2/protocol/minifi-edge3/bootstrap.conf index be5568ee78..71678f9fc1 100644 --- a/minifi/minifi-integration-tests/src/test/resources/c2/protocol/minifi-edge3/bootstrap.conf +++ b/minifi/minifi-integration-tests/src/test/resources/c2/protocol/minifi-edge3/bootstrap.conf @@ -15,13 +15,75 @@ # limitations under the License. # -# C2 Properties +# Java command to use when running MiNiFi +java=java + +# Username to use when running MiNiFi. This value will be ignored on Windows. +run.as= + +# Configure where MiNiFi's lib and conf directories live +# When running as a Windows service set full paths instead of relative paths +lib.dir=./lib +conf.dir=./conf + +# How long to wait after telling MiNiFi to shutdown before explicitly killing the Process +graceful.shutdown.seconds=20 + +# Disable JSR 199 so that we can use JSP's without running a JDK +java.arg.1=-Dorg.apache.jasper.compiler.disablejsr199=true + +# JVM memory settings +java.arg.2=-Xms256m +java.arg.3=-Xmx256m + +# allowRestrictedHeaders is required for Cluster/Node communications to work properly +java.arg.5=-Dsun.net.http.allowRestrictedHeaders=true +java.arg.6=-Djava.protocol.handler.pkgs=sun.net.www.protocol + +# Sets the provider of SecureRandom to /dev/urandom to prevent blocking on VMs +java.arg.7=-Djava.security.egd=file:/dev/urandom + +# Garbage Collector Algorithm +java.arg.13=-XX:+UseG1GC + +#Set headless mode by default +java.arg.14=-Djava.awt.headless=true + +# Enable Remote Debugging +#java.arg.debug=-agentlib:jdwp=transport=dt_socket,server=y,suspend=n,address=8000 + + +### MiNiFi Flow Config File +nifi.minifi.flow.config=./conf/flow.json.gz + + +### MiNiFi Security Properties +nifi.minifi.security.keystore= +nifi.minifi.security.keystoreType= +nifi.minifi.security.keystorePasswd= +nifi.minifi.security.keyPasswd= +nifi.minifi.security.truststore= +nifi.minifi.security.truststoreType= +nifi.minifi.security.truststorePasswd= +nifi.minifi.security.ssl.protocol= + +# Ignore all processor SSL controller services and use parent minifi SSL instead +nifi.minifi.flow.use.parent.ssl=false + + +### MiNiFi Sensitive Properties Settings +nifi.minifi.sensitive.props.key= +nifi.minifi.sensitive.props.algorithm= + + +### MiNiFi Command & Control (C2) Configuration c2.enable=true c2.rest.url=https://c2-authoritative:10443/c2/config/heartbeat c2.rest.url.ack=https://c2-authoritative:10443/c2/config/acknowledge c2.rest.connectionTimeout=5 sec c2.rest.readTimeout=5 sec c2.rest.callTimeout=10 sec +c2.rest.http.headers=Accept:application/json c2.agent.heartbeat.period=2000 c2.agent.class=raspi4 c2.config.directory=./conf @@ -34,64 +96,41 @@ c2.security.keystore.location=./conf/keystore.jks c2.security.keystore.password=badKeystorePass c2.security.keystore.type=JKS -# Java command to use when running MiNiFi -java=java -# Username to use when running MiNiFi. This value will be ignored on Windows. -run.as= - -# Configure where MiNiFi's lib and conf directories live -lib.dir=./lib -conf.dir=./conf - -# How long to wait after telling MiNiFi to shutdown before explicitly killing the Process -graceful.shutdown.seconds=20 - -# The location for the configuration file -nifi.minifi.config=./conf/config.yml - -# Notifiers to use for the associated agent, comma separated list of class names -nifi.minifi.notifier.ingestors=org.apache.nifi.minifi.bootstrap.configuration.ingestors.FileChangeIngestor -#nifi.minifi.notifier.ingestors=org.apache.nifi.minifi.bootstrap.configuration.ingestors.RestChangeIngestor -#nifi.minifi.notifier.ingestors=org.apache.nifi.minifi.bootstrap.configuration.ingestors.PullHttpChangeIngestor - -# File change notifier configuration +### MiNiFi Notifiers For Flow Updates +# If C2 is enabled these should be disabled +# Notifiers to use for the associated agent, comma separated list of class names, choose from the options below +#nifi.minifi.notifier.ingestors= +# File change notifier configuration - org.apache.nifi.minifi.bootstrap.configuration.ingestors.FileChangeIngestor # Path of the file to monitor for changes. When these occur, the FileChangeNotifier, if configured, will begin the configuration reloading process -nifi.minifi.notifier.ingestors.file.config.path=./conf/config-new.yml +#nifi.minifi.notifier.ingestors.file.config.path= # How frequently the file specified by 'nifi.minifi.notifier.file.config.path' should be evaluated for changes. -nifi.minifi.notifier.ingestors.file.polling.period.seconds=2 - -# Rest change notifier configuration +#nifi.minifi.notifier.ingestors.file.polling.period.seconds=5 +# Rest change notifier configuration - org.apache.nifi.minifi.bootstrap.configuration.ingestors.RestChangeIngestor # Port on which the Jetty server will bind to, keep commented for a random open port #nifi.minifi.notifier.ingestors.receive.http.port=8338 -#Pull HTTP change notifier configuration - +#Pull HTTP change notifier configuration - org.apache.nifi.minifi.bootstrap.configuration.ingestors.PullHttpChangeIngestor # Hostname on which to pull configurations from -# nifi.minifi.notifier.ingestors.pull.http.hostname=c2-authoritative +#nifi.minifi.notifier.ingestors.pull.http.hostname=localhost # Port on which to pull configurations from -# nifi.minifi.notifier.ingestors.pull.http.port=10443 +#nifi.minifi.notifier.ingestors.pull.http.port=4567 # Path to pull configurations from -# nifi.minifi.notifier.ingestors.pull.http.path=/c2/config +#nifi.minifi.notifier.ingestors.pull.http.path=/c2/config # Query string to pull configurations with -# nifi.minifi.notifier.ingestors.pull.http.query=net=edge1&class=raspi4 +#nifi.minifi.notifier.ingestors.pull.http.query=class=raspi3 # Period on which to pull configurations from, defaults to 5 minutes if commented out -# nifi.minifi.notifier.ingestors.pull.http.period.ms=3000 +#nifi.minifi.notifier.ingestors.pull.http.period.ms=300000 +# Comma separated list of HTTP headers, eg: Accept:application/json,Accept:application/yaml +#nifi.minifi.notifier.ingestors.pull.http.headers= -# nifi.minifi.notifier.ingestors.pull.http.keystore.location=./conf/keystore.jks -# nifi.minifi.notifier.ingestors.pull.http.keystore.type=JKS -# nifi.minifi.notifier.ingestors.pull.http.keystore.password=badKeystorePass -# nifi.minifi.notifier.ingestors.pull.http.truststore.location=./conf/truststore.jks -# nifi.minifi.notifier.ingestors.pull.http.truststore.type=JKS -# nifi.minifi.notifier.ingestors.pull.http.truststore.password=badTrustPass +### MiNiFi Status Reporter # Periodic Status Reporters to use for the associated agent, comma separated list of class names #nifi.minifi.status.reporter.components=org.apache.nifi.minifi.bootstrap.status.reporters.StatusLogger - # Periodic Status Logger configuration - # The FlowStatus query to submit to the MiNiFi instance #nifi.minifi.status.reporter.log.query=instance:health,bulletins # The log level at which the status will be logged @@ -99,25 +138,45 @@ nifi.minifi.notifier.ingestors.file.polling.period.seconds=2 # The period (in milliseconds) at which to log the status #nifi.minifi.status.reporter.log.period=60000 -# Disable JSR 199 so that we can use JSP's without running a JDK -java.arg.1=-Dorg.apache.jasper.compiler.disablejsr199=true -# JVM memory settings -java.arg.2=-Xms256m -java.arg.3=-Xmx256m +### MiNiFi Provenance Reporting Properties +#nifi.minifi.provenance.reporting.comment= +#nifi.minifi.provenance.reporting.scheduling.strategy= +#nifi.minifi.provenance.reporting.scheduling.period= +#nifi.minifi.provenance.reporting.destination.url= +#nifi.minifi.provenance.reporting.input.port.name= +#nifi.minifi.provenance.reporting.instance.url= +#nifi.minifi.provenance.reporting.batch.size= +#nifi.minifi.provenance.reporting.communications.timeout= +#nifi.minifi.provenance.reporting.compress.events= -# Enable Remote Debugging -# java.arg.debug=-agentlib:jdwp=transport=dt_socket,server=y,suspend=y,address=8000 -# allowRestrictedHeaders is required for Cluster/Node communications to work properly -java.arg.5=-Dsun.net.http.allowRestrictedHeaders=true -java.arg.6=-Djava.protocol.handler.pkgs=sun.net.www.protocol - -# The G1GC is still considered experimental but has proven to be very advantageous in providing great -# performance without significant "stop-the-world" delays. -#java.arg.13=-XX:+UseG1GC - -#Set headless mode by default -java.arg.14=-Djava.awt.headless=true - -java.arg.15=-Djava.security.egd=file:/dev/./urandom \ No newline at end of file +### NiFi Framework Properties +# Core Properties +nifi.flowcontroller.graceful.shutdown.period=10 sec +nifi.flowservice.writedelay.interval=500 ms +nifi.administrative.yield.duration=30 sec +nifi.variable.registry.properties= +nifi.bored.yield.duration=10 millis +# FlowFile Repository +nifi.flowfile.repository.implementation=org.apache.nifi.controller.repository.WriteAheadFlowFileRepository +nifi.flowfile.repository.checkpoint.interval=20 secs +nifi.flowfile.repository.always.sync=false +# Content Repository +nifi.content.repository.implementation=org.apache.nifi.controller.repository.FileSystemRepository +nifi.content.claim.max.appendable.size=50 KB +nifi.content.repository.archive.max.retention.period=7 days +nifi.content.repository.archive.max.usage.percentage=50% +nifi.content.repository.archive.enabled=false +# Provenance Repository +nifi.provenance.repository.implementation=org.apache.nifi.provenance.NoOpProvenanceRepository +nifi.provenance.repository.rollover.time= +nifi.provenance.repository.index.shard.size= +nifi.provenance.repository.max.storage.size= +nifi.provenance.repository.max.storage.time= +# Components Status Repository +nifi.components.status.repository.implementation=org.apache.nifi.controller.status.history.VolatileComponentStatusRepository +nifi.components.status.snapshot.frequency=1 min +# Swap Manager +nifi.swap.manager.implementation=org.apache.nifi.controller.FileSystemSwapManager +nifi.queue.swap.threshold=20000 diff --git a/minifi/minifi-integration-tests/src/test/resources/docker-compose-v1-standalone.yml b/minifi/minifi-integration-tests/src/test/resources/docker-compose-standalone.yml similarity index 98% rename from minifi/minifi-integration-tests/src/test/resources/docker-compose-v1-standalone.yml rename to minifi/minifi-integration-tests/src/test/resources/docker-compose-standalone.yml index 19686aa10a..74ffc8cc89 100644 --- a/minifi/minifi-integration-tests/src/test/resources/docker-compose-v1-standalone.yml +++ b/minifi/minifi-integration-tests/src/test/resources/docker-compose-standalone.yml @@ -29,7 +29,7 @@ services: - ./bootstrap.conf:/opt/minifi/minifi-${minifi.version}/conf/bootstrap.conf - ./logback.xml:/opt/minifi/minifi-${minifi.version}/conf/logback.xml - - REPLACED_WITH_CONFIG_FILE:/opt/minifi/minifi-${minifi.version}/conf/config.yml + - REPLACED_WITH_CONFIG_FILE:/opt/minifi/minifi-${minifi.version}/conf/flow.json.raw entrypoint: - sh - -c diff --git a/minifi/minifi-integration-tests/src/test/resources/standalone/CsvToJson/CsvToJson.json b/minifi/minifi-integration-tests/src/test/resources/standalone/CsvToJson/CsvToJson.json new file mode 100644 index 0000000000..ac34a38fd1 --- /dev/null +++ b/minifi/minifi-integration-tests/src/test/resources/standalone/CsvToJson/CsvToJson.json @@ -0,0 +1,415 @@ +{ + "encodingVersion": { + "majorVersion": 2, + "minorVersion": 0 + }, + "maxTimerDrivenThreadCount": 1, + "maxEventDrivenThreadCount": 1, + "registries": [], + "parameterContexts": [], + "parameterProviders": [], + "controllerServices": [], + "reportingTasks": [], + "templates": [], + "rootGroup": { + "identifier": "f4cd72f7-b235-394b-928a-ffb0e7cf2db6", + "instanceIdentifier": "Root-Group", + "name": "CsvToJsonWorking", + "comments": "", + "position": { + "x": 0.0, + "y": 0.0 + }, + "processGroups": [], + "remoteProcessGroups": [], + "processors": [ + { + "identifier": "dc370c01-1c12-32f9-ba7b-c5bf71004b33", + "instanceIdentifier": "b72a73f2-66b6-34da-bff5-2680672dde90", + "name": "ExtractText", + "comments": "", + "position": { + "x": 0.0, + "y": 0.0 + }, + "type": "org.apache.nifi.processors.standard.ExtractText", + "bundle": { + "group": "org.apache.nifi.minifi", + "artifact": "minifi-standard-nar", + "version": "2.0.0-SNAPSHOT" + }, + "properties": { + "Enable Unicode Predefined Character Classes": "false", + "Permit Whitespace and Comments in Pattern": "false", + "Enable Unicode-aware Case Folding": "false", + "csv": "(.+),(.+),(.+),(.+)", + "Enable DOTALL Mode": "false", + "Enable Unix Lines Mode": "false", + "extract-text-enable-named-groups": "false", + "Maximum Buffer Size": "1 MB", + "Enable Canonical Equivalence": "false", + "Enable Case-insensitive Matching": "false", + "Enable Multiline Mode": "false", + "Maximum Capture Group Length": "1024", + "Enable Literal Parsing of the Pattern": "false", + "Character Set": "UTF-8", + "Include Capture Group 0": "false", + "extract-text-enable-repeating-capture-group": "false" + }, + "propertyDescriptors": {}, + "style": {}, + "schedulingPeriod": "0 sec", + "schedulingStrategy": "TIMER_DRIVEN", + "executionNode": "ALL", + "penaltyDuration": "30 sec", + "yieldDuration": "1 sec", + "bulletinLevel": "WARN", + "runDurationMillis": 0, + "concurrentlySchedulableTaskCount": 1, + "autoTerminatedRelationships": [ + "unmatched" + ], + "scheduledState": "RUNNING", + "retryCount": 10, + "retriedRelationships": [], + "backoffMechanism": "PENALIZE_FLOWFILE", + "maxBackoffPeriod": "10 mins", + "componentType": "PROCESSOR", + "groupIdentifier": "f4cd72f7-b235-394b-928a-ffb0e7cf2db6" + }, + { + "identifier": "52f88bf6-4da1-3ff3-8a5d-98a450afef72", + "instanceIdentifier": "7558839f-1a75-32c1-a7e6-bfa0f4e9481a", + "name": "ReplaceText", + "comments": "", + "position": { + "x": 0.0, + "y": 0.0 + }, + "type": "org.apache.nifi.processors.standard.ReplaceText", + "bundle": { + "group": "org.apache.nifi.minifi", + "artifact": "minifi-standard-nar", + "version": "2.0.0-SNAPSHOT" + }, + "properties": { + "Regular Expression": "(?s:^.*$)", + "Replacement Value": "a,b,c,d", + "Evaluation Mode": "Entire text", + "Line-by-Line Evaluation Mode": "All", + "Character Set": "UTF-8", + "Maximum Buffer Size": "1 MB", + "Replacement Strategy": "Regex Replace" + }, + "propertyDescriptors": {}, + "style": {}, + "schedulingPeriod": "0 sec", + "schedulingStrategy": "TIMER_DRIVEN", + "executionNode": "ALL", + "penaltyDuration": "30 sec", + "yieldDuration": "1 sec", + "bulletinLevel": "WARN", + "runDurationMillis": 0, + "concurrentlySchedulableTaskCount": 1, + "autoTerminatedRelationships": [ + "failure" + ], + "scheduledState": "RUNNING", + "retryCount": 10, + "retriedRelationships": [], + "backoffMechanism": "PENALIZE_FLOWFILE", + "maxBackoffPeriod": "10 mins", + "componentType": "PROCESSOR", + "groupIdentifier": "f4cd72f7-b235-394b-928a-ffb0e7cf2db6" + }, + { + "identifier": "7b18e7a8-2208-3922-a256-a6f3581e5928", + "instanceIdentifier": "8091b00c-5185-3b31-b096-82b8e301e0ab", + "name": "GenerateFlowFile", + "comments": "", + "position": { + "x": 0.0, + "y": 0.0 + }, + "type": "org.apache.nifi.processors.standard.GenerateFlowFile", + "bundle": { + "group": "org.apache.nifi.minifi", + "artifact": "minifi-standard-nar", + "version": "2.0.0-SNAPSHOT" + }, + "properties": { + "character-set": "UTF-8", + "File Size": "1 b", + "Batch Size": "1", + "Unique FlowFiles": "false", + "Data Format": "Binary" + }, + "propertyDescriptors": {}, + "style": {}, + "schedulingPeriod": "1 sec", + "schedulingStrategy": "TIMER_DRIVEN", + "executionNode": "ALL", + "penaltyDuration": "30 sec", + "yieldDuration": "1 sec", + "bulletinLevel": "WARN", + "runDurationMillis": 0, + "concurrentlySchedulableTaskCount": 1, + "autoTerminatedRelationships": [], + "scheduledState": "RUNNING", + "retryCount": 10, + "retriedRelationships": [], + "backoffMechanism": "PENALIZE_FLOWFILE", + "maxBackoffPeriod": "10 mins", + "componentType": "PROCESSOR", + "groupIdentifier": "f4cd72f7-b235-394b-928a-ffb0e7cf2db6" + }, + { + "identifier": "e484e3cf-3cf0-31bf-8376-3de608d6df33", + "instanceIdentifier": "16facfcf-a160-3b5d-95be-50e352c3495b", + "name": "ReplaceText2", + "comments": "", + "position": { + "x": 0.0, + "y": 0.0 + }, + "type": "org.apache.nifi.processors.standard.ReplaceText", + "bundle": { + "group": "org.apache.nifi.minifi", + "artifact": "minifi-standard-nar", + "version": "2.0.0-SNAPSHOT" + }, + "properties": { + "Regular Expression": "(?s:^.*$)", + "Replacement Value": "{ \"field1\" : \"${csv.1}\", \"field2\" : \"${csv.2}\",\n \"field3\" : \"${csv.3}\", \"field4\" : \"${csv.4}\" }\n ", + "Evaluation Mode": "Entire text", + "Line-by-Line Evaluation Mode": "All", + "Character Set": "UTF-8", + "Maximum Buffer Size": "1 MB", + "Replacement Strategy": "Regex Replace" + }, + "propertyDescriptors": {}, + "style": {}, + "schedulingPeriod": "0 sec", + "schedulingStrategy": "TIMER_DRIVEN", + "executionNode": "ALL", + "penaltyDuration": "30 sec", + "yieldDuration": "1 sec", + "bulletinLevel": "WARN", + "runDurationMillis": 0, + "concurrentlySchedulableTaskCount": 1, + "autoTerminatedRelationships": [ + "failure" + ], + "scheduledState": "RUNNING", + "retryCount": 10, + "retriedRelationships": [], + "backoffMechanism": "PENALIZE_FLOWFILE", + "maxBackoffPeriod": "10 mins", + "componentType": "PROCESSOR", + "groupIdentifier": "f4cd72f7-b235-394b-928a-ffb0e7cf2db6" + }, + { + "identifier": "0b39a9da-1d8a-3f0a-9fda-d51a79d603e9", + "instanceIdentifier": "2aac227f-e8e9-370f-87c8-4f970e0b260e", + "name": "UpdateAttribute", + "comments": "", + "position": { + "x": 0.0, + "y": 0.0 + }, + "type": "org.apache.nifi.processors.attributes.UpdateAttribute", + "bundle": { + "group": "org.apache.nifi.minifi", + "artifact": "minifi-update-attribute-nar", + "version": "2.0.0-SNAPSHOT" + }, + "properties": { + "Store State": "Do not store state", + "canonical-value-lookup-cache-size": "100" + }, + "propertyDescriptors": {}, + "style": {}, + "schedulingPeriod": "0 sec", + "schedulingStrategy": "TIMER_DRIVEN", + "executionNode": "ALL", + "penaltyDuration": "30 sec", + "yieldDuration": "1 sec", + "bulletinLevel": "WARN", + "runDurationMillis": 0, + "concurrentlySchedulableTaskCount": 1, + "autoTerminatedRelationships": [ + "success" + ], + "scheduledState": "RUNNING", + "retryCount": 10, + "retriedRelationships": [], + "backoffMechanism": "PENALIZE_FLOWFILE", + "maxBackoffPeriod": "10 mins", + "componentType": "PROCESSOR", + "groupIdentifier": "f4cd72f7-b235-394b-928a-ffb0e7cf2db6" + } + ], + "inputPorts": [], + "outputPorts": [], + "connections": [ + { + "identifier": "00880aef-b3fe-32e0-8fbc-ca87f8d35a3d", + "instanceIdentifier": "3bc2238e-763d-3f6d-a966-8505a6fbab10", + "name": "ReplaceText2/success/UpdateAttribute", + "source": { + "id": "e484e3cf-3cf0-31bf-8376-3de608d6df33", + "type": "PROCESSOR", + "groupId": "f4cd72f7-b235-394b-928a-ffb0e7cf2db6", + "name": "ReplaceText2", + "comments": "", + "instanceIdentifier": "16facfcf-a160-3b5d-95be-50e352c3495b" + }, + "destination": { + "id": "0b39a9da-1d8a-3f0a-9fda-d51a79d603e9", + "type": "PROCESSOR", + "groupId": "f4cd72f7-b235-394b-928a-ffb0e7cf2db6", + "name": "UpdateAttribute", + "comments": "", + "instanceIdentifier": "2aac227f-e8e9-370f-87c8-4f970e0b260e" + }, + "labelIndex": 1, + "zIndex": 0, + "selectedRelationships": [ + "success" + ], + "backPressureObjectThreshold": 0, + "backPressureDataSizeThreshold": "0 MB", + "flowFileExpiration": "0 sec", + "prioritizers": [ + "org.apache.nifi.prioritizer.FirstInFirstOutPrioritizer" + ], + "bends": [], + "loadBalanceStrategy": "DO_NOT_LOAD_BALANCE", + "loadBalanceCompression": "DO_NOT_COMPRESS", + "componentType": "CONNECTION", + "groupIdentifier": "f4cd72f7-b235-394b-928a-ffb0e7cf2db6" + }, + { + "identifier": "be1c5052-f74c-3dea-8450-13314c64e065", + "instanceIdentifier": "73c910c8-8553-3c7b-8bf4-89413e06ebfb", + "name": "ReplaceText/success/ExtractText", + "source": { + "id": "52f88bf6-4da1-3ff3-8a5d-98a450afef72", + "type": "PROCESSOR", + "groupId": "f4cd72f7-b235-394b-928a-ffb0e7cf2db6", + "name": "ReplaceText", + "comments": "", + "instanceIdentifier": "7558839f-1a75-32c1-a7e6-bfa0f4e9481a" + }, + "destination": { + "id": "dc370c01-1c12-32f9-ba7b-c5bf71004b33", + "type": "PROCESSOR", + "groupId": "f4cd72f7-b235-394b-928a-ffb0e7cf2db6", + "name": "ExtractText", + "comments": "", + "instanceIdentifier": "b72a73f2-66b6-34da-bff5-2680672dde90" + }, + "labelIndex": 1, + "zIndex": 0, + "selectedRelationships": [ + "success" + ], + "backPressureObjectThreshold": 0, + "backPressureDataSizeThreshold": "0 MB", + "flowFileExpiration": "0 sec", + "prioritizers": [ + "org.apache.nifi.prioritizer.FirstInFirstOutPrioritizer" + ], + "bends": [], + "loadBalanceStrategy": "DO_NOT_LOAD_BALANCE", + "loadBalanceCompression": "DO_NOT_COMPRESS", + "componentType": "CONNECTION", + "groupIdentifier": "f4cd72f7-b235-394b-928a-ffb0e7cf2db6" + }, + { + "identifier": "c4e3fae5-0d63-3794-9feb-566d5dac9d14", + "instanceIdentifier": "818ebab4-1fab-3a0d-b0e8-2cd266602353", + "name": "ExtractText/matched/ReplaceText2", + "source": { + "id": "dc370c01-1c12-32f9-ba7b-c5bf71004b33", + "type": "PROCESSOR", + "groupId": "f4cd72f7-b235-394b-928a-ffb0e7cf2db6", + "name": "ExtractText", + "comments": "", + "instanceIdentifier": "b72a73f2-66b6-34da-bff5-2680672dde90" + }, + "destination": { + "id": "e484e3cf-3cf0-31bf-8376-3de608d6df33", + "type": "PROCESSOR", + "groupId": "f4cd72f7-b235-394b-928a-ffb0e7cf2db6", + "name": "ReplaceText2", + "comments": "", + "instanceIdentifier": "16facfcf-a160-3b5d-95be-50e352c3495b" + }, + "labelIndex": 1, + "zIndex": 0, + "selectedRelationships": [ + "matched" + ], + "backPressureObjectThreshold": 0, + "backPressureDataSizeThreshold": "0 MB", + "flowFileExpiration": "0 sec", + "prioritizers": [ + "org.apache.nifi.prioritizer.FirstInFirstOutPrioritizer" + ], + "bends": [], + "loadBalanceStrategy": "DO_NOT_LOAD_BALANCE", + "loadBalanceCompression": "DO_NOT_COMPRESS", + "componentType": "CONNECTION", + "groupIdentifier": "f4cd72f7-b235-394b-928a-ffb0e7cf2db6" + }, + { + "identifier": "8a3935b3-9b5d-3bb9-a582-68241cd76c22", + "instanceIdentifier": "2df4cf99-1702-33a6-a94d-0a9dc37b987e", + "name": "GenerateFlowFile/success/ReplaceText", + "source": { + "id": "7b18e7a8-2208-3922-a256-a6f3581e5928", + "type": "PROCESSOR", + "groupId": "f4cd72f7-b235-394b-928a-ffb0e7cf2db6", + "name": "GenerateFlowFile", + "comments": "", + "instanceIdentifier": "8091b00c-5185-3b31-b096-82b8e301e0ab" + }, + "destination": { + "id": "52f88bf6-4da1-3ff3-8a5d-98a450afef72", + "type": "PROCESSOR", + "groupId": "f4cd72f7-b235-394b-928a-ffb0e7cf2db6", + "name": "ReplaceText", + "comments": "", + "instanceIdentifier": "7558839f-1a75-32c1-a7e6-bfa0f4e9481a" + }, + "labelIndex": 1, + "zIndex": 0, + "selectedRelationships": [ + "success" + ], + "backPressureObjectThreshold": 0, + "backPressureDataSizeThreshold": "0 MB", + "flowFileExpiration": "0 sec", + "prioritizers": [ + "org.apache.nifi.prioritizer.FirstInFirstOutPrioritizer" + ], + "bends": [], + "loadBalanceStrategy": "DO_NOT_LOAD_BALANCE", + "loadBalanceCompression": "DO_NOT_COMPRESS", + "componentType": "CONNECTION", + "groupIdentifier": "f4cd72f7-b235-394b-928a-ffb0e7cf2db6" + } + ], + "labels": [], + "funnels": [], + "controllerServices": [], + "variables": {}, + "defaultFlowFileExpiration": "0 sec", + "defaultBackPressureObjectThreshold": 10000, + "defaultBackPressureDataSizeThreshold": "1 GB", + "componentType": "PROCESS_GROUP", + "flowFileConcurrency": "UNBOUNDED", + "flowFileOutboundPolicy": "STREAM_WHEN_AVAILABLE" + } +} \ No newline at end of file diff --git a/minifi/minifi-integration-tests/src/test/resources/standalone/v1/CsvToJson/yml/expected.json b/minifi/minifi-integration-tests/src/test/resources/standalone/CsvToJson/expected.json similarity index 100% rename from minifi/minifi-integration-tests/src/test/resources/standalone/v1/CsvToJson/yml/expected.json rename to minifi/minifi-integration-tests/src/test/resources/standalone/CsvToJson/expected.json diff --git a/minifi/minifi-integration-tests/src/test/resources/standalone/DecompressionCircularFlow/DecompressionCircularFlow.json b/minifi/minifi-integration-tests/src/test/resources/standalone/DecompressionCircularFlow/DecompressionCircularFlow.json new file mode 100644 index 0000000000..20bfc3d5ef --- /dev/null +++ b/minifi/minifi-integration-tests/src/test/resources/standalone/DecompressionCircularFlow/DecompressionCircularFlow.json @@ -0,0 +1,920 @@ +{ + "encodingVersion": { + "majorVersion": 2, + "minorVersion": 0 + }, + "maxTimerDrivenThreadCount": 1, + "maxEventDrivenThreadCount": 1, + "registries": [], + "parameterContexts": [], + "parameterProviders": [], + "controllerServices": [], + "reportingTasks": [], + "templates": [], + "rootGroup": { + "identifier": "f4cd72f7-b235-394b-928a-ffb0e7cf2db6", + "instanceIdentifier": "Root-Group", + "name": "DecompressionCircularFlow2", + "comments": "", + "position": { + "x": 0.0, + "y": 0.0 + }, + "processGroups": [], + "remoteProcessGroups": [], + "processors": [ + { + "identifier": "53e6b983-8ed5-369e-8d1a-5b768286b396", + "instanceIdentifier": "87d03875-3fed-321a-8098-84ef62a7c1a9", + "name": "Uncompress GZIP", + "comments": "", + "position": { + "x": 0.0, + "y": 0.0 + }, + "type": "org.apache.nifi.processors.standard.CompressContent", + "bundle": { + "group": "org.apache.nifi.minifi", + "artifact": "minifi-standard-nar", + "version": "2.0.0-SNAPSHOT" + }, + "properties": { + "Compression Format": "gzip", + "Mode": "decompress", + "Compression Level": "1", + "Update Filename": "false" + }, + "propertyDescriptors": {}, + "style": {}, + "schedulingPeriod": "0 sec", + "schedulingStrategy": "TIMER_DRIVEN", + "executionNode": "ALL", + "penaltyDuration": "30 sec", + "yieldDuration": "1 sec", + "bulletinLevel": "WARN", + "runDurationMillis": 0, + "concurrentlySchedulableTaskCount": 1, + "autoTerminatedRelationships": [], + "scheduledState": "RUNNING", + "retryCount": 10, + "retriedRelationships": [], + "backoffMechanism": "PENALIZE_FLOWFILE", + "maxBackoffPeriod": "10 mins", + "componentType": "PROCESSOR", + "groupIdentifier": "f4cd72f7-b235-394b-928a-ffb0e7cf2db6" + }, + { + "identifier": "3288c07e-9b07-3186-a980-88f98b987bb1", + "instanceIdentifier": "a0d668cb-5768-31e3-929f-f508706298f9", + "name": "IdentifyMimeType", + "comments": "", + "position": { + "x": 0.0, + "y": 0.0 + }, + "type": "org.apache.nifi.processors.standard.IdentifyMimeType", + "bundle": { + "group": "org.apache.nifi.minifi", + "artifact": "minifi-standard-nar", + "version": "2.0.0-SNAPSHOT" + }, + "properties": { + "use-filename-in-detection": "true" + }, + "propertyDescriptors": {}, + "style": {}, + "schedulingPeriod": "0 sec", + "schedulingStrategy": "TIMER_DRIVEN", + "executionNode": "ALL", + "penaltyDuration": "30 sec", + "yieldDuration": "1 sec", + "bulletinLevel": "WARN", + "runDurationMillis": 0, + "concurrentlySchedulableTaskCount": 1, + "autoTerminatedRelationships": [], + "scheduledState": "RUNNING", + "retryCount": 10, + "retriedRelationships": [], + "backoffMechanism": "PENALIZE_FLOWFILE", + "maxBackoffPeriod": "10 mins", + "componentType": "PROCESSOR", + "groupIdentifier": "f4cd72f7-b235-394b-928a-ffb0e7cf2db6" + }, + { + "identifier": "b2c7272c-b5dc-3f23-b956-7aa9ea5397d8", + "instanceIdentifier": "119d2969-17e4-34e5-a280-f851ea59cf90", + "name": "GZIP CompressContent2", + "comments": "", + "position": { + "x": 0.0, + "y": 0.0 + }, + "type": "org.apache.nifi.processors.standard.CompressContent", + "bundle": { + "group": "org.apache.nifi.minifi", + "artifact": "minifi-standard-nar", + "version": "2.0.0-SNAPSHOT" + }, + "properties": { + "Compression Format": "gzip", + "Mode": "compress", + "Compression Level": "1", + "Update Filename": "false" + }, + "propertyDescriptors": {}, + "style": {}, + "schedulingPeriod": "0 sec", + "schedulingStrategy": "TIMER_DRIVEN", + "executionNode": "ALL", + "penaltyDuration": "30 sec", + "yieldDuration": "1 sec", + "bulletinLevel": "WARN", + "runDurationMillis": 0, + "concurrentlySchedulableTaskCount": 1, + "autoTerminatedRelationships": [], + "scheduledState": "RUNNING", + "retryCount": 10, + "retriedRelationships": [], + "backoffMechanism": "PENALIZE_FLOWFILE", + "maxBackoffPeriod": "10 mins", + "componentType": "PROCESSOR", + "groupIdentifier": "f4cd72f7-b235-394b-928a-ffb0e7cf2db6" + }, + { + "identifier": "6a95ee4f-164f-31ba-84fb-7e53df2971bd", + "instanceIdentifier": "59198b81-12fc-374f-bead-7fcd1d5d4823", + "name": "LogAttribute", + "comments": "", + "position": { + "x": 0.0, + "y": 0.0 + }, + "type": "org.apache.nifi.processors.standard.LogAttribute", + "bundle": { + "group": "org.apache.nifi.minifi", + "artifact": "minifi-standard-nar", + "version": "2.0.0-SNAPSHOT" + }, + "properties": { + "character-set": "UTF-8", + "Log FlowFile Properties": "true", + "Log Level": "info", + "attributes-to-log-regex": ".*", + "Output Format": "Line per Attribute", + "Log Payload": "false" + }, + "propertyDescriptors": {}, + "style": {}, + "schedulingPeriod": "0 sec", + "schedulingStrategy": "TIMER_DRIVEN", + "executionNode": "ALL", + "penaltyDuration": "30 sec", + "yieldDuration": "1 sec", + "bulletinLevel": "WARN", + "runDurationMillis": 0, + "concurrentlySchedulableTaskCount": 1, + "autoTerminatedRelationships": [ + "success" + ], + "scheduledState": "RUNNING", + "retryCount": 10, + "retriedRelationships": [], + "backoffMechanism": "PENALIZE_FLOWFILE", + "maxBackoffPeriod": "10 mins", + "componentType": "PROCESSOR", + "groupIdentifier": "f4cd72f7-b235-394b-928a-ffb0e7cf2db6" + }, + { + "identifier": "7b18e7a8-2208-3922-a256-a6f3581e5928", + "instanceIdentifier": "8091b00c-5185-3b31-b096-82b8e301e0ab", + "name": "GenerateFlowFile", + "comments": "", + "position": { + "x": 0.0, + "y": 0.0 + }, + "type": "org.apache.nifi.processors.standard.GenerateFlowFile", + "bundle": { + "group": "org.apache.nifi.minifi", + "artifact": "minifi-standard-nar", + "version": "2.0.0-SNAPSHOT" + }, + "properties": { + "character-set": "UTF-8", + "File Size": "10kb", + "Batch Size": "1", + "Unique FlowFiles": "false", + "Data Format": "Text" + }, + "propertyDescriptors": {}, + "style": {}, + "schedulingPeriod": "1 min", + "schedulingStrategy": "TIMER_DRIVEN", + "executionNode": "ALL", + "penaltyDuration": "30 sec", + "yieldDuration": "1 sec", + "bulletinLevel": "WARN", + "runDurationMillis": 0, + "concurrentlySchedulableTaskCount": 1, + "autoTerminatedRelationships": [], + "scheduledState": "RUNNING", + "retryCount": 10, + "retriedRelationships": [], + "backoffMechanism": "PENALIZE_FLOWFILE", + "maxBackoffPeriod": "10 mins", + "componentType": "PROCESSOR", + "groupIdentifier": "f4cd72f7-b235-394b-928a-ffb0e7cf2db6" + }, + { + "identifier": "4aac7713-3c10-30ea-a2ad-9caa4a1f0365", + "instanceIdentifier": "b44f06a7-ef15-36bc-817f-c127cfdcac39", + "name": "GZIP CompressContent3", + "comments": "", + "position": { + "x": 0.0, + "y": 0.0 + }, + "type": "org.apache.nifi.processors.standard.CompressContent", + "bundle": { + "group": "org.apache.nifi.minifi", + "artifact": "minifi-standard-nar", + "version": "2.0.0-SNAPSHOT" + }, + "properties": { + "Compression Format": "gzip", + "Mode": "compress", + "Compression Level": "1", + "Update Filename": "false" + }, + "propertyDescriptors": {}, + "style": {}, + "schedulingPeriod": "0 sec", + "schedulingStrategy": "TIMER_DRIVEN", + "executionNode": "ALL", + "penaltyDuration": "30 sec", + "yieldDuration": "1 sec", + "bulletinLevel": "WARN", + "runDurationMillis": 0, + "concurrentlySchedulableTaskCount": 1, + "autoTerminatedRelationships": [], + "scheduledState": "RUNNING", + "retryCount": 10, + "retriedRelationships": [], + "backoffMechanism": "PENALIZE_FLOWFILE", + "maxBackoffPeriod": "10 mins", + "componentType": "PROCESSOR", + "groupIdentifier": "f4cd72f7-b235-394b-928a-ffb0e7cf2db6" + }, + { + "identifier": "9cb6c6ae-d94e-31ac-8e68-4a0aecbcbbc2", + "instanceIdentifier": "0a6771cc-4527-31bb-8404-e776167bf1ed", + "name": "GZIP CompressContent", + "comments": "", + "position": { + "x": 0.0, + "y": 0.0 + }, + "type": "org.apache.nifi.processors.standard.CompressContent", + "bundle": { + "group": "org.apache.nifi.minifi", + "artifact": "minifi-standard-nar", + "version": "2.0.0-SNAPSHOT" + }, + "properties": { + "Compression Format": "gzip", + "Mode": "compress", + "Compression Level": "1", + "Update Filename": "false" + }, + "propertyDescriptors": {}, + "style": {}, + "schedulingPeriod": "0 sec", + "schedulingStrategy": "TIMER_DRIVEN", + "executionNode": "ALL", + "penaltyDuration": "30 sec", + "yieldDuration": "1 sec", + "bulletinLevel": "WARN", + "runDurationMillis": 0, + "concurrentlySchedulableTaskCount": 1, + "autoTerminatedRelationships": [], + "scheduledState": "RUNNING", + "retryCount": 10, + "retriedRelationships": [], + "backoffMechanism": "PENALIZE_FLOWFILE", + "maxBackoffPeriod": "10 mins", + "componentType": "PROCESSOR", + "groupIdentifier": "f4cd72f7-b235-394b-928a-ffb0e7cf2db6" + }, + { + "identifier": "0596dae5-c840-3ef7-8dda-f65fa558b7eb", + "instanceIdentifier": "96d681db-c086-38bf-8cb3-d57aa43f9e6e", + "name": "Compressed?", + "comments": "", + "position": { + "x": 0.0, + "y": 0.0 + }, + "type": "org.apache.nifi.processors.standard.RouteOnAttribute", + "bundle": { + "group": "org.apache.nifi.minifi", + "artifact": "minifi-standard-nar", + "version": "2.0.0-SNAPSHOT" + }, + "properties": { + "Routing Strategy": "Route to Property name", + "gzip": "${mime.type:toUpper():contains(\"GZIP\")}" + }, + "propertyDescriptors": {}, + "style": {}, + "schedulingPeriod": "0 sec", + "schedulingStrategy": "TIMER_DRIVEN", + "executionNode": "ALL", + "penaltyDuration": "30 sec", + "yieldDuration": "1 sec", + "bulletinLevel": "WARN", + "runDurationMillis": 0, + "concurrentlySchedulableTaskCount": 1, + "autoTerminatedRelationships": [], + "scheduledState": "RUNNING", + "retryCount": 10, + "retriedRelationships": [], + "backoffMechanism": "PENALIZE_FLOWFILE", + "maxBackoffPeriod": "10 mins", + "componentType": "PROCESSOR", + "groupIdentifier": "f4cd72f7-b235-394b-928a-ffb0e7cf2db6" + }, + { + "identifier": "6be5b72f-148e-3187-a079-6538cfabbe76", + "instanceIdentifier": "0666040b-9f34-3a69-ac62-6d4e3c01ee3f", + "name": "GZIP CompressContent4", + "comments": "", + "position": { + "x": 0.0, + "y": 0.0 + }, + "type": "org.apache.nifi.processors.standard.CompressContent", + "bundle": { + "group": "org.apache.nifi.minifi", + "artifact": "minifi-standard-nar", + "version": "2.0.0-SNAPSHOT" + }, + "properties": { + "Compression Format": "gzip", + "Mode": "compress", + "Compression Level": "1", + "Update Filename": "false" + }, + "propertyDescriptors": {}, + "style": {}, + "schedulingPeriod": "0 sec", + "schedulingStrategy": "TIMER_DRIVEN", + "executionNode": "ALL", + "penaltyDuration": "30 sec", + "yieldDuration": "1 sec", + "bulletinLevel": "WARN", + "runDurationMillis": 0, + "concurrentlySchedulableTaskCount": 1, + "autoTerminatedRelationships": [], + "scheduledState": "RUNNING", + "retryCount": 10, + "retriedRelationships": [], + "backoffMechanism": "PENALIZE_FLOWFILE", + "maxBackoffPeriod": "10 mins", + "componentType": "PROCESSOR", + "groupIdentifier": "f4cd72f7-b235-394b-928a-ffb0e7cf2db6" + } + ], + "inputPorts": [], + "outputPorts": [], + "connections": [ + { + "identifier": "4c1e5e72-cd01-3b59-8e3b-8c78f731a468", + "instanceIdentifier": "69b4106e-7e12-339a-947a-bf7ff19bae2f", + "name": "GZIP CompressContent4/success/Uncompress GZIP", + "source": { + "id": "6be5b72f-148e-3187-a079-6538cfabbe76", + "type": "PROCESSOR", + "groupId": "f4cd72f7-b235-394b-928a-ffb0e7cf2db6", + "name": "GZIP CompressContent4", + "comments": "", + "instanceIdentifier": "0666040b-9f34-3a69-ac62-6d4e3c01ee3f" + }, + "destination": { + "id": "53e6b983-8ed5-369e-8d1a-5b768286b396", + "type": "PROCESSOR", + "groupId": "f4cd72f7-b235-394b-928a-ffb0e7cf2db6", + "name": "Uncompress GZIP", + "comments": "", + "instanceIdentifier": "87d03875-3fed-321a-8098-84ef62a7c1a9" + }, + "labelIndex": 1, + "zIndex": 0, + "selectedRelationships": [ + "success" + ], + "backPressureObjectThreshold": 0, + "backPressureDataSizeThreshold": "0 MB", + "flowFileExpiration": "0 sec", + "prioritizers": [ + "org.apache.nifi.prioritizer.FirstInFirstOutPrioritizer" + ], + "bends": [], + "loadBalanceStrategy": "DO_NOT_LOAD_BALANCE", + "loadBalanceCompression": "DO_NOT_COMPRESS", + "componentType": "CONNECTION", + "groupIdentifier": "f4cd72f7-b235-394b-928a-ffb0e7cf2db6" + }, + { + "identifier": "066e2e5d-610e-3d71-af68-08838840a3a2", + "instanceIdentifier": "5bad6b73-313d-32e9-9ac8-bc6550afbea2", + "name": "GZIP CompressContent3/success/GZIP CompressContent4", + "source": { + "id": "4aac7713-3c10-30ea-a2ad-9caa4a1f0365", + "type": "PROCESSOR", + "groupId": "f4cd72f7-b235-394b-928a-ffb0e7cf2db6", + "name": "GZIP CompressContent3", + "comments": "", + "instanceIdentifier": "b44f06a7-ef15-36bc-817f-c127cfdcac39" + }, + "destination": { + "id": "6be5b72f-148e-3187-a079-6538cfabbe76", + "type": "PROCESSOR", + "groupId": "f4cd72f7-b235-394b-928a-ffb0e7cf2db6", + "name": "GZIP CompressContent4", + "comments": "", + "instanceIdentifier": "0666040b-9f34-3a69-ac62-6d4e3c01ee3f" + }, + "labelIndex": 1, + "zIndex": 0, + "selectedRelationships": [ + "success" + ], + "backPressureObjectThreshold": 0, + "backPressureDataSizeThreshold": "0 MB", + "flowFileExpiration": "0 sec", + "prioritizers": [ + "org.apache.nifi.prioritizer.FirstInFirstOutPrioritizer" + ], + "bends": [], + "loadBalanceStrategy": "DO_NOT_LOAD_BALANCE", + "loadBalanceCompression": "DO_NOT_COMPRESS", + "componentType": "CONNECTION", + "groupIdentifier": "f4cd72f7-b235-394b-928a-ffb0e7cf2db6" + }, + { + "identifier": "f9315b47-d93e-3ab6-aec1-3be494746c6e", + "instanceIdentifier": "0965e6d0-573d-3ea2-bdaa-76a64690fe00", + "name": "GZIP CompressContent/success/GZIP CompressContent2", + "source": { + "id": "9cb6c6ae-d94e-31ac-8e68-4a0aecbcbbc2", + "type": "PROCESSOR", + "groupId": "f4cd72f7-b235-394b-928a-ffb0e7cf2db6", + "name": "GZIP CompressContent", + "comments": "", + "instanceIdentifier": "0a6771cc-4527-31bb-8404-e776167bf1ed" + }, + "destination": { + "id": "b2c7272c-b5dc-3f23-b956-7aa9ea5397d8", + "type": "PROCESSOR", + "groupId": "f4cd72f7-b235-394b-928a-ffb0e7cf2db6", + "name": "GZIP CompressContent2", + "comments": "", + "instanceIdentifier": "119d2969-17e4-34e5-a280-f851ea59cf90" + }, + "labelIndex": 1, + "zIndex": 0, + "selectedRelationships": [ + "success" + ], + "backPressureObjectThreshold": 0, + "backPressureDataSizeThreshold": "0 MB", + "flowFileExpiration": "0 sec", + "prioritizers": [ + "org.apache.nifi.prioritizer.FirstInFirstOutPrioritizer" + ], + "bends": [], + "loadBalanceStrategy": "DO_NOT_LOAD_BALANCE", + "loadBalanceCompression": "DO_NOT_COMPRESS", + "componentType": "CONNECTION", + "groupIdentifier": "f4cd72f7-b235-394b-928a-ffb0e7cf2db6" + }, + { + "identifier": "baa96fe3-5d32-3f0c-aa06-0c421da669e8", + "instanceIdentifier": "bdb25d97-7a33-38b2-8aef-eddd9fff4bb2", + "name": "GZIP CompressContent2/success/GZIP CompressContent3", + "source": { + "id": "b2c7272c-b5dc-3f23-b956-7aa9ea5397d8", + "type": "PROCESSOR", + "groupId": "f4cd72f7-b235-394b-928a-ffb0e7cf2db6", + "name": "GZIP CompressContent2", + "comments": "", + "instanceIdentifier": "119d2969-17e4-34e5-a280-f851ea59cf90" + }, + "destination": { + "id": "4aac7713-3c10-30ea-a2ad-9caa4a1f0365", + "type": "PROCESSOR", + "groupId": "f4cd72f7-b235-394b-928a-ffb0e7cf2db6", + "name": "GZIP CompressContent3", + "comments": "", + "instanceIdentifier": "b44f06a7-ef15-36bc-817f-c127cfdcac39" + }, + "labelIndex": 1, + "zIndex": 0, + "selectedRelationships": [ + "success" + ], + "backPressureObjectThreshold": 0, + "backPressureDataSizeThreshold": "0 MB", + "flowFileExpiration": "0 sec", + "prioritizers": [ + "org.apache.nifi.prioritizer.FirstInFirstOutPrioritizer" + ], + "bends": [], + "loadBalanceStrategy": "DO_NOT_LOAD_BALANCE", + "loadBalanceCompression": "DO_NOT_COMPRESS", + "componentType": "CONNECTION", + "groupIdentifier": "f4cd72f7-b235-394b-928a-ffb0e7cf2db6" + }, + { + "identifier": "802f7401-3eb9-3f08-87b2-42a6baf453e6", + "instanceIdentifier": "006766ee-6e19-3900-a7e2-10510b53fdbe", + "name": "Uncompress GZIP/failure/Uncompress GZIP", + "source": { + "id": "53e6b983-8ed5-369e-8d1a-5b768286b396", + "type": "PROCESSOR", + "groupId": "f4cd72f7-b235-394b-928a-ffb0e7cf2db6", + "name": "Uncompress GZIP", + "comments": "", + "instanceIdentifier": "87d03875-3fed-321a-8098-84ef62a7c1a9" + }, + "destination": { + "id": "53e6b983-8ed5-369e-8d1a-5b768286b396", + "type": "PROCESSOR", + "groupId": "f4cd72f7-b235-394b-928a-ffb0e7cf2db6", + "name": "Uncompress GZIP", + "comments": "", + "instanceIdentifier": "87d03875-3fed-321a-8098-84ef62a7c1a9" + }, + "labelIndex": 1, + "zIndex": 0, + "selectedRelationships": [ + "failure" + ], + "backPressureObjectThreshold": 0, + "backPressureDataSizeThreshold": "0 MB", + "flowFileExpiration": "5 sec", + "prioritizers": [ + "org.apache.nifi.prioritizer.FirstInFirstOutPrioritizer" + ], + "bends": [], + "loadBalanceStrategy": "DO_NOT_LOAD_BALANCE", + "loadBalanceCompression": "DO_NOT_COMPRESS", + "componentType": "CONNECTION", + "groupIdentifier": "f4cd72f7-b235-394b-928a-ffb0e7cf2db6" + }, + { + "identifier": "af0e236e-349e-3dc3-b469-4a7e36f802f0", + "instanceIdentifier": "64ab5f49-c799-377d-a62c-c29a29b969ca", + "name": "Compressed?/unmatched/LogAttribute", + "source": { + "id": "0596dae5-c840-3ef7-8dda-f65fa558b7eb", + "type": "PROCESSOR", + "groupId": "f4cd72f7-b235-394b-928a-ffb0e7cf2db6", + "name": "Compressed?", + "comments": "", + "instanceIdentifier": "96d681db-c086-38bf-8cb3-d57aa43f9e6e" + }, + "destination": { + "id": "6a95ee4f-164f-31ba-84fb-7e53df2971bd", + "type": "PROCESSOR", + "groupId": "f4cd72f7-b235-394b-928a-ffb0e7cf2db6", + "name": "LogAttribute", + "comments": "", + "instanceIdentifier": "59198b81-12fc-374f-bead-7fcd1d5d4823" + }, + "labelIndex": 1, + "zIndex": 0, + "selectedRelationships": [ + "unmatched" + ], + "backPressureObjectThreshold": 0, + "backPressureDataSizeThreshold": "0 MB", + "flowFileExpiration": "0 sec", + "prioritizers": [ + "org.apache.nifi.prioritizer.FirstInFirstOutPrioritizer" + ], + "bends": [], + "loadBalanceStrategy": "DO_NOT_LOAD_BALANCE", + "loadBalanceCompression": "DO_NOT_COMPRESS", + "componentType": "CONNECTION", + "groupIdentifier": "f4cd72f7-b235-394b-928a-ffb0e7cf2db6" + }, + { + "identifier": "fa6697a0-61ef-37b9-be5f-d2ccfd204a35", + "instanceIdentifier": "0dd0a085-9201-329f-b3ae-002133e0a362", + "name": "GZIP CompressContent3/failure/GZIP CompressContent3", + "source": { + "id": "4aac7713-3c10-30ea-a2ad-9caa4a1f0365", + "type": "PROCESSOR", + "groupId": "f4cd72f7-b235-394b-928a-ffb0e7cf2db6", + "name": "GZIP CompressContent3", + "comments": "", + "instanceIdentifier": "b44f06a7-ef15-36bc-817f-c127cfdcac39" + }, + "destination": { + "id": "4aac7713-3c10-30ea-a2ad-9caa4a1f0365", + "type": "PROCESSOR", + "groupId": "f4cd72f7-b235-394b-928a-ffb0e7cf2db6", + "name": "GZIP CompressContent3", + "comments": "", + "instanceIdentifier": "b44f06a7-ef15-36bc-817f-c127cfdcac39" + }, + "labelIndex": 1, + "zIndex": 0, + "selectedRelationships": [ + "failure" + ], + "backPressureObjectThreshold": 0, + "backPressureDataSizeThreshold": "0 MB", + "flowFileExpiration": "0 sec", + "prioritizers": [ + "org.apache.nifi.prioritizer.FirstInFirstOutPrioritizer" + ], + "bends": [], + "loadBalanceStrategy": "DO_NOT_LOAD_BALANCE", + "loadBalanceCompression": "DO_NOT_COMPRESS", + "componentType": "CONNECTION", + "groupIdentifier": "f4cd72f7-b235-394b-928a-ffb0e7cf2db6" + }, + { + "identifier": "4add8353-0eb7-33d1-abdf-ec01b0e97de2", + "instanceIdentifier": "2a9a3202-abba-3466-9371-049d28c0e0e0", + "name": "GZIP CompressContent2/failure/GZIP CompressContent2", + "source": { + "id": "b2c7272c-b5dc-3f23-b956-7aa9ea5397d8", + "type": "PROCESSOR", + "groupId": "f4cd72f7-b235-394b-928a-ffb0e7cf2db6", + "name": "GZIP CompressContent2", + "comments": "", + "instanceIdentifier": "119d2969-17e4-34e5-a280-f851ea59cf90" + }, + "destination": { + "id": "b2c7272c-b5dc-3f23-b956-7aa9ea5397d8", + "type": "PROCESSOR", + "groupId": "f4cd72f7-b235-394b-928a-ffb0e7cf2db6", + "name": "GZIP CompressContent2", + "comments": "", + "instanceIdentifier": "119d2969-17e4-34e5-a280-f851ea59cf90" + }, + "labelIndex": 1, + "zIndex": 0, + "selectedRelationships": [ + "failure" + ], + "backPressureObjectThreshold": 0, + "backPressureDataSizeThreshold": "0 MB", + "flowFileExpiration": "0 sec", + "prioritizers": [ + "org.apache.nifi.prioritizer.FirstInFirstOutPrioritizer" + ], + "bends": [], + "loadBalanceStrategy": "DO_NOT_LOAD_BALANCE", + "loadBalanceCompression": "DO_NOT_COMPRESS", + "componentType": "CONNECTION", + "groupIdentifier": "f4cd72f7-b235-394b-928a-ffb0e7cf2db6" + }, + { + "identifier": "65b19815-61e5-344a-8fa0-c45d869f08b6", + "instanceIdentifier": "1acbf4fc-87b6-3479-8aa6-18b21df3ef66", + "name": "Uncompress GZIP/success/IdentifyMimeType", + "source": { + "id": "53e6b983-8ed5-369e-8d1a-5b768286b396", + "type": "PROCESSOR", + "groupId": "f4cd72f7-b235-394b-928a-ffb0e7cf2db6", + "name": "Uncompress GZIP", + "comments": "", + "instanceIdentifier": "87d03875-3fed-321a-8098-84ef62a7c1a9" + }, + "destination": { + "id": "3288c07e-9b07-3186-a980-88f98b987bb1", + "type": "PROCESSOR", + "groupId": "f4cd72f7-b235-394b-928a-ffb0e7cf2db6", + "name": "IdentifyMimeType", + "comments": "", + "instanceIdentifier": "a0d668cb-5768-31e3-929f-f508706298f9" + }, + "labelIndex": 1, + "zIndex": 0, + "selectedRelationships": [ + "success" + ], + "backPressureObjectThreshold": 0, + "backPressureDataSizeThreshold": "0 MB", + "flowFileExpiration": "0 sec", + "prioritizers": [ + "org.apache.nifi.prioritizer.FirstInFirstOutPrioritizer" + ], + "bends": [], + "loadBalanceStrategy": "DO_NOT_LOAD_BALANCE", + "loadBalanceCompression": "DO_NOT_COMPRESS", + "componentType": "CONNECTION", + "groupIdentifier": "f4cd72f7-b235-394b-928a-ffb0e7cf2db6" + }, + { + "identifier": "b4693de6-71ee-3d01-a4cc-b13fa365dbd4", + "instanceIdentifier": "518df694-89e8-326a-8df7-7498e66aa0dd", + "name": "Compressed?/gzip/Uncompress GZIP", + "source": { + "id": "0596dae5-c840-3ef7-8dda-f65fa558b7eb", + "type": "PROCESSOR", + "groupId": "f4cd72f7-b235-394b-928a-ffb0e7cf2db6", + "name": "Compressed?", + "comments": "", + "instanceIdentifier": "96d681db-c086-38bf-8cb3-d57aa43f9e6e" + }, + "destination": { + "id": "53e6b983-8ed5-369e-8d1a-5b768286b396", + "type": "PROCESSOR", + "groupId": "f4cd72f7-b235-394b-928a-ffb0e7cf2db6", + "name": "Uncompress GZIP", + "comments": "", + "instanceIdentifier": "87d03875-3fed-321a-8098-84ef62a7c1a9" + }, + "labelIndex": 1, + "zIndex": 0, + "selectedRelationships": [ + "gzip" + ], + "backPressureObjectThreshold": 0, + "backPressureDataSizeThreshold": "0 MB", + "flowFileExpiration": "0 sec", + "prioritizers": [ + "org.apache.nifi.prioritizer.FirstInFirstOutPrioritizer" + ], + "bends": [], + "loadBalanceStrategy": "DO_NOT_LOAD_BALANCE", + "loadBalanceCompression": "DO_NOT_COMPRESS", + "componentType": "CONNECTION", + "groupIdentifier": "f4cd72f7-b235-394b-928a-ffb0e7cf2db6" + }, + { + "identifier": "eafabc7b-ab9e-3b8b-81f3-8a76ef572a2b", + "instanceIdentifier": "29ce834f-5e46-3a24-939d-dfa8fe59a907", + "name": "GZIP CompressContent/failure/GZIP CompressContent", + "source": { + "id": "9cb6c6ae-d94e-31ac-8e68-4a0aecbcbbc2", + "type": "PROCESSOR", + "groupId": "f4cd72f7-b235-394b-928a-ffb0e7cf2db6", + "name": "GZIP CompressContent", + "comments": "", + "instanceIdentifier": "0a6771cc-4527-31bb-8404-e776167bf1ed" + }, + "destination": { + "id": "9cb6c6ae-d94e-31ac-8e68-4a0aecbcbbc2", + "type": "PROCESSOR", + "groupId": "f4cd72f7-b235-394b-928a-ffb0e7cf2db6", + "name": "GZIP CompressContent", + "comments": "", + "instanceIdentifier": "0a6771cc-4527-31bb-8404-e776167bf1ed" + }, + "labelIndex": 1, + "zIndex": 0, + "selectedRelationships": [ + "failure" + ], + "backPressureObjectThreshold": 0, + "backPressureDataSizeThreshold": "0 MB", + "flowFileExpiration": "0 sec", + "prioritizers": [ + "org.apache.nifi.prioritizer.FirstInFirstOutPrioritizer" + ], + "bends": [], + "loadBalanceStrategy": "DO_NOT_LOAD_BALANCE", + "loadBalanceCompression": "DO_NOT_COMPRESS", + "componentType": "CONNECTION", + "groupIdentifier": "f4cd72f7-b235-394b-928a-ffb0e7cf2db6" + }, + { + "identifier": "9e40e649-7184-3956-9361-81d9e32c4b36", + "instanceIdentifier": "d8d8f381-5d01-31e1-b41c-0352236ea273", + "name": "IdentifyMimeType/success/Compressed?", + "source": { + "id": "3288c07e-9b07-3186-a980-88f98b987bb1", + "type": "PROCESSOR", + "groupId": "f4cd72f7-b235-394b-928a-ffb0e7cf2db6", + "name": "IdentifyMimeType", + "comments": "", + "instanceIdentifier": "a0d668cb-5768-31e3-929f-f508706298f9" + }, + "destination": { + "id": "0596dae5-c840-3ef7-8dda-f65fa558b7eb", + "type": "PROCESSOR", + "groupId": "f4cd72f7-b235-394b-928a-ffb0e7cf2db6", + "name": "Compressed?", + "comments": "", + "instanceIdentifier": "96d681db-c086-38bf-8cb3-d57aa43f9e6e" + }, + "labelIndex": 1, + "zIndex": 0, + "selectedRelationships": [ + "success" + ], + "backPressureObjectThreshold": 0, + "backPressureDataSizeThreshold": "0 MB", + "flowFileExpiration": "0 sec", + "prioritizers": [ + "org.apache.nifi.prioritizer.FirstInFirstOutPrioritizer" + ], + "bends": [], + "loadBalanceStrategy": "DO_NOT_LOAD_BALANCE", + "loadBalanceCompression": "DO_NOT_COMPRESS", + "componentType": "CONNECTION", + "groupIdentifier": "f4cd72f7-b235-394b-928a-ffb0e7cf2db6" + }, + { + "identifier": "a6fa16bc-eeff-3d8c-88d8-455c024207a2", + "instanceIdentifier": "6047ab4f-f000-39b7-add9-b2a2e6879bbf", + "name": "GZIP CompressContent4/failure/GZIP CompressContent4", + "source": { + "id": "6be5b72f-148e-3187-a079-6538cfabbe76", + "type": "PROCESSOR", + "groupId": "f4cd72f7-b235-394b-928a-ffb0e7cf2db6", + "name": "GZIP CompressContent4", + "comments": "", + "instanceIdentifier": "0666040b-9f34-3a69-ac62-6d4e3c01ee3f" + }, + "destination": { + "id": "6be5b72f-148e-3187-a079-6538cfabbe76", + "type": "PROCESSOR", + "groupId": "f4cd72f7-b235-394b-928a-ffb0e7cf2db6", + "name": "GZIP CompressContent4", + "comments": "", + "instanceIdentifier": "0666040b-9f34-3a69-ac62-6d4e3c01ee3f" + }, + "labelIndex": 1, + "zIndex": 0, + "selectedRelationships": [ + "failure" + ], + "backPressureObjectThreshold": 0, + "backPressureDataSizeThreshold": "0 MB", + "flowFileExpiration": "0 sec", + "prioritizers": [ + "org.apache.nifi.prioritizer.FirstInFirstOutPrioritizer" + ], + "bends": [], + "loadBalanceStrategy": "DO_NOT_LOAD_BALANCE", + "loadBalanceCompression": "DO_NOT_COMPRESS", + "componentType": "CONNECTION", + "groupIdentifier": "f4cd72f7-b235-394b-928a-ffb0e7cf2db6" + }, + { + "identifier": "73ec2fef-9296-309c-9172-b73154996077", + "instanceIdentifier": "80138a3f-a53e-3288-93e3-5263a4a31bf2", + "name": "GenerateFlowFile/success/GZIP CompressContent", + "source": { + "id": "7b18e7a8-2208-3922-a256-a6f3581e5928", + "type": "PROCESSOR", + "groupId": "f4cd72f7-b235-394b-928a-ffb0e7cf2db6", + "name": "GenerateFlowFile", + "comments": "", + "instanceIdentifier": "8091b00c-5185-3b31-b096-82b8e301e0ab" + }, + "destination": { + "id": "9cb6c6ae-d94e-31ac-8e68-4a0aecbcbbc2", + "type": "PROCESSOR", + "groupId": "f4cd72f7-b235-394b-928a-ffb0e7cf2db6", + "name": "GZIP CompressContent", + "comments": "", + "instanceIdentifier": "0a6771cc-4527-31bb-8404-e776167bf1ed" + }, + "labelIndex": 1, + "zIndex": 0, + "selectedRelationships": [ + "success" + ], + "backPressureObjectThreshold": 0, + "backPressureDataSizeThreshold": "0 MB", + "flowFileExpiration": "0 sec", + "prioritizers": [ + "org.apache.nifi.prioritizer.FirstInFirstOutPrioritizer" + ], + "bends": [], + "loadBalanceStrategy": "DO_NOT_LOAD_BALANCE", + "loadBalanceCompression": "DO_NOT_COMPRESS", + "componentType": "CONNECTION", + "groupIdentifier": "f4cd72f7-b235-394b-928a-ffb0e7cf2db6" + } + ], + "labels": [], + "funnels": [], + "controllerServices": [], + "variables": {}, + "defaultFlowFileExpiration": "0 sec", + "defaultBackPressureObjectThreshold": 10000, + "defaultBackPressureDataSizeThreshold": "1 GB", + "componentType": "PROCESS_GROUP", + "flowFileConcurrency": "UNBOUNDED", + "flowFileOutboundPolicy": "STREAM_WHEN_AVAILABLE" + } +} \ No newline at end of file diff --git a/minifi/minifi-integration-tests/src/test/resources/standalone/v1/DecompressionCircularFlow/yml/expected.json b/minifi/minifi-integration-tests/src/test/resources/standalone/DecompressionCircularFlow/expected.json similarity index 100% rename from minifi/minifi-integration-tests/src/test/resources/standalone/v1/DecompressionCircularFlow/yml/expected.json rename to minifi/minifi-integration-tests/src/test/resources/standalone/DecompressionCircularFlow/expected.json diff --git a/minifi/minifi-integration-tests/src/test/resources/standalone/MiNiFiTailLogAttribute/MiNiFiTailLogAttribute.json b/minifi/minifi-integration-tests/src/test/resources/standalone/MiNiFiTailLogAttribute/MiNiFiTailLogAttribute.json new file mode 100644 index 0000000000..87249f8740 --- /dev/null +++ b/minifi/minifi-integration-tests/src/test/resources/standalone/MiNiFiTailLogAttribute/MiNiFiTailLogAttribute.json @@ -0,0 +1,168 @@ +{ + "encodingVersion": { + "majorVersion": 2, + "minorVersion": 0 + }, + "maxTimerDrivenThreadCount": 1, + "maxEventDrivenThreadCount": 1, + "registries": [], + "parameterContexts": [], + "parameterProviders": [], + "controllerServices": [], + "reportingTasks": [], + "templates": [], + "rootGroup": { + "identifier": "f4cd72f7-b235-394b-928a-ffb0e7cf2db6", + "instanceIdentifier": "Root-Group", + "name": "MiNiFi TailTail LogAttribute", + "comments": "", + "position": { + "x": 0.0, + "y": 0.0 + }, + "processGroups": [], + "remoteProcessGroups": [], + "processors": [ + { + "identifier": "05b31636-8fdd-35c2-a0d5-84ec755dcbff", + "instanceIdentifier": "94b8e610-b4ed-3ec9-b26f-c839931bf3e2", + "name": "TailFile", + "comments": "", + "position": { + "x": 0.0, + "y": 0.0 + }, + "type": "org.apache.nifi.processors.standard.TailFile", + "bundle": { + "group": "org.apache.nifi.minifi", + "artifact": "minifi-standard-nar", + "version": "2.0.0-SNAPSHOT" + }, + "properties": { + "tailfile-recursive-lookup": "false", + "File Location": "Local", + "Initial Start Position": "Beginning of File", + "pre-allocated-buffer-size": "65536 B", + "Post-Rollover Tail Period": "0 sec", + "reread-on-nul": "false", + "tailfile-maximum-age": "24 hours", + "Max Buffer Size": "64 KB", + "File to Tail": "./logs/minifi-app.log", + "tailfile-lookup-frequency": "10 minutes", + "tail-mode": "Single file" + }, + "propertyDescriptors": {}, + "style": {}, + "schedulingPeriod": "0 sec", + "schedulingStrategy": "TIMER_DRIVEN", + "executionNode": "ALL", + "penaltyDuration": "30 sec", + "yieldDuration": "1 sec", + "bulletinLevel": "WARN", + "runDurationMillis": 0, + "concurrentlySchedulableTaskCount": 1, + "autoTerminatedRelationships": [], + "scheduledState": "RUNNING", + "retryCount": 10, + "retriedRelationships": [], + "backoffMechanism": "PENALIZE_FLOWFILE", + "maxBackoffPeriod": "10 mins", + "componentType": "PROCESSOR", + "groupIdentifier": "f4cd72f7-b235-394b-928a-ffb0e7cf2db6" + }, + { + "identifier": "6a95ee4f-164f-31ba-84fb-7e53df2971bd", + "instanceIdentifier": "59198b81-12fc-374f-bead-7fcd1d5d4823", + "name": "LogAttribute", + "comments": "", + "position": { + "x": 0.0, + "y": 0.0 + }, + "type": "org.apache.nifi.processors.standard.LogAttribute", + "bundle": { + "group": "org.apache.nifi.minifi", + "artifact": "minifi-standard-nar", + "version": "2.0.0-SNAPSHOT" + }, + "properties": { + "character-set": "UTF-8", + "Log FlowFile Properties": "true", + "Log Level": "info", + "attributes-to-log-regex": ".*", + "Output Format": "Line per Attribute", + "Log Payload": "false" + }, + "propertyDescriptors": {}, + "style": {}, + "schedulingPeriod": "0 sec", + "schedulingStrategy": "TIMER_DRIVEN", + "executionNode": "ALL", + "penaltyDuration": "30 sec", + "yieldDuration": "1 sec", + "bulletinLevel": "WARN", + "runDurationMillis": 0, + "concurrentlySchedulableTaskCount": 1, + "autoTerminatedRelationships": [ + "success" + ], + "scheduledState": "RUNNING", + "retryCount": 10, + "retriedRelationships": [], + "backoffMechanism": "PENALIZE_FLOWFILE", + "maxBackoffPeriod": "10 mins", + "componentType": "PROCESSOR", + "groupIdentifier": "f4cd72f7-b235-394b-928a-ffb0e7cf2db6" + } + ], + "inputPorts": [], + "outputPorts": [], + "connections": [ + { + "identifier": "42ce776b-7885-360a-8930-7803d063b6c1", + "instanceIdentifier": "24697bea-f40b-32f6-8f77-821674527db7", + "name": "TailFile/success/LogAttribute", + "source": { + "id": "05b31636-8fdd-35c2-a0d5-84ec755dcbff", + "type": "PROCESSOR", + "groupId": "f4cd72f7-b235-394b-928a-ffb0e7cf2db6", + "name": "TailFile", + "comments": "", + "instanceIdentifier": "94b8e610-b4ed-3ec9-b26f-c839931bf3e2" + }, + "destination": { + "id": "6a95ee4f-164f-31ba-84fb-7e53df2971bd", + "type": "PROCESSOR", + "groupId": "f4cd72f7-b235-394b-928a-ffb0e7cf2db6", + "name": "LogAttribute", + "comments": "", + "instanceIdentifier": "59198b81-12fc-374f-bead-7fcd1d5d4823" + }, + "labelIndex": 1, + "zIndex": 0, + "selectedRelationships": [ + "success" + ], + "backPressureObjectThreshold": 10000, + "backPressureDataSizeThreshold": "1 GB", + "flowFileExpiration": "0 sec", + "prioritizers": [], + "bends": [], + "loadBalanceStrategy": "DO_NOT_LOAD_BALANCE", + "loadBalanceCompression": "DO_NOT_COMPRESS", + "componentType": "CONNECTION", + "groupIdentifier": "f4cd72f7-b235-394b-928a-ffb0e7cf2db6" + } + ], + "labels": [], + "funnels": [], + "controllerServices": [], + "variables": {}, + "defaultFlowFileExpiration": "0 sec", + "defaultBackPressureObjectThreshold": 10000, + "defaultBackPressureDataSizeThreshold": "1 GB", + "componentType": "PROCESS_GROUP", + "flowFileConcurrency": "UNBOUNDED", + "flowFileOutboundPolicy": "STREAM_WHEN_AVAILABLE" + } +} \ No newline at end of file diff --git a/minifi/minifi-integration-tests/src/test/resources/standalone/v1/MiNiFiTailLogAttribute/yml/expected.json b/minifi/minifi-integration-tests/src/test/resources/standalone/MiNiFiTailLogAttribute/expected.json similarity index 100% rename from minifi/minifi-integration-tests/src/test/resources/standalone/v1/MiNiFiTailLogAttribute/yml/expected.json rename to minifi/minifi-integration-tests/src/test/resources/standalone/MiNiFiTailLogAttribute/expected.json diff --git a/minifi/minifi-integration-tests/src/test/resources/standalone/MultipleRelationships/MultipleRelationships.json b/minifi/minifi-integration-tests/src/test/resources/standalone/MultipleRelationships/MultipleRelationships.json new file mode 100644 index 0000000000..ed2a4934cc --- /dev/null +++ b/minifi/minifi-integration-tests/src/test/resources/standalone/MultipleRelationships/MultipleRelationships.json @@ -0,0 +1,310 @@ +{ + "encodingVersion": { + "majorVersion": 2, + "minorVersion": 0 + }, + "maxTimerDrivenThreadCount": 1, + "maxEventDrivenThreadCount": 1, + "registries": [], + "parameterContexts": [], + "parameterProviders": [], + "controllerServices": [], + "reportingTasks": [], + "templates": [], + "rootGroup": { + "identifier": "f4cd72f7-b235-394b-928a-ffb0e7cf2db6", + "instanceIdentifier": "Root-Group", + "name": "MultipleRelationships", + "comments": "", + "position": { + "x": 0.0, + "y": 0.0 + }, + "processGroups": [], + "remoteProcessGroups": [], + "processors": [ + { + "identifier": "1056e96c-36cd-3dfe-a1c6-126b9da9a339", + "instanceIdentifier": "7c755ed6-0157-1000-0000-000000000000", + "name": "GenerateFlowFile", + "comments": "", + "position": { + "x": 0.0, + "y": 0.0 + }, + "type": "org.apache.nifi.processors.standard.GenerateFlowFile", + "bundle": { + "group": "org.apache.nifi.minifi", + "artifact": "minifi-standard-nar", + "version": "2.0.0-SNAPSHOT" + }, + "properties": { + "character-set": "UTF-8", + "File Size": "0 B", + "Batch Size": "1", + "Unique FlowFiles": "false", + "Data Format": "Binary" + }, + "propertyDescriptors": {}, + "style": {}, + "schedulingPeriod": "0 sec", + "schedulingStrategy": "TIMER_DRIVEN", + "executionNode": "ALL", + "penaltyDuration": "30 sec", + "yieldDuration": "1 sec", + "bulletinLevel": "WARN", + "runDurationMillis": 0, + "concurrentlySchedulableTaskCount": 1, + "autoTerminatedRelationships": [], + "scheduledState": "RUNNING", + "retryCount": 10, + "retriedRelationships": [], + "backoffMechanism": "PENALIZE_FLOWFILE", + "maxBackoffPeriod": "10 mins", + "componentType": "PROCESSOR", + "groupIdentifier": "f4cd72f7-b235-394b-928a-ffb0e7cf2db6" + }, + { + "identifier": "ec1198c9-759d-3da5-b3ea-fffe0f7439da", + "instanceIdentifier": "7c79ba25-0157-1000-0000-000000000000", + "name": "UpdateAttribute", + "comments": "", + "position": { + "x": 0.0, + "y": 0.0 + }, + "type": "org.apache.nifi.processors.attributes.UpdateAttribute", + "bundle": { + "group": "org.apache.nifi.minifi", + "artifact": "minifi-update-attribute-nar", + "version": "2.0.0-SNAPSHOT" + }, + "properties": { + "filename": "abc", + "Store State": "Do not store state", + "canonical-value-lookup-cache-size": "100" + }, + "propertyDescriptors": {}, + "style": {}, + "schedulingPeriod": "0 sec", + "schedulingStrategy": "TIMER_DRIVEN", + "executionNode": "ALL", + "penaltyDuration": "30 sec", + "yieldDuration": "1 sec", + "bulletinLevel": "WARN", + "runDurationMillis": 0, + "concurrentlySchedulableTaskCount": 1, + "autoTerminatedRelationships": [], + "scheduledState": "RUNNING", + "retryCount": 10, + "retriedRelationships": [], + "backoffMechanism": "PENALIZE_FLOWFILE", + "maxBackoffPeriod": "10 mins", + "componentType": "PROCESSOR", + "groupIdentifier": "f4cd72f7-b235-394b-928a-ffb0e7cf2db6" + }, + { + "identifier": "be82d14a-1986-3c0c-b74f-216adae00156", + "instanceIdentifier": "7c75ab71-0157-1000-0000-000000000000", + "name": "LogAttribute", + "comments": "", + "position": { + "x": 0.0, + "y": 0.0 + }, + "type": "org.apache.nifi.processors.standard.LogAttribute", + "bundle": { + "group": "org.apache.nifi.minifi", + "artifact": "minifi-standard-nar", + "version": "2.0.0-SNAPSHOT" + }, + "properties": { + "character-set": "UTF-8", + "Log FlowFile Properties": "true", + "Log Level": "info", + "attributes-to-log-regex": ".*", + "Output Format": "Line per Attribute", + "Log Payload": "false" + }, + "propertyDescriptors": {}, + "style": {}, + "schedulingPeriod": "0 sec", + "schedulingStrategy": "TIMER_DRIVEN", + "executionNode": "ALL", + "penaltyDuration": "30 sec", + "yieldDuration": "1 sec", + "bulletinLevel": "WARN", + "runDurationMillis": 0, + "concurrentlySchedulableTaskCount": 1, + "autoTerminatedRelationships": [ + "success" + ], + "scheduledState": "RUNNING", + "retryCount": 10, + "retriedRelationships": [], + "backoffMechanism": "PENALIZE_FLOWFILE", + "maxBackoffPeriod": "10 mins", + "componentType": "PROCESSOR", + "groupIdentifier": "f4cd72f7-b235-394b-928a-ffb0e7cf2db6" + }, + { + "identifier": "7c0ec2c7-bbcb-3790-a5dc-6eeae7aa23be", + "instanceIdentifier": "7c768622-0157-1000-0000-000000000000", + "name": "RouteOnAttribute", + "comments": "", + "position": { + "x": 0.0, + "y": 0.0 + }, + "type": "org.apache.nifi.processors.standard.RouteOnAttribute", + "bundle": { + "group": "org.apache.nifi.minifi", + "artifact": "minifi-standard-nar", + "version": "2.0.0-SNAPSHOT" + }, + "properties": { + "Routing Strategy": "Route to Property name", + "abc": "${filename:equals('abc')}" + }, + "propertyDescriptors": {}, + "style": {}, + "schedulingPeriod": "0 sec", + "schedulingStrategy": "TIMER_DRIVEN", + "executionNode": "ALL", + "penaltyDuration": "30 sec", + "yieldDuration": "1 sec", + "bulletinLevel": "WARN", + "runDurationMillis": 0, + "concurrentlySchedulableTaskCount": 1, + "autoTerminatedRelationships": [], + "scheduledState": "RUNNING", + "retryCount": 10, + "retriedRelationships": [], + "backoffMechanism": "PENALIZE_FLOWFILE", + "maxBackoffPeriod": "10 mins", + "componentType": "PROCESSOR", + "groupIdentifier": "f4cd72f7-b235-394b-928a-ffb0e7cf2db6" + } + ], + "inputPorts": [], + "outputPorts": [], + "connections": [ + { + "identifier": "22ab97af-8218-3f39-9d24-3dc86498502a", + "instanceIdentifier": "7c79d576-0157-1000-0000-000000000000", + "name": "UpdateAttribute/success/RouteOnAttribute", + "source": { + "id": "ec1198c9-759d-3da5-b3ea-fffe0f7439da", + "type": "PROCESSOR", + "groupId": "f4cd72f7-b235-394b-928a-ffb0e7cf2db6", + "name": "UpdateAttribute", + "comments": "", + "instanceIdentifier": "7c79ba25-0157-1000-0000-000000000000" + }, + "destination": { + "id": "7c0ec2c7-bbcb-3790-a5dc-6eeae7aa23be", + "type": "PROCESSOR", + "groupId": "f4cd72f7-b235-394b-928a-ffb0e7cf2db6", + "name": "RouteOnAttribute", + "comments": "", + "instanceIdentifier": "7c768622-0157-1000-0000-000000000000" + }, + "labelIndex": 1, + "zIndex": 0, + "selectedRelationships": [ + "success" + ], + "backPressureObjectThreshold": 10000, + "backPressureDataSizeThreshold": "1 GB", + "flowFileExpiration": "0 sec", + "prioritizers": [], + "bends": [], + "loadBalanceStrategy": "DO_NOT_LOAD_BALANCE", + "loadBalanceCompression": "DO_NOT_COMPRESS", + "componentType": "CONNECTION", + "groupIdentifier": "f4cd72f7-b235-394b-928a-ffb0e7cf2db6" + }, + { + "identifier": "91c84ff9-6ca1-30e0-8e8e-1e9cef9bba06", + "instanceIdentifier": "7c798ca3-0157-1000-0000-000000000000", + "name": "RouteOnAttribute/abc/LogAttribute", + "source": { + "id": "7c0ec2c7-bbcb-3790-a5dc-6eeae7aa23be", + "type": "PROCESSOR", + "groupId": "f4cd72f7-b235-394b-928a-ffb0e7cf2db6", + "name": "RouteOnAttribute", + "comments": "", + "instanceIdentifier": "7c768622-0157-1000-0000-000000000000" + }, + "destination": { + "id": "be82d14a-1986-3c0c-b74f-216adae00156", + "type": "PROCESSOR", + "groupId": "f4cd72f7-b235-394b-928a-ffb0e7cf2db6", + "name": "LogAttribute", + "comments": "", + "instanceIdentifier": "7c75ab71-0157-1000-0000-000000000000" + }, + "labelIndex": 1, + "zIndex": 0, + "selectedRelationships": [ + "abc", + "unmatched" + ], + "backPressureObjectThreshold": 10000, + "backPressureDataSizeThreshold": "1 GB", + "flowFileExpiration": "0 sec", + "prioritizers": [], + "bends": [], + "loadBalanceStrategy": "DO_NOT_LOAD_BALANCE", + "loadBalanceCompression": "DO_NOT_COMPRESS", + "componentType": "CONNECTION", + "groupIdentifier": "f4cd72f7-b235-394b-928a-ffb0e7cf2db6" + }, + { + "identifier": "20f99d34-e8ed-3219-835f-07132e0e48f9", + "instanceIdentifier": "7c79cce3-0157-1000-0000-000000000000", + "name": "GenerateFlowFile/success/UpdateAttribute", + "source": { + "id": "1056e96c-36cd-3dfe-a1c6-126b9da9a339", + "type": "PROCESSOR", + "groupId": "f4cd72f7-b235-394b-928a-ffb0e7cf2db6", + "name": "GenerateFlowFile", + "comments": "", + "instanceIdentifier": "7c755ed6-0157-1000-0000-000000000000" + }, + "destination": { + "id": "ec1198c9-759d-3da5-b3ea-fffe0f7439da", + "type": "PROCESSOR", + "groupId": "f4cd72f7-b235-394b-928a-ffb0e7cf2db6", + "name": "UpdateAttribute", + "comments": "", + "instanceIdentifier": "7c79ba25-0157-1000-0000-000000000000" + }, + "labelIndex": 1, + "zIndex": 0, + "selectedRelationships": [ + "success" + ], + "backPressureObjectThreshold": 10000, + "backPressureDataSizeThreshold": "1 GB", + "flowFileExpiration": "0 sec", + "prioritizers": [], + "bends": [], + "loadBalanceStrategy": "DO_NOT_LOAD_BALANCE", + "loadBalanceCompression": "DO_NOT_COMPRESS", + "componentType": "CONNECTION", + "groupIdentifier": "f4cd72f7-b235-394b-928a-ffb0e7cf2db6" + } + ], + "labels": [], + "funnels": [], + "controllerServices": [], + "variables": {}, + "defaultFlowFileExpiration": "0 sec", + "defaultBackPressureObjectThreshold": 10000, + "defaultBackPressureDataSizeThreshold": "1 GB", + "componentType": "PROCESS_GROUP", + "flowFileConcurrency": "UNBOUNDED", + "flowFileOutboundPolicy": "STREAM_WHEN_AVAILABLE" + } +} \ No newline at end of file diff --git a/minifi/minifi-integration-tests/src/test/resources/standalone/v2/MultipleRelationships/xml/expected.json b/minifi/minifi-integration-tests/src/test/resources/standalone/MultipleRelationships/expected.json similarity index 100% rename from minifi/minifi-integration-tests/src/test/resources/standalone/v2/MultipleRelationships/xml/expected.json rename to minifi/minifi-integration-tests/src/test/resources/standalone/MultipleRelationships/expected.json diff --git a/minifi/minifi-integration-tests/src/test/resources/standalone/ProcessGroups/ProcessGroups.json b/minifi/minifi-integration-tests/src/test/resources/standalone/ProcessGroups/ProcessGroups.json new file mode 100644 index 0000000000..2b44706d7d --- /dev/null +++ b/minifi/minifi-integration-tests/src/test/resources/standalone/ProcessGroups/ProcessGroups.json @@ -0,0 +1,375 @@ +{ + "encodingVersion": { + "majorVersion": 2, + "minorVersion": 0 + }, + "maxTimerDrivenThreadCount": 1, + "maxEventDrivenThreadCount": 1, + "registries": [], + "parameterContexts": [], + "parameterProviders": [], + "controllerServices": [], + "reportingTasks": [], + "templates": [], + "rootGroup": { + "identifier": "f4cd72f7-b235-394b-928a-ffb0e7cf2db6", + "instanceIdentifier": "Root-Group", + "name": "pgtest", + "comments": "", + "position": { + "x": 0.0, + "y": 0.0 + }, + "processGroups": [ + { + "identifier": "a1a9c843-c00d-3205-854f-55aa3a78d6cf", + "instanceIdentifier": "e25cbc20-0157-1000-0000-000000000000", + "name": "black box", + "comments": "", + "position": { + "x": 0.0, + "y": 0.0 + }, + "processGroups": [], + "remoteProcessGroups": [], + "processors": [ + { + "identifier": "0d2d66fb-2b9b-320d-ba49-ae0603f6ff48", + "instanceIdentifier": "e25d2588-0157-1000-0000-000000000000", + "name": "LogAttribute", + "comments": "", + "position": { + "x": 0.0, + "y": 0.0 + }, + "type": "org.apache.nifi.processors.standard.LogAttribute", + "bundle": { + "group": "org.apache.nifi.minifi", + "artifact": "minifi-standard-nar", + "version": "2.0.0-SNAPSHOT" + }, + "properties": { + "Log prefix": "INSIDE", + "character-set": "UTF-8", + "Log FlowFile Properties": "true", + "Log Level": "info", + "attributes-to-log-regex": ".*", + "Output Format": "Line per Attribute", + "Log Payload": "false" + }, + "propertyDescriptors": {}, + "style": {}, + "schedulingPeriod": "0 sec", + "schedulingStrategy": "TIMER_DRIVEN", + "executionNode": "ALL", + "penaltyDuration": "30 sec", + "yieldDuration": "1 sec", + "bulletinLevel": "WARN", + "runDurationMillis": 0, + "concurrentlySchedulableTaskCount": 1, + "autoTerminatedRelationships": [], + "scheduledState": "RUNNING", + "retryCount": 10, + "retriedRelationships": [], + "backoffMechanism": "PENALIZE_FLOWFILE", + "maxBackoffPeriod": "10 mins", + "componentType": "PROCESSOR", + "groupIdentifier": "a1a9c843-c00d-3205-854f-55aa3a78d6cf" + } + ], + "inputPorts": [ + { + "identifier": "e69c6857-2cee-3d4d-b786-48659a1b2978", + "instanceIdentifier": "e25d0cef-0157-1000-0000-000000000000", + "name": "input", + "comments": "", + "position": { + "x": 0.0, + "y": 0.0 + }, + "type": "INPUT_PORT", + "concurrentlySchedulableTaskCount": 1, + "scheduledState": "RUNNING", + "allowRemoteAccess": false, + "componentType": "INPUT_PORT", + "groupIdentifier": "a1a9c843-c00d-3205-854f-55aa3a78d6cf" + } + ], + "outputPorts": [ + { + "identifier": "f917854a-2f09-3056-9124-149a36699f74", + "instanceIdentifier": "e25d578c-0157-1000-0000-000000000000", + "name": "output", + "comments": "", + "position": { + "x": 0.0, + "y": 0.0 + }, + "type": "OUTPUT_PORT", + "concurrentlySchedulableTaskCount": 1, + "scheduledState": "RUNNING", + "allowRemoteAccess": false, + "componentType": "OUTPUT_PORT", + "groupIdentifier": "a1a9c843-c00d-3205-854f-55aa3a78d6cf" + } + ], + "connections": [ + { + "identifier": "837f3cfd-b52a-3dda-a61a-a3c079b1ece8", + "instanceIdentifier": "e25d6bd6-0157-1000-0000-000000000000", + "name": "LogAttribute/success/output", + "source": { + "id": "0d2d66fb-2b9b-320d-ba49-ae0603f6ff48", + "type": "PROCESSOR", + "groupId": "a1a9c843-c00d-3205-854f-55aa3a78d6cf", + "name": "LogAttribute", + "comments": "", + "instanceIdentifier": "e25d2588-0157-1000-0000-000000000000" + }, + "destination": { + "id": "f917854a-2f09-3056-9124-149a36699f74", + "type": "OUTPUT_PORT", + "groupId": "a1a9c843-c00d-3205-854f-55aa3a78d6cf", + "name": "output", + "comments": "", + "instanceIdentifier": "e25d578c-0157-1000-0000-000000000000" + }, + "labelIndex": 1, + "zIndex": 0, + "selectedRelationships": [ + "success" + ], + "backPressureObjectThreshold": 10000, + "backPressureDataSizeThreshold": "1 GB", + "flowFileExpiration": "0 sec", + "prioritizers": [], + "bends": [], + "loadBalanceStrategy": "DO_NOT_LOAD_BALANCE", + "loadBalanceCompression": "DO_NOT_COMPRESS", + "componentType": "CONNECTION", + "groupIdentifier": "a1a9c843-c00d-3205-854f-55aa3a78d6cf" + }, + { + "identifier": "ebeab073-928b-34bd-9e52-756ed0145973", + "instanceIdentifier": "e25d35dc-0157-1000-0000-000000000000", + "name": "input//LogAttribute", + "source": { + "id": "e69c6857-2cee-3d4d-b786-48659a1b2978", + "type": "INPUT_PORT", + "groupId": "a1a9c843-c00d-3205-854f-55aa3a78d6cf", + "name": "input", + "comments": "", + "instanceIdentifier": "e25d0cef-0157-1000-0000-000000000000" + }, + "destination": { + "id": "0d2d66fb-2b9b-320d-ba49-ae0603f6ff48", + "type": "PROCESSOR", + "groupId": "a1a9c843-c00d-3205-854f-55aa3a78d6cf", + "name": "LogAttribute", + "comments": "", + "instanceIdentifier": "e25d2588-0157-1000-0000-000000000000" + }, + "labelIndex": 1, + "zIndex": 0, + "selectedRelationships": [ + "" + ], + "backPressureObjectThreshold": 10000, + "backPressureDataSizeThreshold": "1 GB", + "flowFileExpiration": "0 sec", + "prioritizers": [], + "bends": [], + "loadBalanceStrategy": "DO_NOT_LOAD_BALANCE", + "loadBalanceCompression": "DO_NOT_COMPRESS", + "componentType": "CONNECTION", + "groupIdentifier": "a1a9c843-c00d-3205-854f-55aa3a78d6cf" + } + ], + "labels": [], + "funnels": [], + "controllerServices": [], + "variables": {}, + "defaultFlowFileExpiration": "0 sec", + "defaultBackPressureObjectThreshold": 10000, + "defaultBackPressureDataSizeThreshold": "1 GB", + "componentType": "PROCESS_GROUP", + "flowFileConcurrency": "UNBOUNDED", + "flowFileOutboundPolicy": "STREAM_WHEN_AVAILABLE", + "groupIdentifier": "f4cd72f7-b235-394b-928a-ffb0e7cf2db6" + } + ], + "remoteProcessGroups": [], + "processors": [ + { + "identifier": "95b83270-c063-32ed-9cb7-e154dedfc5b2", + "instanceIdentifier": "e25cd92a-0157-1000-0000-000000000000", + "name": "GenerateFlowFile", + "comments": "", + "position": { + "x": 0.0, + "y": 0.0 + }, + "type": "org.apache.nifi.processors.standard.GenerateFlowFile", + "bundle": { + "group": "org.apache.nifi.minifi", + "artifact": "minifi-standard-nar", + "version": "2.0.0-SNAPSHOT" + }, + "properties": { + "character-set": "UTF-8", + "File Size": "1 b", + "Batch Size": "1", + "Unique FlowFiles": "false", + "Data Format": "Binary" + }, + "propertyDescriptors": {}, + "style": {}, + "schedulingPeriod": "0 sec", + "schedulingStrategy": "TIMER_DRIVEN", + "executionNode": "ALL", + "penaltyDuration": "30 sec", + "yieldDuration": "1 sec", + "bulletinLevel": "WARN", + "runDurationMillis": 0, + "concurrentlySchedulableTaskCount": 1, + "autoTerminatedRelationships": [], + "scheduledState": "RUNNING", + "retryCount": 10, + "retriedRelationships": [], + "backoffMechanism": "PENALIZE_FLOWFILE", + "maxBackoffPeriod": "10 mins", + "componentType": "PROCESSOR", + "groupIdentifier": "f4cd72f7-b235-394b-928a-ffb0e7cf2db6" + }, + { + "identifier": "b2b62017-d96e-3804-a64c-e90473b7d748", + "instanceIdentifier": "e25e0e6e-0157-1000-0000-000000000000", + "name": "LogAttribute", + "comments": "", + "position": { + "x": 0.0, + "y": 0.0 + }, + "type": "org.apache.nifi.processors.standard.LogAttribute", + "bundle": { + "group": "org.apache.nifi.minifi", + "artifact": "minifi-standard-nar", + "version": "2.0.0-SNAPSHOT" + }, + "properties": { + "Log prefix": "OUTSIDE", + "character-set": "UTF-8", + "Log FlowFile Properties": "true", + "Log Level": "info", + "attributes-to-log-regex": ".*", + "Output Format": "Line per Attribute", + "Log Payload": "false" + }, + "propertyDescriptors": {}, + "style": {}, + "schedulingPeriod": "0 sec", + "schedulingStrategy": "TIMER_DRIVEN", + "executionNode": "ALL", + "penaltyDuration": "30 sec", + "yieldDuration": "1 sec", + "bulletinLevel": "WARN", + "runDurationMillis": 0, + "concurrentlySchedulableTaskCount": 1, + "autoTerminatedRelationships": [ + "success" + ], + "scheduledState": "RUNNING", + "retryCount": 10, + "retriedRelationships": [], + "backoffMechanism": "PENALIZE_FLOWFILE", + "maxBackoffPeriod": "10 mins", + "componentType": "PROCESSOR", + "groupIdentifier": "f4cd72f7-b235-394b-928a-ffb0e7cf2db6" + } + ], + "inputPorts": [], + "outputPorts": [], + "connections": [ + { + "identifier": "1680023b-c94e-3ce8-b6db-a61fc4726af2", + "instanceIdentifier": "e25dbff3-0157-1000-0000-000000000000", + "name": "GenerateFlowFile/success/input", + "source": { + "id": "95b83270-c063-32ed-9cb7-e154dedfc5b2", + "type": "PROCESSOR", + "groupId": "f4cd72f7-b235-394b-928a-ffb0e7cf2db6", + "name": "GenerateFlowFile", + "comments": "", + "instanceIdentifier": "e25cd92a-0157-1000-0000-000000000000" + }, + "destination": { + "id": "e69c6857-2cee-3d4d-b786-48659a1b2978", + "type": "INPUT_PORT", + "groupId": "a1a9c843-c00d-3205-854f-55aa3a78d6cf", + "name": "input", + "comments": "", + "instanceIdentifier": "e25d0cef-0157-1000-0000-000000000000" + }, + "labelIndex": 1, + "zIndex": 0, + "selectedRelationships": [ + "success" + ], + "backPressureObjectThreshold": 10000, + "backPressureDataSizeThreshold": "1 GB", + "flowFileExpiration": "0 sec", + "prioritizers": [], + "bends": [], + "loadBalanceStrategy": "DO_NOT_LOAD_BALANCE", + "loadBalanceCompression": "DO_NOT_COMPRESS", + "componentType": "CONNECTION", + "groupIdentifier": "f4cd72f7-b235-394b-928a-ffb0e7cf2db6" + }, + { + "identifier": "780119d8-eb9d-3116-b2ba-0e0dacae123f", + "instanceIdentifier": "e25e22b1-0157-1000-0000-000000000000", + "name": "output//LogAttribute", + "source": { + "id": "f917854a-2f09-3056-9124-149a36699f74", + "type": "OUTPUT_PORT", + "groupId": "a1a9c843-c00d-3205-854f-55aa3a78d6cf", + "name": "output", + "comments": "", + "instanceIdentifier": "e25d578c-0157-1000-0000-000000000000" + }, + "destination": { + "id": "b2b62017-d96e-3804-a64c-e90473b7d748", + "type": "PROCESSOR", + "groupId": "f4cd72f7-b235-394b-928a-ffb0e7cf2db6", + "name": "LogAttribute", + "comments": "", + "instanceIdentifier": "e25e0e6e-0157-1000-0000-000000000000" + }, + "labelIndex": 1, + "zIndex": 0, + "selectedRelationships": [ + "" + ], + "backPressureObjectThreshold": 10000, + "backPressureDataSizeThreshold": "1 GB", + "flowFileExpiration": "0 sec", + "prioritizers": [], + "bends": [], + "loadBalanceStrategy": "DO_NOT_LOAD_BALANCE", + "loadBalanceCompression": "DO_NOT_COMPRESS", + "componentType": "CONNECTION", + "groupIdentifier": "f4cd72f7-b235-394b-928a-ffb0e7cf2db6" + } + ], + "labels": [], + "funnels": [], + "controllerServices": [], + "variables": {}, + "defaultFlowFileExpiration": "0 sec", + "defaultBackPressureObjectThreshold": 10000, + "defaultBackPressureDataSizeThreshold": "1 GB", + "componentType": "PROCESS_GROUP", + "flowFileConcurrency": "UNBOUNDED", + "flowFileOutboundPolicy": "STREAM_WHEN_AVAILABLE" + } +} \ No newline at end of file diff --git a/minifi/minifi-integration-tests/src/test/resources/standalone/v2/ProcessGroups/xml/expected.json b/minifi/minifi-integration-tests/src/test/resources/standalone/ProcessGroups/expected.json similarity index 100% rename from minifi/minifi-integration-tests/src/test/resources/standalone/v2/ProcessGroups/xml/expected.json rename to minifi/minifi-integration-tests/src/test/resources/standalone/ProcessGroups/expected.json diff --git a/minifi/minifi-integration-tests/src/test/resources/standalone/ReplaceTextExpressionLanguageCSVReformatting/ReplaceTextExpressionLanguageCSVReformatting.json b/minifi/minifi-integration-tests/src/test/resources/standalone/ReplaceTextExpressionLanguageCSVReformatting/ReplaceTextExpressionLanguageCSVReformatting.json new file mode 100644 index 0000000000..8647e34e68 --- /dev/null +++ b/minifi/minifi-integration-tests/src/test/resources/standalone/ReplaceTextExpressionLanguageCSVReformatting/ReplaceTextExpressionLanguageCSVReformatting.json @@ -0,0 +1,324 @@ +{ + "encodingVersion": { + "majorVersion": 2, + "minorVersion": 0 + }, + "maxTimerDrivenThreadCount": 1, + "maxEventDrivenThreadCount": 1, + "registries": [], + "parameterContexts": [], + "parameterProviders": [], + "controllerServices": [], + "reportingTasks": [], + "templates": [], + "rootGroup": { + "identifier": "f4cd72f7-b235-394b-928a-ffb0e7cf2db6", + "instanceIdentifier": "Root-Group", + "name": "ReplaceTextExpressionLanguageCSVReformatting", + "comments": "", + "position": { + "x": 0.0, + "y": 0.0 + }, + "processGroups": [], + "remoteProcessGroups": [], + "processors": [ + { + "identifier": "a72307a6-334b-3f99-9c2d-5afb7b5586df", + "instanceIdentifier": "65eff572-176c-3c6b-87d8-a998fc4c84fc", + "name": "Generate Empty File", + "comments": "", + "position": { + "x": 0.0, + "y": 0.0 + }, + "type": "org.apache.nifi.processors.standard.GenerateFlowFile", + "bundle": { + "group": "org.apache.nifi.minifi", + "artifact": "minifi-standard-nar", + "version": "2.0.0-SNAPSHOT" + }, + "properties": { + "character-set": "UTF-8", + "File Size": "0b", + "Batch Size": "1", + "Unique FlowFiles": "false", + "Data Format": "Text" + }, + "propertyDescriptors": {}, + "style": {}, + "schedulingPeriod": "1 sec", + "schedulingStrategy": "TIMER_DRIVEN", + "executionNode": "ALL", + "penaltyDuration": "30 sec", + "yieldDuration": "1 sec", + "bulletinLevel": "WARN", + "runDurationMillis": 0, + "concurrentlySchedulableTaskCount": 1, + "autoTerminatedRelationships": [], + "scheduledState": "RUNNING", + "retryCount": 10, + "retriedRelationships": [], + "backoffMechanism": "PENALIZE_FLOWFILE", + "maxBackoffPeriod": "10 mins", + "componentType": "PROCESSOR", + "groupIdentifier": "f4cd72f7-b235-394b-928a-ffb0e7cf2db6" + }, + { + "identifier": "743f266d-082c-3bbb-b457-a5eb6940d929", + "instanceIdentifier": "d123f462-2b54-39b8-a717-e394fb09e040", + "name": "Reformat Date Column", + "comments": "", + "position": { + "x": 0.0, + "y": 0.0 + }, + "type": "org.apache.nifi.processors.standard.ReplaceText", + "bundle": { + "group": "org.apache.nifi.minifi", + "artifact": "minifi-standard-nar", + "version": "2.0.0-SNAPSHOT" + }, + "properties": { + "Regular Expression": "(.*?),(.*?),(\\d+.*)", + "Replacement Value": "$1,$2,${ '$3':toDate('ddMMMyyyy'):format('yyyy/MM/dd') }", + "Evaluation Mode": "Line-by-Line", + "Line-by-Line Evaluation Mode": "All", + "Character Set": "UTF-8", + "Maximum Buffer Size": "1 MB", + "Replacement Strategy": "Regex Replace" + }, + "propertyDescriptors": {}, + "style": {}, + "schedulingPeriod": "0 sec", + "schedulingStrategy": "TIMER_DRIVEN", + "executionNode": "ALL", + "penaltyDuration": "30 sec", + "yieldDuration": "1 sec", + "bulletinLevel": "WARN", + "runDurationMillis": 0, + "concurrentlySchedulableTaskCount": 1, + "autoTerminatedRelationships": [ + "failure" + ], + "scheduledState": "RUNNING", + "retryCount": 10, + "retriedRelationships": [], + "backoffMechanism": "PENALIZE_FLOWFILE", + "maxBackoffPeriod": "10 mins", + "componentType": "PROCESSOR", + "groupIdentifier": "f4cd72f7-b235-394b-928a-ffb0e7cf2db6" + }, + { + "identifier": "d3d76b70-a28c-31ea-8336-1ffa4b982615", + "instanceIdentifier": "8c6c54be-7db6-333f-8c3b-d6e7b02411ff", + "name": "No-Op Termination", + "comments": "", + "position": { + "x": 0.0, + "y": 0.0 + }, + "type": "org.apache.nifi.processors.attributes.UpdateAttribute", + "bundle": { + "group": "org.apache.nifi.minifi", + "artifact": "minifi-update-attribute-nar", + "version": "2.0.0-SNAPSHOT" + }, + "properties": { + "Store State": "Do not store state", + "canonical-value-lookup-cache-size": "100" + }, + "propertyDescriptors": {}, + "style": {}, + "schedulingPeriod": "0 sec", + "schedulingStrategy": "TIMER_DRIVEN", + "executionNode": "ALL", + "penaltyDuration": "30 sec", + "yieldDuration": "1 sec", + "bulletinLevel": "WARN", + "runDurationMillis": 0, + "concurrentlySchedulableTaskCount": 1, + "autoTerminatedRelationships": [ + "success" + ], + "scheduledState": "RUNNING", + "retryCount": 10, + "retriedRelationships": [], + "backoffMechanism": "PENALIZE_FLOWFILE", + "maxBackoffPeriod": "10 mins", + "componentType": "PROCESSOR", + "groupIdentifier": "f4cd72f7-b235-394b-928a-ffb0e7cf2db6" + }, + { + "identifier": "77d72911-c2d7-3c59-bef9-1cb83402d671", + "instanceIdentifier": "b1df6002-f956-30e0-aa23-d433eb8dbe2f", + "name": "Set CSV Content", + "comments": "", + "position": { + "x": 0.0, + "y": 0.0 + }, + "type": "org.apache.nifi.processors.standard.ReplaceText", + "bundle": { + "group": "org.apache.nifi.minifi", + "artifact": "minifi-standard-nar", + "version": "2.0.0-SNAPSHOT" + }, + "properties": { + "Regular Expression": "(?s)(^.*$)", + "Replacement Value": "2006,10-01-2004,10may2004\n2007,15-05-2006,10jun2005\n2009,8-8-2008,10aug2008", + "Evaluation Mode": "Line-by-Line", + "Line-by-Line Evaluation Mode": "All", + "Character Set": "UTF-8", + "Maximum Buffer Size": "1 MB", + "Replacement Strategy": "Regex Replace" + }, + "propertyDescriptors": {}, + "style": {}, + "schedulingPeriod": "0 sec", + "schedulingStrategy": "TIMER_DRIVEN", + "executionNode": "ALL", + "penaltyDuration": "30 sec", + "yieldDuration": "1 sec", + "bulletinLevel": "WARN", + "runDurationMillis": 0, + "concurrentlySchedulableTaskCount": 1, + "autoTerminatedRelationships": [ + "failure" + ], + "scheduledState": "RUNNING", + "retryCount": 10, + "retriedRelationships": [], + "backoffMechanism": "PENALIZE_FLOWFILE", + "maxBackoffPeriod": "10 mins", + "componentType": "PROCESSOR", + "groupIdentifier": "f4cd72f7-b235-394b-928a-ffb0e7cf2db6" + } + ], + "inputPorts": [], + "outputPorts": [], + "connections": [ + { + "identifier": "a20347bd-fd67-3add-b5d2-8e9afb1837de", + "instanceIdentifier": "b2314dba-819a-31c9-ae88-6334102062e2", + "name": "Generate Empty File/success/Set CSV Content", + "source": { + "id": "a72307a6-334b-3f99-9c2d-5afb7b5586df", + "type": "PROCESSOR", + "groupId": "f4cd72f7-b235-394b-928a-ffb0e7cf2db6", + "name": "Generate Empty File", + "comments": "", + "instanceIdentifier": "65eff572-176c-3c6b-87d8-a998fc4c84fc" + }, + "destination": { + "id": "77d72911-c2d7-3c59-bef9-1cb83402d671", + "type": "PROCESSOR", + "groupId": "f4cd72f7-b235-394b-928a-ffb0e7cf2db6", + "name": "Set CSV Content", + "comments": "", + "instanceIdentifier": "b1df6002-f956-30e0-aa23-d433eb8dbe2f" + }, + "labelIndex": 1, + "zIndex": 0, + "selectedRelationships": [ + "success" + ], + "backPressureObjectThreshold": 0, + "backPressureDataSizeThreshold": "0 MB", + "flowFileExpiration": "0 sec", + "prioritizers": [ + "org.apache.nifi.prioritizer.FirstInFirstOutPrioritizer" + ], + "bends": [], + "loadBalanceStrategy": "DO_NOT_LOAD_BALANCE", + "loadBalanceCompression": "DO_NOT_COMPRESS", + "componentType": "CONNECTION", + "groupIdentifier": "f4cd72f7-b235-394b-928a-ffb0e7cf2db6" + }, + { + "identifier": "8448d199-a002-3c9c-bad1-ab7fc8ec15dc", + "instanceIdentifier": "d6fa1548-86ff-367a-b37a-4202543ad298", + "name": "Set CSV Content/success/Reformat Date Column", + "source": { + "id": "77d72911-c2d7-3c59-bef9-1cb83402d671", + "type": "PROCESSOR", + "groupId": "f4cd72f7-b235-394b-928a-ffb0e7cf2db6", + "name": "Set CSV Content", + "comments": "", + "instanceIdentifier": "b1df6002-f956-30e0-aa23-d433eb8dbe2f" + }, + "destination": { + "id": "743f266d-082c-3bbb-b457-a5eb6940d929", + "type": "PROCESSOR", + "groupId": "f4cd72f7-b235-394b-928a-ffb0e7cf2db6", + "name": "Reformat Date Column", + "comments": "", + "instanceIdentifier": "d123f462-2b54-39b8-a717-e394fb09e040" + }, + "labelIndex": 1, + "zIndex": 0, + "selectedRelationships": [ + "success" + ], + "backPressureObjectThreshold": 0, + "backPressureDataSizeThreshold": "0 MB", + "flowFileExpiration": "0 sec", + "prioritizers": [ + "org.apache.nifi.prioritizer.FirstInFirstOutPrioritizer" + ], + "bends": [], + "loadBalanceStrategy": "DO_NOT_LOAD_BALANCE", + "loadBalanceCompression": "DO_NOT_COMPRESS", + "componentType": "CONNECTION", + "groupIdentifier": "f4cd72f7-b235-394b-928a-ffb0e7cf2db6" + }, + { + "identifier": "90c17470-19e0-312d-8b30-fc3013705fa1", + "instanceIdentifier": "9c98948a-3ff1-344c-b890-2230c8008079", + "name": "Reformat Date Column/success/No-Op Termination", + "source": { + "id": "743f266d-082c-3bbb-b457-a5eb6940d929", + "type": "PROCESSOR", + "groupId": "f4cd72f7-b235-394b-928a-ffb0e7cf2db6", + "name": "Reformat Date Column", + "comments": "", + "instanceIdentifier": "d123f462-2b54-39b8-a717-e394fb09e040" + }, + "destination": { + "id": "d3d76b70-a28c-31ea-8336-1ffa4b982615", + "type": "PROCESSOR", + "groupId": "f4cd72f7-b235-394b-928a-ffb0e7cf2db6", + "name": "No-Op Termination", + "comments": "", + "instanceIdentifier": "8c6c54be-7db6-333f-8c3b-d6e7b02411ff" + }, + "labelIndex": 1, + "zIndex": 0, + "selectedRelationships": [ + "success" + ], + "backPressureObjectThreshold": 0, + "backPressureDataSizeThreshold": "0 MB", + "flowFileExpiration": "0 sec", + "prioritizers": [ + "org.apache.nifi.prioritizer.FirstInFirstOutPrioritizer" + ], + "bends": [], + "loadBalanceStrategy": "DO_NOT_LOAD_BALANCE", + "loadBalanceCompression": "DO_NOT_COMPRESS", + "componentType": "CONNECTION", + "groupIdentifier": "f4cd72f7-b235-394b-928a-ffb0e7cf2db6" + } + ], + "labels": [], + "funnels": [], + "controllerServices": [], + "variables": {}, + "defaultFlowFileExpiration": "0 sec", + "defaultBackPressureObjectThreshold": 10000, + "defaultBackPressureDataSizeThreshold": "1 GB", + "componentType": "PROCESS_GROUP", + "flowFileConcurrency": "UNBOUNDED", + "flowFileOutboundPolicy": "STREAM_WHEN_AVAILABLE" + } +} \ No newline at end of file diff --git a/minifi/minifi-integration-tests/src/test/resources/standalone/v1/ReplaceTextExpressionLanguageCSVReformatting/yml/expected.json b/minifi/minifi-integration-tests/src/test/resources/standalone/ReplaceTextExpressionLanguageCSVReformatting/expected.json similarity index 100% rename from minifi/minifi-integration-tests/src/test/resources/standalone/v1/ReplaceTextExpressionLanguageCSVReformatting/yml/expected.json rename to minifi/minifi-integration-tests/src/test/resources/standalone/ReplaceTextExpressionLanguageCSVReformatting/expected.json diff --git a/minifi/minifi-integration-tests/src/test/resources/standalone/StressTestFramework/StressTestFramework.json b/minifi/minifi-integration-tests/src/test/resources/standalone/StressTestFramework/StressTestFramework.json new file mode 100644 index 0000000000..08f7728d8d --- /dev/null +++ b/minifi/minifi-integration-tests/src/test/resources/standalone/StressTestFramework/StressTestFramework.json @@ -0,0 +1,508 @@ +{ + "encodingVersion": { + "majorVersion": 2, + "minorVersion": 0 + }, + "maxTimerDrivenThreadCount": 1, + "maxEventDrivenThreadCount": 1, + "registries": [], + "parameterContexts": [], + "parameterProviders": [], + "controllerServices": [], + "reportingTasks": [], + "templates": [], + "rootGroup": { + "identifier": "f4cd72f7-b235-394b-928a-ffb0e7cf2db6", + "instanceIdentifier": "Root-Group", + "name": "StressTestFramework", + "comments": "", + "position": { + "x": 0.0, + "y": 0.0 + }, + "processGroups": [], + "remoteProcessGroups": [], + "processors": [ + { + "identifier": "db0d54bd-55c3-383e-9481-c0d4ff976397", + "instanceIdentifier": "16a47794-5391-4ad2-0000-000000000000", + "name": "GenerateFlowFile", + "comments": "", + "position": { + "x": 0.0, + "y": 0.0 + }, + "type": "org.apache.nifi.processors.standard.GenerateFlowFile", + "bundle": { + "group": "org.apache.nifi.minifi", + "artifact": "minifi-standard-nar", + "version": "2.0.0-SNAPSHOT" + }, + "properties": { + "character-set": "UTF-8", + "File Size": "0 KB", + "Batch Size": "100", + "Unique FlowFiles": "false", + "Data Format": "Binary" + }, + "propertyDescriptors": {}, + "style": {}, + "schedulingPeriod": "0 sec", + "schedulingStrategy": "TIMER_DRIVEN", + "executionNode": "ALL", + "penaltyDuration": "30 sec", + "yieldDuration": "1 sec", + "bulletinLevel": "WARN", + "runDurationMillis": 0, + "concurrentlySchedulableTaskCount": 1, + "autoTerminatedRelationships": [], + "scheduledState": "RUNNING", + "retryCount": 10, + "retriedRelationships": [], + "backoffMechanism": "PENALIZE_FLOWFILE", + "maxBackoffPeriod": "10 mins", + "componentType": "PROCESSOR", + "groupIdentifier": "f4cd72f7-b235-394b-928a-ffb0e7cf2db6" + }, + { + "identifier": "b932fab7-11ae-3039-ba6f-1bd00155b033", + "instanceIdentifier": "397a4910-cc01-4c6b-0000-000000000000", + "name": "RouteOnAttribute", + "comments": "", + "position": { + "x": 0.0, + "y": 0.0 + }, + "type": "org.apache.nifi.processors.standard.RouteOnAttribute", + "bundle": { + "group": "org.apache.nifi.minifi", + "artifact": "minifi-standard-nar", + "version": "2.0.0-SNAPSHOT" + }, + "properties": { + "Routing Strategy": "Route to Property name" + }, + "propertyDescriptors": {}, + "style": {}, + "schedulingPeriod": "0 sec", + "schedulingStrategy": "TIMER_DRIVEN", + "executionNode": "ALL", + "penaltyDuration": "30 sec", + "yieldDuration": "1 sec", + "bulletinLevel": "WARN", + "runDurationMillis": 0, + "concurrentlySchedulableTaskCount": 2, + "autoTerminatedRelationships": [ + "unmatched" + ], + "scheduledState": "RUNNING", + "retryCount": 10, + "retriedRelationships": [], + "backoffMechanism": "PENALIZE_FLOWFILE", + "maxBackoffPeriod": "10 mins", + "componentType": "PROCESSOR", + "groupIdentifier": "f4cd72f7-b235-394b-928a-ffb0e7cf2db6" + }, + { + "identifier": "76f39e00-74fc-3ca9-816c-4ebb8caf72b8", + "instanceIdentifier": "4ad21391-16a4-1794-0000-000000000000", + "name": "GenerateFlowFile", + "comments": "", + "position": { + "x": 0.0, + "y": 0.0 + }, + "type": "org.apache.nifi.processors.standard.GenerateFlowFile", + "bundle": { + "group": "org.apache.nifi.minifi", + "artifact": "minifi-standard-nar", + "version": "2.0.0-SNAPSHOT" + }, + "properties": { + "character-set": "UTF-8", + "File Size": "0 KB", + "Batch Size": "100", + "Unique FlowFiles": "false", + "Data Format": "Binary" + }, + "propertyDescriptors": {}, + "style": {}, + "schedulingPeriod": "0 sec", + "schedulingStrategy": "TIMER_DRIVEN", + "executionNode": "ALL", + "penaltyDuration": "30 sec", + "yieldDuration": "1 sec", + "bulletinLevel": "WARN", + "runDurationMillis": 0, + "concurrentlySchedulableTaskCount": 1, + "autoTerminatedRelationships": [], + "scheduledState": "RUNNING", + "retryCount": 10, + "retriedRelationships": [], + "backoffMechanism": "PENALIZE_FLOWFILE", + "maxBackoffPeriod": "10 mins", + "componentType": "PROCESSOR", + "groupIdentifier": "f4cd72f7-b235-394b-928a-ffb0e7cf2db6" + }, + { + "identifier": "6869a309-fe8e-3629-81f0-bb15559628fe", + "instanceIdentifier": "92557c76-f251-45a4-0000-000000000000", + "name": "UpdateAttribute", + "comments": "", + "position": { + "x": 0.0, + "y": 0.0 + }, + "type": "org.apache.nifi.processors.attributes.UpdateAttribute", + "bundle": { + "group": "org.apache.nifi.minifi", + "artifact": "minifi-update-attribute-nar", + "version": "2.0.0-SNAPSHOT" + }, + "properties": { + "Store State": "Do not store state", + "canonical-value-lookup-cache-size": "100", + "property 2": "value 2 ${nextInt()}", + "property 1": "value 1" + }, + "propertyDescriptors": {}, + "style": {}, + "schedulingPeriod": "0 sec", + "schedulingStrategy": "TIMER_DRIVEN", + "executionNode": "ALL", + "penaltyDuration": "30 sec", + "yieldDuration": "1 sec", + "bulletinLevel": "WARN", + "runDurationMillis": 0, + "concurrentlySchedulableTaskCount": 2, + "autoTerminatedRelationships": [], + "scheduledState": "RUNNING", + "retryCount": 10, + "retriedRelationships": [], + "backoffMechanism": "PENALIZE_FLOWFILE", + "maxBackoffPeriod": "10 mins", + "componentType": "PROCESSOR", + "groupIdentifier": "f4cd72f7-b235-394b-928a-ffb0e7cf2db6" + }, + { + "identifier": "94c991ec-cf01-3f75-8cf8-9124bf9a5a2e", + "instanceIdentifier": "53914ad2-7794-16a4-0000-000000000000", + "name": "GenerateFlowFile", + "comments": "", + "position": { + "x": 0.0, + "y": 0.0 + }, + "type": "org.apache.nifi.processors.standard.GenerateFlowFile", + "bundle": { + "group": "org.apache.nifi.minifi", + "artifact": "minifi-standard-nar", + "version": "2.0.0-SNAPSHOT" + }, + "properties": { + "character-set": "UTF-8", + "File Size": "0 KB", + "Batch Size": "100", + "Unique FlowFiles": "false", + "Data Format": "Binary" + }, + "propertyDescriptors": {}, + "style": {}, + "schedulingPeriod": "0 sec", + "schedulingStrategy": "TIMER_DRIVEN", + "executionNode": "ALL", + "penaltyDuration": "30 sec", + "yieldDuration": "1 sec", + "bulletinLevel": "WARN", + "runDurationMillis": 0, + "concurrentlySchedulableTaskCount": 1, + "autoTerminatedRelationships": [], + "scheduledState": "RUNNING", + "retryCount": 10, + "retriedRelationships": [], + "backoffMechanism": "PENALIZE_FLOWFILE", + "maxBackoffPeriod": "10 mins", + "componentType": "PROCESSOR", + "groupIdentifier": "f4cd72f7-b235-394b-928a-ffb0e7cf2db6" + }, + { + "identifier": "d4be76a7-d1c7-3117-8163-d0836df83ad4", + "instanceIdentifier": "779416a4-4ad2-1391-0000-000000000000", + "name": "GenerateFlowFile", + "comments": "", + "position": { + "x": 0.0, + "y": 0.0 + }, + "type": "org.apache.nifi.processors.standard.GenerateFlowFile", + "bundle": { + "group": "org.apache.nifi.minifi", + "artifact": "minifi-standard-nar", + "version": "2.0.0-SNAPSHOT" + }, + "properties": { + "character-set": "UTF-8", + "File Size": "0 KB", + "Batch Size": "100", + "Unique FlowFiles": "false", + "Data Format": "Binary" + }, + "propertyDescriptors": {}, + "style": {}, + "schedulingPeriod": "0 sec", + "schedulingStrategy": "TIMER_DRIVEN", + "executionNode": "ALL", + "penaltyDuration": "30 sec", + "yieldDuration": "1 sec", + "bulletinLevel": "WARN", + "runDurationMillis": 0, + "concurrentlySchedulableTaskCount": 1, + "autoTerminatedRelationships": [], + "scheduledState": "RUNNING", + "retryCount": 10, + "retriedRelationships": [], + "backoffMechanism": "PENALIZE_FLOWFILE", + "maxBackoffPeriod": "10 mins", + "componentType": "PROCESSOR", + "groupIdentifier": "f4cd72f7-b235-394b-928a-ffb0e7cf2db6" + } + ], + "inputPorts": [], + "outputPorts": [], + "connections": [ + { + "identifier": "f8c12557-c197-3114-88f1-cb970067baf7", + "instanceIdentifier": "4ad21393-16a4-1794-0000-000000000000", + "name": "GenerateFlowFile/success/4ad21392-16a4-1794-0000-000000000000", + "source": { + "id": "db0d54bd-55c3-383e-9481-c0d4ff976397", + "type": "PROCESSOR", + "groupId": "f4cd72f7-b235-394b-928a-ffb0e7cf2db6", + "name": "GenerateFlowFile", + "comments": "", + "instanceIdentifier": "16a47794-5391-4ad2-0000-000000000000" + }, + "destination": { + "id": "949f0492-b6ea-3344-861e-d42ebd947b8e", + "type": "FUNNEL", + "groupId": "f4cd72f7-b235-394b-928a-ffb0e7cf2db6", + "name": "Funnel", + "comments": "", + "instanceIdentifier": "4ad21392-16a4-1794-0000-000000000000" + }, + "labelIndex": 1, + "zIndex": 0, + "selectedRelationships": [ + "success" + ], + "backPressureObjectThreshold": 10000, + "backPressureDataSizeThreshold": "1 GB", + "flowFileExpiration": "0 sec", + "prioritizers": [], + "bends": [], + "loadBalanceStrategy": "DO_NOT_LOAD_BALANCE", + "loadBalanceCompression": "DO_NOT_COMPRESS", + "componentType": "CONNECTION", + "groupIdentifier": "f4cd72f7-b235-394b-928a-ffb0e7cf2db6" + }, + { + "identifier": "406f8473-3462-34a7-be70-2ecca5604e63", + "instanceIdentifier": "4c53556e-eb46-458c-0000-000000000000", + "name": "UpdateAttribute/success/RouteOnAttribute", + "source": { + "id": "6869a309-fe8e-3629-81f0-bb15559628fe", + "type": "PROCESSOR", + "groupId": "f4cd72f7-b235-394b-928a-ffb0e7cf2db6", + "name": "UpdateAttribute", + "comments": "", + "instanceIdentifier": "92557c76-f251-45a4-0000-000000000000" + }, + "destination": { + "id": "b932fab7-11ae-3039-ba6f-1bd00155b033", + "type": "PROCESSOR", + "groupId": "f4cd72f7-b235-394b-928a-ffb0e7cf2db6", + "name": "RouteOnAttribute", + "comments": "", + "instanceIdentifier": "397a4910-cc01-4c6b-0000-000000000000" + }, + "labelIndex": 1, + "zIndex": 0, + "selectedRelationships": [ + "success" + ], + "backPressureObjectThreshold": 2000, + "backPressureDataSizeThreshold": "0 MB", + "flowFileExpiration": "0 sec", + "prioritizers": [ + "org.apache.nifi.prioritizer.FirstInFirstOutPrioritizer" + ], + "bends": [], + "loadBalanceStrategy": "DO_NOT_LOAD_BALANCE", + "loadBalanceCompression": "DO_NOT_COMPRESS", + "componentType": "CONNECTION", + "groupIdentifier": "f4cd72f7-b235-394b-928a-ffb0e7cf2db6" + }, + { + "identifier": "46aff3e2-5378-3b6c-b21e-0ee7e9545ab3", + "instanceIdentifier": "4ad21397-16a4-1794-0000-000000000000", + "name": "4ad21392-16a4-1794-0000-000000000000//UpdateAttribute", + "source": { + "id": "949f0492-b6ea-3344-861e-d42ebd947b8e", + "type": "FUNNEL", + "groupId": "f4cd72f7-b235-394b-928a-ffb0e7cf2db6", + "name": "Funnel", + "comments": "", + "instanceIdentifier": "4ad21392-16a4-1794-0000-000000000000" + }, + "destination": { + "id": "6869a309-fe8e-3629-81f0-bb15559628fe", + "type": "PROCESSOR", + "groupId": "f4cd72f7-b235-394b-928a-ffb0e7cf2db6", + "name": "UpdateAttribute", + "comments": "", + "instanceIdentifier": "92557c76-f251-45a4-0000-000000000000" + }, + "labelIndex": 1, + "zIndex": 0, + "selectedRelationships": [ + "" + ], + "backPressureObjectThreshold": 10000, + "backPressureDataSizeThreshold": "1 GB", + "flowFileExpiration": "0 sec", + "prioritizers": [], + "bends": [], + "loadBalanceStrategy": "DO_NOT_LOAD_BALANCE", + "loadBalanceCompression": "DO_NOT_COMPRESS", + "componentType": "CONNECTION", + "groupIdentifier": "f4cd72f7-b235-394b-928a-ffb0e7cf2db6" + }, + { + "identifier": "7c6c4095-8cdd-3e24-af76-fa77dabbf134", + "instanceIdentifier": "4ad21395-16a4-1794-0000-000000000000", + "name": "GenerateFlowFile/success/4ad21392-16a4-1794-0000-000000000000", + "source": { + "id": "d4be76a7-d1c7-3117-8163-d0836df83ad4", + "type": "PROCESSOR", + "groupId": "f4cd72f7-b235-394b-928a-ffb0e7cf2db6", + "name": "GenerateFlowFile", + "comments": "", + "instanceIdentifier": "779416a4-4ad2-1391-0000-000000000000" + }, + "destination": { + "id": "949f0492-b6ea-3344-861e-d42ebd947b8e", + "type": "FUNNEL", + "groupId": "f4cd72f7-b235-394b-928a-ffb0e7cf2db6", + "name": "Funnel", + "comments": "", + "instanceIdentifier": "4ad21392-16a4-1794-0000-000000000000" + }, + "labelIndex": 1, + "zIndex": 0, + "selectedRelationships": [ + "success" + ], + "backPressureObjectThreshold": 10000, + "backPressureDataSizeThreshold": "1 GB", + "flowFileExpiration": "0 sec", + "prioritizers": [], + "bends": [], + "loadBalanceStrategy": "DO_NOT_LOAD_BALANCE", + "loadBalanceCompression": "DO_NOT_COMPRESS", + "componentType": "CONNECTION", + "groupIdentifier": "f4cd72f7-b235-394b-928a-ffb0e7cf2db6" + }, + { + "identifier": "d6086ea1-3226-3b04-b242-10a4ad422732", + "instanceIdentifier": "4ad21394-16a4-1794-0000-000000000000", + "name": "GenerateFlowFile/success/4ad21392-16a4-1794-0000-000000000000", + "source": { + "id": "94c991ec-cf01-3f75-8cf8-9124bf9a5a2e", + "type": "PROCESSOR", + "groupId": "f4cd72f7-b235-394b-928a-ffb0e7cf2db6", + "name": "GenerateFlowFile", + "comments": "", + "instanceIdentifier": "53914ad2-7794-16a4-0000-000000000000" + }, + "destination": { + "id": "949f0492-b6ea-3344-861e-d42ebd947b8e", + "type": "FUNNEL", + "groupId": "f4cd72f7-b235-394b-928a-ffb0e7cf2db6", + "name": "Funnel", + "comments": "", + "instanceIdentifier": "4ad21392-16a4-1794-0000-000000000000" + }, + "labelIndex": 1, + "zIndex": 0, + "selectedRelationships": [ + "success" + ], + "backPressureObjectThreshold": 10000, + "backPressureDataSizeThreshold": "1 GB", + "flowFileExpiration": "0 sec", + "prioritizers": [], + "bends": [], + "loadBalanceStrategy": "DO_NOT_LOAD_BALANCE", + "loadBalanceCompression": "DO_NOT_COMPRESS", + "componentType": "CONNECTION", + "groupIdentifier": "f4cd72f7-b235-394b-928a-ffb0e7cf2db6" + }, + { + "identifier": "7a74b780-30fd-342f-8c6a-1e31c7ebd6fe", + "instanceIdentifier": "4ad21396-16a4-1794-0000-000000000000", + "name": "GenerateFlowFile/success/4ad21392-16a4-1794-0000-000000000000", + "source": { + "id": "76f39e00-74fc-3ca9-816c-4ebb8caf72b8", + "type": "PROCESSOR", + "groupId": "f4cd72f7-b235-394b-928a-ffb0e7cf2db6", + "name": "GenerateFlowFile", + "comments": "", + "instanceIdentifier": "4ad21391-16a4-1794-0000-000000000000" + }, + "destination": { + "id": "949f0492-b6ea-3344-861e-d42ebd947b8e", + "type": "FUNNEL", + "groupId": "f4cd72f7-b235-394b-928a-ffb0e7cf2db6", + "name": "Funnel", + "comments": "", + "instanceIdentifier": "4ad21392-16a4-1794-0000-000000000000" + }, + "labelIndex": 1, + "zIndex": 0, + "selectedRelationships": [ + "success" + ], + "backPressureObjectThreshold": 10000, + "backPressureDataSizeThreshold": "1 GB", + "flowFileExpiration": "0 sec", + "prioritizers": [], + "bends": [], + "loadBalanceStrategy": "DO_NOT_LOAD_BALANCE", + "loadBalanceCompression": "DO_NOT_COMPRESS", + "componentType": "CONNECTION", + "groupIdentifier": "f4cd72f7-b235-394b-928a-ffb0e7cf2db6" + } + ], + "labels": [], + "funnels": [ + { + "identifier": "949f0492-b6ea-3344-861e-d42ebd947b8e", + "instanceIdentifier": "4ad21392-16a4-1794-0000-000000000000", + "position": { + "x": 0.0, + "y": 0.0 + }, + "componentType": "FUNNEL", + "groupIdentifier": "f4cd72f7-b235-394b-928a-ffb0e7cf2db6" + } + ], + "controllerServices": [], + "variables": {}, + "defaultFlowFileExpiration": "0 sec", + "defaultBackPressureObjectThreshold": 10000, + "defaultBackPressureDataSizeThreshold": "1 GB", + "componentType": "PROCESS_GROUP", + "flowFileConcurrency": "UNBOUNDED", + "flowFileOutboundPolicy": "STREAM_WHEN_AVAILABLE" + } +} \ No newline at end of file diff --git a/minifi/minifi-integration-tests/src/test/resources/standalone/v2/StressTestFramework/xml/expected.json b/minifi/minifi-integration-tests/src/test/resources/standalone/StressTestFramework/expected.json similarity index 100% rename from minifi/minifi-integration-tests/src/test/resources/standalone/v2/StressTestFramework/xml/expected.json rename to minifi/minifi-integration-tests/src/test/resources/standalone/StressTestFramework/expected.json diff --git a/minifi/minifi-integration-tests/src/test/resources/standalone/v1/CsvToJson/xml/CsvToJson.xml b/minifi/minifi-integration-tests/src/test/resources/standalone/v1/CsvToJson/xml/CsvToJson.xml deleted file mode 100644 index 60f52960fa..0000000000 --- a/minifi/minifi-integration-tests/src/test/resources/standalone/v1/CsvToJson/xml/CsvToJson.xml +++ /dev/null @@ -1,400 +0,0 @@ - - - \ No newline at end of file diff --git a/minifi/minifi-integration-tests/src/test/resources/standalone/v2/MultipleRelationships/yml/MultipleRelationships.yml b/minifi/minifi-integration-tests/src/test/resources/standalone/v2/MultipleRelationships/yml/MultipleRelationships.yml deleted file mode 100644 index ab3f54f5cc..0000000000 --- a/minifi/minifi-integration-tests/src/test/resources/standalone/v2/MultipleRelationships/yml/MultipleRelationships.yml +++ /dev/null @@ -1,153 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the \"License\"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an \"AS IS\" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -MiNiFi Config Version: 2 -Flow Controller: - name: MultipleRelationships - comment: '' -Core Properties: - flow controller graceful shutdown period: 10 sec - flow service write delay interval: 500 ms - administrative yield duration: 30 sec - bored yield duration: 10 millis - max concurrent threads: 1 -FlowFile Repository: - partitions: 256 - checkpoint interval: 2 mins - always sync: false - Swap: - threshold: 20000 - in period: 5 sec - in threads: 1 - out period: 5 sec - out threads: 4 -Content Repository: - content claim max appendable size: 10 MB - content claim max flow files: 100 - always sync: false -Provenance Repository: - provenance rollover time: 1 min -Component Status Repository: - buffer size: 1440 - snapshot frequency: 1 min -Security Properties: - keystore: '' - keystore type: '' - keystore password: '' - key password: '' - truststore: '' - truststore type: '' - truststore password: '' - ssl protocol: '' - Sensitive Props: - key: - algorithm: NIFI_PBKDF2_AES_GCM_256 - provider: BC -Processors: -- id: 7c755ed6-0157-1000-0000-000000000000 - name: GenerateFlowFile - class: org.apache.nifi.processors.standard.GenerateFlowFile - max concurrent tasks: 1 - scheduling strategy: TIMER_DRIVEN - scheduling period: 0 sec - penalization period: 30 sec - yield period: 1 sec - run duration nanos: 0 - auto-terminated relationships list: [] - Properties: - Batch Size: '1' - Data Format: Binary - File Size: 0 B - Unique FlowFiles: 'false' -- id: 7c75ab71-0157-1000-0000-000000000000 - name: LogAttribute - class: org.apache.nifi.processors.standard.LogAttribute - max concurrent tasks: 1 - scheduling strategy: TIMER_DRIVEN - scheduling period: 0 sec - penalization period: 30 sec - yield period: 1 sec - run duration nanos: 0 - auto-terminated relationships list: - - success - Properties: - Attributes to Ignore: - Attributes to Log: - Log Level: info - Log Payload: 'false' - Log prefix: -- id: 7c768622-0157-1000-0000-000000000000 - name: RouteOnAttribute - class: org.apache.nifi.processors.standard.RouteOnAttribute - max concurrent tasks: 1 - scheduling strategy: TIMER_DRIVEN - scheduling period: 0 sec - penalization period: 30 sec - yield period: 1 sec - run duration nanos: 0 - auto-terminated relationships list: [] - Properties: - Routing Strategy: Route to Property name - abc: ${filename:equals('abc')} -- id: 7c79ba25-0157-1000-0000-000000000000 - name: UpdateAttribute - class: org.apache.nifi.processors.attributes.UpdateAttribute - max concurrent tasks: 1 - scheduling strategy: TIMER_DRIVEN - scheduling period: 0 sec - penalization period: 30 sec - yield period: 1 sec - run duration nanos: 0 - auto-terminated relationships list: [] - Properties: - Delete Attributes Expression: - filename: abc -Process Groups: [] -Input Ports: [] -Output Ports: [] -Funnels: [] -Connections: -- id: 7c79cce3-0157-1000-0000-000000000000 - name: GenerateFlowFile/success/UpdateAttribute - source id: 7c755ed6-0157-1000-0000-000000000000 - source relationship names: - - success - destination id: 7c79ba25-0157-1000-0000-000000000000 - max work queue size: 10000 - max work queue data size: 1 GB - flowfile expiration: 0 sec - queue prioritizer class: '' -- id: 7c798ca3-0157-1000-0000-000000000000 - name: RouteOnAttribute/abc/LogAttribute - source id: 7c768622-0157-1000-0000-000000000000 - source relationship names: - - abc - - unmatched - destination id: 7c75ab71-0157-1000-0000-000000000000 - max work queue size: 10000 - max work queue data size: 1 GB - flowfile expiration: 0 sec - queue prioritizer class: '' -- id: 7c79d576-0157-1000-0000-000000000000 - name: UpdateAttribute/success/RouteOnAttribute - source id: 7c79ba25-0157-1000-0000-000000000000 - source relationship names: - - success - destination id: 7c768622-0157-1000-0000-000000000000 - max work queue size: 10000 - max work queue data size: 1 GB - flowfile expiration: 0 sec - queue prioritizer class: '' -Remote Process Groups: [] diff --git a/minifi/minifi-integration-tests/src/test/resources/standalone/v2/MultipleRelationships/yml/expected.json b/minifi/minifi-integration-tests/src/test/resources/standalone/v2/MultipleRelationships/yml/expected.json deleted file mode 100644 index 02b1c066df..0000000000 --- a/minifi/minifi-integration-tests/src/test/resources/standalone/v2/MultipleRelationships/yml/expected.json +++ /dev/null @@ -1,6 +0,0 @@ -[ - { - "pattern": "o.a.n.c.r.StandardProcessSession StandardProcessSession\\[.*\\] for LogAttribute\\[id=7c75ab71-0157-1000-0000-000000000000\\], committed the following events: Transferred FlowFiles \\[.*\\] to 'success'", - "occurrences": 20 - } -] diff --git a/minifi/minifi-integration-tests/src/test/resources/standalone/v2/ProcessGroups/xml/ProcessGroups.xml b/minifi/minifi-integration-tests/src/test/resources/standalone/v2/ProcessGroups/xml/ProcessGroups.xml deleted file mode 100644 index c5d581295a..0000000000 --- a/minifi/minifi-integration-tests/src/test/resources/standalone/v2/ProcessGroups/xml/ProcessGroups.xml +++ /dev/null @@ -1,370 +0,0 @@ - - - - diff --git a/minifi/minifi-integration-tests/src/test/resources/standalone/v2/ProcessGroups/yml/ProcessGroups.yml b/minifi/minifi-integration-tests/src/test/resources/standalone/v2/ProcessGroups/yml/ProcessGroups.yml deleted file mode 100644 index 8a03330761..0000000000 --- a/minifi/minifi-integration-tests/src/test/resources/standalone/v2/ProcessGroups/yml/ProcessGroups.yml +++ /dev/null @@ -1,163 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the \"License\"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an \"AS IS\" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -MiNiFi Config Version: 2 -Flow Controller: - name: pgtest - comment: '' -Core Properties: - flow controller graceful shutdown period: 10 sec - flow service write delay interval: 500 ms - administrative yield duration: 30 sec - bored yield duration: 10 millis - max concurrent threads: 1 -FlowFile Repository: - partitions: 256 - checkpoint interval: 2 mins - always sync: false - Swap: - threshold: 20000 - in period: 5 sec - in threads: 1 - out period: 5 sec - out threads: 4 -Content Repository: - content claim max appendable size: 10 MB - content claim max flow files: 100 - always sync: false -Provenance Repository: - provenance rollover time: 1 min -Component Status Repository: - buffer size: 1440 - snapshot frequency: 1 min -Security Properties: - keystore: '' - keystore type: '' - keystore password: '' - key password: '' - truststore: '' - truststore type: '' - truststore password: '' - ssl protocol: '' - Sensitive Props: - key: - algorithm: NIFI_PBKDF2_AES_GCM_256 - provider: BC -Processors: -- id: e25cd92a-0157-1000-0000-000000000000 - name: GenerateFlowFile - class: org.apache.nifi.processors.standard.GenerateFlowFile - max concurrent tasks: 1 - scheduling strategy: TIMER_DRIVEN - scheduling period: 0 sec - penalization period: 30 sec - yield period: 1 sec - run duration nanos: 0 - auto-terminated relationships list: [] - Properties: - Batch Size: '1' - Data Format: Binary - File Size: 1 b - Unique FlowFiles: 'false' -- id: e25e0e6e-0157-1000-0000-000000000000 - name: LogAttribute - class: org.apache.nifi.processors.standard.LogAttribute - max concurrent tasks: 1 - scheduling strategy: TIMER_DRIVEN - scheduling period: 0 sec - penalization period: 30 sec - yield period: 1 sec - run duration nanos: 0 - auto-terminated relationships list: - - success - Properties: - Attributes to Ignore: - Attributes to Log: - Log Level: info - Log Payload: 'false' - Log prefix: OUTSIDE -Process Groups: -- id: e25cbc20-0157-1000-0000-000000000000 - name: black box - Processors: - - id: e25d2588-0157-1000-0000-000000000000 - name: LogAttribute - class: org.apache.nifi.processors.standard.LogAttribute - max concurrent tasks: 1 - scheduling strategy: TIMER_DRIVEN - scheduling period: 0 sec - penalization period: 30 sec - yield period: 1 sec - run duration nanos: 0 - auto-terminated relationships list: [] - Properties: - Attributes to Ignore: - Attributes to Log: - Log Level: info - Log Payload: 'false' - Log prefix: INSIDE - Process Groups: [] - Input Ports: - - id: e25d0cef-0157-1000-0000-000000000000 - name: input - Output Ports: - - id: e25d578c-0157-1000-0000-000000000000 - name: output - Funnels: [] - Connections: - - id: e25d6bd6-0157-1000-0000-000000000000 - name: LogAttribute/success/output - source id: e25d2588-0157-1000-0000-000000000000 - source relationship names: - - success - destination id: e25d578c-0157-1000-0000-000000000000 - max work queue size: 10000 - max work queue data size: 1 GB - flowfile expiration: 0 sec - queue prioritizer class: '' - - id: e25d35dc-0157-1000-0000-000000000000 - name: input//LogAttribute - source id: e25d0cef-0157-1000-0000-000000000000 - source relationship names: [] - destination id: e25d2588-0157-1000-0000-000000000000 - max work queue size: 10000 - max work queue data size: 1 GB - flowfile expiration: 0 sec - queue prioritizer class: '' - Remote Process Groups: [] -Input Ports: [] -Output Ports: [] -Funnels: [] -Connections: -- id: e25dbff3-0157-1000-0000-000000000000 - name: GenerateFlowFile/success/input - source id: e25cd92a-0157-1000-0000-000000000000 - source relationship names: - - success - destination id: e25d0cef-0157-1000-0000-000000000000 - max work queue size: 10000 - max work queue data size: 1 GB - flowfile expiration: 0 sec - queue prioritizer class: '' -- id: e25e22b1-0157-1000-0000-000000000000 - name: output//LogAttribute - source id: e25d578c-0157-1000-0000-000000000000 - source relationship names: [] - destination id: e25e0e6e-0157-1000-0000-000000000000 - max work queue size: 10000 - max work queue data size: 1 GB - flowfile expiration: 0 sec - queue prioritizer class: '' -Remote Process Groups: [] diff --git a/minifi/minifi-integration-tests/src/test/resources/standalone/v2/ProcessGroups/yml/expected.json b/minifi/minifi-integration-tests/src/test/resources/standalone/v2/ProcessGroups/yml/expected.json deleted file mode 100644 index 7f9e81c193..0000000000 --- a/minifi/minifi-integration-tests/src/test/resources/standalone/v2/ProcessGroups/yml/expected.json +++ /dev/null @@ -1,6 +0,0 @@ -[ - { - "pattern": "o.a.n.c.r.StandardProcessSession StandardProcessSession\\[id=.*\\] for LogAttribute\\[id=e25e0e6e-0157-1000-0000-000000000000\\], committed the following events: Transferred FlowFiles \\[.*\\] to 'success'", - "occurrences": 20 - } -] diff --git a/minifi/minifi-integration-tests/src/test/resources/standalone/v2/StressTestFramework/xml/StressTestFramework.xml b/minifi/minifi-integration-tests/src/test/resources/standalone/v2/StressTestFramework/xml/StressTestFramework.xml deleted file mode 100644 index 423417030f..0000000000 --- a/minifi/minifi-integration-tests/src/test/resources/standalone/v2/StressTestFramework/xml/StressTestFramework.xml +++ /dev/null @@ -1,539 +0,0 @@ - - - - \ No newline at end of file diff --git a/minifi/minifi-integration-tests/src/test/resources/standalone/v2/StressTestFramework/yml/StressTestFramework.yml b/minifi/minifi-integration-tests/src/test/resources/standalone/v2/StressTestFramework/yml/StressTestFramework.yml deleted file mode 100644 index 220f9da14d..0000000000 --- a/minifi/minifi-integration-tests/src/test/resources/standalone/v2/StressTestFramework/yml/StressTestFramework.yml +++ /dev/null @@ -1,211 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the \"License\"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an \"AS IS\" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -MiNiFi Config Version: 2 -Flow Controller: - name: StressTestFramework - comment: '' -Core Properties: - flow controller graceful shutdown period: 10 sec - flow service write delay interval: 500 ms - administrative yield duration: 30 sec - bored yield duration: 10 millis - max concurrent threads: 1 -FlowFile Repository: - partitions: 256 - checkpoint interval: 2 mins - always sync: false - Swap: - threshold: 20000 - in period: 5 sec - in threads: 1 - out period: 5 sec - out threads: 4 -Content Repository: - content claim max appendable size: 10 MB - content claim max flow files: 100 - always sync: false -Provenance Repository: - provenance rollover time: 1 min -Component Status Repository: - buffer size: 1440 - snapshot frequency: 1 min -Security Properties: - keystore: '' - keystore type: '' - keystore password: '' - key password: '' - truststore: '' - truststore type: '' - truststore password: '' - ssl protocol: '' - Sensitive Props: - key: - algorithm: NIFI_PBKDF2_AES_GCM_256 - provider: BC -Processors: -- id: 16a47794-5391-4ad2-0000-000000000000 - name: GenerateFlowFile - class: org.apache.nifi.processors.standard.GenerateFlowFile - max concurrent tasks: 1 - scheduling strategy: TIMER_DRIVEN - scheduling period: 0 sec - penalization period: 30 sec - yield period: 1 sec - run duration nanos: 0 - auto-terminated relationships list: [] - Properties: - Batch Size: '100' - Data Format: Binary - File Size: 0 KB - Unique FlowFiles: 'false' -- id: 4ad21391-16a4-1794-0000-000000000000 - name: GenerateFlowFile - class: org.apache.nifi.processors.standard.GenerateFlowFile - max concurrent tasks: 1 - scheduling strategy: TIMER_DRIVEN - scheduling period: 0 sec - penalization period: 30 sec - yield period: 1 sec - run duration nanos: 0 - auto-terminated relationships list: [] - Properties: - Batch Size: '100' - Data Format: Binary - File Size: 0 KB - Unique FlowFiles: 'false' -- id: 53914ad2-7794-16a4-0000-000000000000 - name: GenerateFlowFile - class: org.apache.nifi.processors.standard.GenerateFlowFile - max concurrent tasks: 1 - scheduling strategy: TIMER_DRIVEN - scheduling period: 0 sec - penalization period: 30 sec - yield period: 1 sec - run duration nanos: 0 - auto-terminated relationships list: [] - Properties: - Batch Size: '100' - Data Format: Binary - File Size: 0 KB - Unique FlowFiles: 'false' -- id: 779416a4-4ad2-1391-0000-000000000000 - name: GenerateFlowFile - class: org.apache.nifi.processors.standard.GenerateFlowFile - max concurrent tasks: 1 - scheduling strategy: TIMER_DRIVEN - scheduling period: 0 sec - penalization period: 30 sec - yield period: 1 sec - run duration nanos: 0 - auto-terminated relationships list: [] - Properties: - Batch Size: '100' - Data Format: Binary - File Size: 0 KB - Unique FlowFiles: 'false' -- id: 397a4910-cc01-4c6b-0000-000000000000 - name: RouteOnAttribute - class: org.apache.nifi.processors.standard.RouteOnAttribute - max concurrent tasks: 2 - scheduling strategy: TIMER_DRIVEN - scheduling period: 0 sec - penalization period: 30 sec - yield period: 1 sec - run duration nanos: 50000 - auto-terminated relationships list: - - unmatched - Properties: - Routing Strategy: Route to Property name -- id: 92557c76-f251-45a4-0000-000000000000 - name: UpdateAttribute - class: org.apache.nifi.processors.attributes.UpdateAttribute - max concurrent tasks: 2 - scheduling strategy: TIMER_DRIVEN - scheduling period: 0 sec - penalization period: 30 sec - yield period: 1 sec - run duration nanos: 100000 - auto-terminated relationships list: [] - Properties: - Delete Attributes Expression: - property 1: value 1 - property 2: value 2 ${nextInt()} -Process Groups: [] -Input Ports: [] -Output Ports: [] -Funnels: -- id: 4ad21392-16a4-1794-0000-000000000000 -Connections: -- id: 4ad21397-16a4-1794-0000-000000000000 - name: 4ad21392-16a4-1794-0000-000000000000//UpdateAttribute - source id: 4ad21392-16a4-1794-0000-000000000000 - source relationship names: [] - destination id: 92557c76-f251-45a4-0000-000000000000 - max work queue size: 10000 - max work queue data size: 1 GB - flowfile expiration: 0 sec - queue prioritizer class: '' -- id: 4ad21393-16a4-1794-0000-000000000000 - name: GenerateFlowFile/success/4ad21392-16a4-1794-0000-000000000000 - source id: 16a47794-5391-4ad2-0000-000000000000 - source relationship names: - - success - destination id: 4ad21392-16a4-1794-0000-000000000000 - max work queue size: 10000 - max work queue data size: 1 GB - flowfile expiration: 0 sec - queue prioritizer class: '' -- id: 4ad21394-16a4-1794-0000-000000000000 - name: GenerateFlowFile/success/4ad21392-16a4-1794-0000-000000000000 - source id: 53914ad2-7794-16a4-0000-000000000000 - source relationship names: - - success - destination id: 4ad21392-16a4-1794-0000-000000000000 - max work queue size: 10000 - max work queue data size: 1 GB - flowfile expiration: 0 sec - queue prioritizer class: '' -- id: 4ad21395-16a4-1794-0000-000000000000 - name: GenerateFlowFile/success/4ad21392-16a4-1794-0000-000000000000 - source id: 779416a4-4ad2-1391-0000-000000000000 - source relationship names: - - success - destination id: 4ad21392-16a4-1794-0000-000000000000 - max work queue size: 10000 - max work queue data size: 1 GB - flowfile expiration: 0 sec - queue prioritizer class: '' -- id: 4ad21396-16a4-1794-0000-000000000000 - name: GenerateFlowFile/success/4ad21392-16a4-1794-0000-000000000000 - source id: 4ad21391-16a4-1794-0000-000000000000 - source relationship names: - - success - destination id: 4ad21392-16a4-1794-0000-000000000000 - max work queue size: 10000 - max work queue data size: 1 GB - flowfile expiration: 0 sec - queue prioritizer class: '' -- id: 4c53556e-eb46-458c-0000-000000000000 - name: UpdateAttribute/success/RouteOnAttribute - source id: 92557c76-f251-45a4-0000-000000000000 - source relationship names: - - success - destination id: 397a4910-cc01-4c6b-0000-000000000000 - max work queue size: 2000 - max work queue data size: 0 MB - flowfile expiration: 0 sec - queue prioritizer class: org.apache.nifi.prioritizer.FirstInFirstOutPrioritizer -Remote Process Groups: [] diff --git a/minifi/minifi-integration-tests/src/test/resources/standalone/v2/StressTestFramework/yml/expected.json b/minifi/minifi-integration-tests/src/test/resources/standalone/v2/StressTestFramework/yml/expected.json deleted file mode 100644 index 5589030688..0000000000 --- a/minifi/minifi-integration-tests/src/test/resources/standalone/v2/StressTestFramework/yml/expected.json +++ /dev/null @@ -1,6 +0,0 @@ -[ - { - "pattern": "o.a.n.c.r.StandardProcessSession StandardProcessSession\\[id=.*\\] for RouteOnAttribute\\[id=397a4910-cc01-4c6b-0000-000000000000\\], committed the following events: Transferred FlowFiles \\[.*\\] to 'unmatched'", - "occurrences": 20 - } -] diff --git a/minifi/minifi-nar-bundles/minifi-framework-bundle/minifi-framework/minifi-framework-core/pom.xml b/minifi/minifi-nar-bundles/minifi-framework-bundle/minifi-framework/minifi-framework-core/pom.xml index 07bf29c5f0..acfe3ed9c4 100644 --- a/minifi/minifi-nar-bundles/minifi-framework-bundle/minifi-framework/minifi-framework-core/pom.xml +++ b/minifi/minifi-nar-bundles/minifi-framework-bundle/minifi-framework/minifi-framework-core/pom.xml @@ -31,6 +31,11 @@ limitations under the License. minifi-commons-api 2.0.0-SNAPSHOT + + org.apache.nifi.minifi + minifi-commons-framework + 2.0.0-SNAPSHOT + org.apache.nifi.minifi minifi-framework-api diff --git a/minifi/minifi-nar-bundles/minifi-framework-bundle/minifi-framework/minifi-framework-core/src/main/java/org/apache/nifi/minifi/c2/C2NifiClientService.java b/minifi/minifi-nar-bundles/minifi-framework-bundle/minifi-framework/minifi-framework-core/src/main/java/org/apache/nifi/minifi/c2/C2NifiClientService.java index 02ad5f7054..74109a6887 100644 --- a/minifi/minifi-nar-bundles/minifi-framework-bundle/minifi-framework/minifi-framework-core/src/main/java/org/apache/nifi/minifi/c2/C2NifiClientService.java +++ b/minifi/minifi-nar-bundles/minifi-framework-bundle/minifi-framework/minifi-framework-core/src/main/java/org/apache/nifi/minifi/c2/C2NifiClientService.java @@ -18,50 +18,44 @@ package org.apache.nifi.minifi.c2; import static java.util.Optional.ofNullable; -import static org.apache.nifi.minifi.MiNiFiProperties.C2_AGENT_CLASS; -import static org.apache.nifi.minifi.MiNiFiProperties.C2_AGENT_HEARTBEAT_PERIOD; -import static org.apache.nifi.minifi.MiNiFiProperties.C2_AGENT_IDENTIFIER; -import static org.apache.nifi.minifi.MiNiFiProperties.C2_ASSET_DIRECTORY; -import static org.apache.nifi.minifi.MiNiFiProperties.C2_CONFIG_DIRECTORY; -import static org.apache.nifi.minifi.MiNiFiProperties.C2_FULL_HEARTBEAT; -import static org.apache.nifi.minifi.MiNiFiProperties.C2_KEEP_ALIVE_DURATION; -import static org.apache.nifi.minifi.MiNiFiProperties.C2_MAX_IDLE_CONNECTIONS; -import static org.apache.nifi.minifi.MiNiFiProperties.C2_REQUEST_COMPRESSION; -import static org.apache.nifi.minifi.MiNiFiProperties.C2_REST_PATH_BASE; -import static org.apache.nifi.minifi.MiNiFiProperties.C2_REST_CALL_TIMEOUT; -import static org.apache.nifi.minifi.MiNiFiProperties.C2_REST_CONNECTION_TIMEOUT; -import static org.apache.nifi.minifi.MiNiFiProperties.C2_REST_PATH_ACKNOWLEDGE; -import static org.apache.nifi.minifi.MiNiFiProperties.C2_REST_PATH_HEARTBEAT; -import static org.apache.nifi.minifi.MiNiFiProperties.C2_REST_READ_TIMEOUT; -import static org.apache.nifi.minifi.MiNiFiProperties.C2_REST_URL; -import static org.apache.nifi.minifi.MiNiFiProperties.C2_REST_URL_ACK; -import static org.apache.nifi.minifi.MiNiFiProperties.C2_RUNTIME_MANIFEST_IDENTIFIER; -import static org.apache.nifi.minifi.MiNiFiProperties.C2_RUNTIME_TYPE; -import static org.apache.nifi.minifi.MiNiFiProperties.C2_SECURITY_KEYSTORE_LOCATION; -import static org.apache.nifi.minifi.MiNiFiProperties.C2_SECURITY_KEYSTORE_PASSWORD; -import static org.apache.nifi.minifi.MiNiFiProperties.C2_SECURITY_KEYSTORE_TYPE; -import static org.apache.nifi.minifi.MiNiFiProperties.C2_SECURITY_TRUSTSTORE_LOCATION; -import static org.apache.nifi.minifi.MiNiFiProperties.C2_SECURITY_TRUSTSTORE_PASSWORD; -import static org.apache.nifi.minifi.MiNiFiProperties.C2_SECURITY_TRUSTSTORE_TYPE; -import static org.apache.nifi.minifi.MiNiFiProperties.CONF_DIR; -import static org.apache.nifi.minifi.commons.api.MiNiFiConstants.CONFIG_UPDATED_FILE_NAME; +import static java.util.stream.Collectors.toMap; +import static org.apache.nifi.minifi.commons.api.MiNiFiProperties.C2_AGENT_CLASS; +import static org.apache.nifi.minifi.commons.api.MiNiFiProperties.C2_AGENT_HEARTBEAT_PERIOD; +import static org.apache.nifi.minifi.commons.api.MiNiFiProperties.C2_AGENT_IDENTIFIER; +import static org.apache.nifi.minifi.commons.api.MiNiFiProperties.C2_ASSET_DIRECTORY; +import static org.apache.nifi.minifi.commons.api.MiNiFiProperties.C2_CONFIG_DIRECTORY; +import static org.apache.nifi.minifi.commons.api.MiNiFiProperties.C2_FULL_HEARTBEAT; +import static org.apache.nifi.minifi.commons.api.MiNiFiProperties.C2_KEEP_ALIVE_DURATION; +import static org.apache.nifi.minifi.commons.api.MiNiFiProperties.C2_MAX_IDLE_CONNECTIONS; +import static org.apache.nifi.minifi.commons.api.MiNiFiProperties.C2_REQUEST_COMPRESSION; +import static org.apache.nifi.minifi.commons.api.MiNiFiProperties.C2_REST_CALL_TIMEOUT; +import static org.apache.nifi.minifi.commons.api.MiNiFiProperties.C2_REST_CONNECTION_TIMEOUT; +import static org.apache.nifi.minifi.commons.api.MiNiFiProperties.C2_REST_HTTP_HEADERS; +import static org.apache.nifi.minifi.commons.api.MiNiFiProperties.C2_REST_PATH_ACKNOWLEDGE; +import static org.apache.nifi.minifi.commons.api.MiNiFiProperties.C2_REST_PATH_BASE; +import static org.apache.nifi.minifi.commons.api.MiNiFiProperties.C2_REST_PATH_HEARTBEAT; +import static org.apache.nifi.minifi.commons.api.MiNiFiProperties.C2_REST_READ_TIMEOUT; +import static org.apache.nifi.minifi.commons.api.MiNiFiProperties.C2_REST_URL; +import static org.apache.nifi.minifi.commons.api.MiNiFiProperties.C2_REST_URL_ACK; +import static org.apache.nifi.minifi.commons.api.MiNiFiProperties.C2_RUNTIME_MANIFEST_IDENTIFIER; +import static org.apache.nifi.minifi.commons.api.MiNiFiProperties.C2_RUNTIME_TYPE; +import static org.apache.nifi.minifi.commons.api.MiNiFiProperties.C2_SECURITY_KEYSTORE_LOCATION; +import static org.apache.nifi.minifi.commons.api.MiNiFiProperties.C2_SECURITY_KEYSTORE_PASSWORD; +import static org.apache.nifi.minifi.commons.api.MiNiFiProperties.C2_SECURITY_KEYSTORE_TYPE; +import static org.apache.nifi.minifi.commons.api.MiNiFiProperties.C2_SECURITY_TRUSTSTORE_LOCATION; +import static org.apache.nifi.minifi.commons.api.MiNiFiProperties.C2_SECURITY_TRUSTSTORE_PASSWORD; +import static org.apache.nifi.minifi.commons.api.MiNiFiProperties.C2_SECURITY_TRUSTSTORE_TYPE; +import static org.apache.nifi.util.NiFiProperties.FLOW_CONFIGURATION_JSON_FILE; import com.fasterxml.jackson.databind.ObjectMapper; -import java.io.File; import java.io.IOException; import java.io.UncheckedIOException; -import java.nio.charset.StandardCharsets; -import java.nio.file.Files; -import java.nio.file.Path; -import java.util.Arrays; -import java.util.Collection; -import java.util.Collections; -import java.util.HashMap; -import java.util.Iterator; +import java.util.List; import java.util.Map; import java.util.Optional; import java.util.concurrent.ScheduledThreadPoolExecutor; import java.util.concurrent.TimeUnit; +import org.apache.commons.lang3.tuple.Pair; import org.apache.nifi.bootstrap.BootstrapCommunicator; import org.apache.nifi.c2.client.C2ClientConfig; import org.apache.nifi.c2.client.http.C2HttpClient; @@ -80,6 +74,7 @@ import org.apache.nifi.c2.client.service.operation.SupportedOperationsProvider; import org.apache.nifi.c2.client.service.operation.TransferDebugOperationHandler; import org.apache.nifi.c2.client.service.operation.UpdateAssetOperationHandler; import org.apache.nifi.c2.client.service.operation.UpdateConfigurationOperationHandler; +import org.apache.nifi.c2.client.service.operation.UpdateConfigurationStrategy; import org.apache.nifi.c2.client.service.operation.UpdatePropertiesOperationHandler; import org.apache.nifi.c2.protocol.api.AgentManifest; import org.apache.nifi.c2.protocol.api.AgentRepositories; @@ -91,20 +86,20 @@ import org.apache.nifi.c2.protocol.api.C2OperationState.OperationState; import org.apache.nifi.c2.protocol.api.FlowQueueStatus; import org.apache.nifi.c2.serializer.C2JacksonSerializer; import org.apache.nifi.controller.FlowController; -import org.apache.nifi.controller.status.ConnectionStatus; -import org.apache.nifi.controller.status.ProcessGroupStatus; -import org.apache.nifi.diagnostics.StorageUsage; import org.apache.nifi.diagnostics.SystemDiagnostics; import org.apache.nifi.extension.manifest.parser.ExtensionManifestParser; import org.apache.nifi.extension.manifest.parser.jaxb.JAXBExtensionManifestParser; import org.apache.nifi.manifest.RuntimeManifestService; import org.apache.nifi.manifest.StandardRuntimeManifestService; +import org.apache.nifi.minifi.c2.command.DefaultUpdateConfigurationStrategy; import org.apache.nifi.minifi.c2.command.PropertiesPersister; import org.apache.nifi.minifi.c2.command.TransferDebugCommandHelper; import org.apache.nifi.minifi.c2.command.UpdateAssetCommandHelper; import org.apache.nifi.minifi.c2.command.UpdatePropertiesPropertyProvider; import org.apache.nifi.minifi.commons.api.MiNiFiCommandState; +import org.apache.nifi.minifi.commons.service.FlowEnrichService; import org.apache.nifi.nar.ExtensionManagerHolder; +import org.apache.nifi.services.FlowService; import org.apache.nifi.util.FormatUtils; import org.apache.nifi.util.NiFiProperties; import org.slf4j.Logger; @@ -113,40 +108,40 @@ import org.slf4j.LoggerFactory; public class C2NifiClientService { private static final Logger LOGGER = LoggerFactory.getLogger(C2NifiClientService.class); - private static final String TARGET_CONFIG_FILE = "/" + CONFIG_UPDATED_FILE_NAME; private static final String ROOT_GROUP_ID = "root"; private static final Long INITIAL_DELAY = 10000L; private static final Integer TERMINATION_WAIT = 5000; private static final int MINIFI_RESTART_TIMEOUT_SECONDS = 60; private static final String ACKNOWLEDGE_OPERATION = "ACKNOWLEDGE_OPERATION"; private static final int IS_ACK_RECEIVED_POLL_INTERVAL = 1000; - private static final Map OPERATION_STATE_MAP = getOperationStateMap(); private static final int MAX_WAIT_FOR_BOOTSTRAP_ACK_MS = 20000; + private static final Map OPERATION_STATE_MAP = Map.of( + MiNiFiCommandState.FULLY_APPLIED, OperationState.FULLY_APPLIED, + MiNiFiCommandState.NO_OPERATION, OperationState.NO_OPERATION, + MiNiFiCommandState.NOT_APPLIED_WITH_RESTART, OperationState.NOT_APPLIED, + MiNiFiCommandState.NOT_APPLIED_WITHOUT_RESTART, OperationState.NOT_APPLIED); + private final C2ClientService c2ClientService; - private final FlowController flowController; - private final String propertiesDir; - private final ScheduledThreadPoolExecutor heartbeatExecutorService = new ScheduledThreadPoolExecutor(1); - private final ScheduledThreadPoolExecutor bootstrapAcknowledgeExecutorService = new ScheduledThreadPoolExecutor(1); - private final ExtensionManifestParser extensionManifestParser = new JAXBExtensionManifestParser(); - + private final ScheduledThreadPoolExecutor heartbeatExecutorService; + private final ScheduledThreadPoolExecutor bootstrapAcknowledgeExecutorService; + private final ExtensionManifestParser extensionManifestParser; private final RuntimeManifestService runtimeManifestService; - private final SupportedOperationsProvider supportedOperationsProvider; private final RequestedOperationDAO requestedOperationDAO; private final BootstrapCommunicator bootstrapCommunicator; - private volatile boolean ackReceived = false; - private final UpdatePropertiesPropertyProvider updatePropertiesPropertyProvider; - private final PropertiesPersister propertiesPersister; - private final ObjectMapper objectMapper; - private final long heartbeatPeriod; - public C2NifiClientService(NiFiProperties niFiProperties, FlowController flowController, BootstrapCommunicator bootstrapCommunicator) { + private volatile boolean ackReceived = false; + + public C2NifiClientService(NiFiProperties niFiProperties, FlowController flowController, BootstrapCommunicator bootstrapCommunicator, FlowService flowService) { + this.heartbeatExecutorService = new ScheduledThreadPoolExecutor(1); + this.bootstrapAcknowledgeExecutorService = new ScheduledThreadPoolExecutor(1); + this.extensionManifestParser = new JAXBExtensionManifestParser(); + C2ClientConfig clientConfig = generateClientConfig(niFiProperties); - FlowIdHolder flowIdHolder = new FlowIdHolder(clientConfig.getConfDirectory()); - this.propertiesDir = niFiProperties.getProperty(NiFiProperties.PROPERTIES_FILE_PATH, null); + this.runtimeManifestService = new StandardRuntimeManifestService( ExtensionManagerHolder.getExtensionManager(), extensionManifestParser, @@ -157,29 +152,20 @@ public class C2NifiClientService { this.flowController = flowController; C2HttpClient client = C2HttpClient.create(clientConfig, new C2JacksonSerializer()); + FlowIdHolder flowIdHolder = new FlowIdHolder(clientConfig.getConfDirectory()); C2HeartbeatFactory heartbeatFactory = new C2HeartbeatFactory(clientConfig, flowIdHolder, new ManifestHashProvider()); - OperandPropertiesProvider emptyOperandPropertiesProvider = new EmptyOperandPropertiesProvider(); - TransferDebugCommandHelper transferDebugCommandHelper = new TransferDebugCommandHelper(niFiProperties); - UpdateAssetCommandHelper updateAssetCommandHelper = new UpdateAssetCommandHelper(clientConfig.getC2AssetDirectory()); - objectMapper = new ObjectMapper(); - updateAssetCommandHelper.createAssetDirectory(); - this.bootstrapCommunicator = bootstrapCommunicator; - requestedOperationDAO = new FileBasedRequestedOperationDAO(niFiProperties.getProperty("org.apache.nifi.minifi.bootstrap.config.pid.dir", "bin"), objectMapper); + + this.requestedOperationDAO = new FileBasedRequestedOperationDAO(niFiProperties.getProperty("org.apache.nifi.minifi.bootstrap.config.pid.dir", "bin"), new ObjectMapper()); String bootstrapConfigFileLocation = niFiProperties.getProperty("nifi.minifi.bootstrap.file"); - updatePropertiesPropertyProvider = new UpdatePropertiesPropertyProvider(bootstrapConfigFileLocation); - propertiesPersister = new PropertiesPersister(updatePropertiesPropertyProvider, bootstrapConfigFileLocation); - C2OperationHandlerProvider c2OperationHandlerProvider = new C2OperationHandlerProvider(Arrays.asList( - new UpdateConfigurationOperationHandler(client, flowIdHolder, this::updateFlowContent, emptyOperandPropertiesProvider), - new DescribeManifestOperationHandler(heartbeatFactory, this::generateRuntimeInfo, emptyOperandPropertiesProvider), - TransferDebugOperationHandler.create(client, emptyOperandPropertiesProvider, - transferDebugCommandHelper.debugBundleFiles(), transferDebugCommandHelper::excludeSensitiveText), - UpdateAssetOperationHandler.create(client, emptyOperandPropertiesProvider, - updateAssetCommandHelper::assetUpdatePrecondition, updateAssetCommandHelper::assetPersistFunction), - new UpdatePropertiesOperationHandler(updatePropertiesPropertyProvider, propertiesPersister::persistProperties) - )); + + C2OperationHandlerProvider c2OperationHandlerProvider = c2OperationHandlerProvider(niFiProperties, flowController, flowService, flowIdHolder, + client, heartbeatFactory, bootstrapConfigFileLocation, clientConfig.getC2AssetDirectory()); + this.c2ClientService = new C2ClientService(client, heartbeatFactory, c2OperationHandlerProvider, requestedOperationDAO, this::registerOperation); this.supportedOperationsProvider = new SupportedOperationsProvider(c2OperationHandlerProvider.getHandlers()); - bootstrapCommunicator.registerMessageHandler(ACKNOWLEDGE_OPERATION, (params, output) -> acknowledgeHandler(params)); + + this.bootstrapCommunicator = bootstrapCommunicator; + this.bootstrapCommunicator.registerMessageHandler(ACKNOWLEDGE_OPERATION, (params, output) -> acknowledgeHandler(params)); } private C2ClientConfig generateClientConfig(NiFiProperties properties) { @@ -198,6 +184,7 @@ public class C2NifiClientService { .maxIdleConnections(Integer.parseInt(properties.getProperty(C2_MAX_IDLE_CONNECTIONS.getKey(), C2_MAX_IDLE_CONNECTIONS.getDefaultValue()))) .keepAliveDuration((long) FormatUtils.getPreciseTimeDuration(properties.getProperty(C2_KEEP_ALIVE_DURATION.getKey(), C2_KEEP_ALIVE_DURATION.getDefaultValue()), TimeUnit.MILLISECONDS)) + .httpHeaders(properties.getProperty(C2_REST_HTTP_HEADERS.getKey(), C2_REST_HTTP_HEADERS.getDefaultValue())) .c2RequestCompression(properties.getProperty(C2_REQUEST_COMPRESSION.getKey(), C2_REQUEST_COMPRESSION.getDefaultValue())) .c2AssetDirectory(properties.getProperty(C2_ASSET_DIRECTORY.getKey(), C2_ASSET_DIRECTORY.getDefaultValue())) .confDirectory(properties.getProperty(C2_CONFIG_DIRECTORY.getKey(), C2_CONFIG_DIRECTORY.getDefaultValue())) @@ -217,6 +204,31 @@ public class C2NifiClientService { .build(); } + private C2OperationHandlerProvider c2OperationHandlerProvider(NiFiProperties niFiProperties, FlowController flowController, FlowService flowService, + FlowIdHolder flowIdHolder, C2HttpClient client, C2HeartbeatFactory heartbeatFactory, + String bootstrapConfigFileLocation, String c2AssetDirectory) { + OperandPropertiesProvider emptyOperandPropertiesProvider = new EmptyOperandPropertiesProvider(); + TransferDebugCommandHelper transferDebugCommandHelper = new TransferDebugCommandHelper(niFiProperties); + UpdateAssetCommandHelper updateAssetCommandHelper = new UpdateAssetCommandHelper(c2AssetDirectory); + updateAssetCommandHelper.createAssetDirectory(); + UpdatePropertiesPropertyProvider updatePropertiesPropertyProvider = new UpdatePropertiesPropertyProvider(bootstrapConfigFileLocation); + PropertiesPersister propertiesPersister = new PropertiesPersister(updatePropertiesPropertyProvider, bootstrapConfigFileLocation); + + FlowEnrichService flowEnrichService = new FlowEnrichService(niFiProperties); + UpdateConfigurationStrategy updateConfigurationStrategy = + new DefaultUpdateConfigurationStrategy(flowController, flowService, flowEnrichService, niFiProperties.getProperty(FLOW_CONFIGURATION_JSON_FILE)); + + return new C2OperationHandlerProvider(List.of( + new UpdateConfigurationOperationHandler(client, flowIdHolder, updateConfigurationStrategy, emptyOperandPropertiesProvider), + new DescribeManifestOperationHandler(heartbeatFactory, this::generateRuntimeInfo, emptyOperandPropertiesProvider), + TransferDebugOperationHandler.create(client, emptyOperandPropertiesProvider, + transferDebugCommandHelper.debugBundleFiles(), transferDebugCommandHelper::excludeSensitiveText), + UpdateAssetOperationHandler.create(client, emptyOperandPropertiesProvider, + updateAssetCommandHelper::assetUpdatePrecondition, updateAssetCommandHelper::assetPersistFunction), + new UpdatePropertiesOperationHandler(updatePropertiesPropertyProvider, propertiesPersister::persistProperties) + )); + } + public void start() { handleOngoingOperations(); heartbeatExecutorService.scheduleAtFixedRate(() -> c2ClientService.sendHeartbeat(generateRuntimeInfo()), INITIAL_DELAY, heartbeatPeriod, TimeUnit.MILLISECONDS); @@ -242,7 +254,7 @@ public class C2NifiClientService { private void waitForAcknowledgeFromBootstrap() { LOGGER.info("Waiting for ACK signal from Bootstrap"); int currentWaitTime = 0; - while(!ackReceived) { + while (!ackReceived) { try { Thread.sleep(IS_ACK_RECEIVED_POLL_INTERVAL); } catch (InterruptedException e) { @@ -260,7 +272,7 @@ public class C2NifiClientService { try { ackReceived = false; registerAcknowledgeTimeoutTask(c2Operation); - String command = Optional.ofNullable(c2Operation.getOperand()) + String command = ofNullable(c2Operation.getOperand()) .map(operand -> c2Operation.getOperation().name() + "_" + operand.name()) .orElse(c2Operation.getOperation().name()); bootstrapCommunicator.sendCommand(command); @@ -329,78 +341,47 @@ public class C2NifiClientService { } private AgentRepositories getAgentRepositories() { - final SystemDiagnostics systemDiagnostics = flowController.getSystemDiagnostics(); + SystemDiagnostics systemDiagnostics = flowController.getSystemDiagnostics(); - final AgentRepositories repos = new AgentRepositories(); - final AgentRepositoryStatus flowFileRepoStatus = new AgentRepositoryStatus(); - final StorageUsage ffRepoStorageUsage = systemDiagnostics.getFlowFileRepositoryStorageUsage(); - flowFileRepoStatus.setDataSize(ffRepoStorageUsage.getUsedSpace()); - flowFileRepoStatus.setDataSizeMax(ffRepoStorageUsage.getTotalSpace()); - repos.setFlowFile(flowFileRepoStatus); + AgentRepositoryStatus agentFlowRepositoryStatus = ofNullable(systemDiagnostics.getFlowFileRepositoryStorageUsage()) + .map(flowFileRepositoryStorageUsage -> { + AgentRepositoryStatus flowRepositoryStatus = new AgentRepositoryStatus(); + flowRepositoryStatus.setDataSize(flowFileRepositoryStorageUsage.getUsedSpace()); + flowRepositoryStatus.setDataSizeMax(flowFileRepositoryStorageUsage.getTotalSpace()); + return flowRepositoryStatus; + }) + .orElseGet(AgentRepositoryStatus::new); - final AgentRepositoryStatus provRepoStatus = new AgentRepositoryStatus(); - final Iterator> provRepoStorageUsages = systemDiagnostics.getProvenanceRepositoryStorageUsage().entrySet().iterator(); - if (provRepoStorageUsages.hasNext()) { - final StorageUsage provRepoStorageUsage = provRepoStorageUsages.next().getValue(); - provRepoStatus.setDataSize(provRepoStorageUsage.getUsedSpace()); - provRepoStatus.setDataSizeMax(provRepoStorageUsage.getTotalSpace()); - } + AgentRepositoryStatus agentProvenanceRepositoryStatus = systemDiagnostics.getProvenanceRepositoryStorageUsage().entrySet().stream() + .findFirst() + .map(Map.Entry::getValue) + .map(provRepoStorageUsage -> { + AgentRepositoryStatus provenanceRepositoryStatus = new AgentRepositoryStatus(); + provenanceRepositoryStatus.setDataSize(provRepoStorageUsage.getUsedSpace()); + provenanceRepositoryStatus.setDataSizeMax(provRepoStorageUsage.getTotalSpace()); + return provenanceRepositoryStatus; + }) + .orElseGet(AgentRepositoryStatus::new); - repos.setProvenance(provRepoStatus); - - return repos; + AgentRepositories agentRepositories = new AgentRepositories(); + agentRepositories.setFlowFile(agentFlowRepositoryStatus); + agentRepositories.setProvenance(agentProvenanceRepositoryStatus); + return agentRepositories; } private Map getQueueStatus() { - ProcessGroupStatus rootProcessGroupStatus = flowController.getEventAccess().getGroupStatus(ROOT_GROUP_ID); - - final Collection connectionStatuses = rootProcessGroupStatus.getConnectionStatus(); - - final Map processGroupStatus = new HashMap<>(); - for (ConnectionStatus connectionStatus : connectionStatuses) { - final FlowQueueStatus flowQueueStatus = new FlowQueueStatus(); - - flowQueueStatus.setSize((long) connectionStatus.getQueuedCount()); - flowQueueStatus.setSizeMax(connectionStatus.getBackPressureObjectThreshold()); - - flowQueueStatus.setDataSize(connectionStatus.getQueuedBytes()); - flowQueueStatus.setDataSizeMax(connectionStatus.getBackPressureBytesThreshold()); - - processGroupStatus.put(connectionStatus.getId(), flowQueueStatus); - } - - return processGroupStatus; - } - - private boolean updateFlowContent(byte[] updateContent) { - if (LOGGER.isDebugEnabled()) { - LOGGER.debug("Update content: \n{}", new String(updateContent, StandardCharsets.UTF_8)); - } - Path path = getTargetConfigFile().toPath(); - try { - Files.write(getTargetConfigFile().toPath(), updateContent); - LOGGER.info("Updated configuration was written to: {}", path); - return true; - } catch (IOException e) { - LOGGER.error("Configuration update failed. File creation was not successful targeting: {}", path, e); - return false; - } - } - - private File getTargetConfigFile() { - return ofNullable(propertiesDir) - .map(File::new) - .map(File::getParent) - .map(parentDir -> new File(parentDir + TARGET_CONFIG_FILE)) - .orElse(new File(CONF_DIR.getDefaultValue() + TARGET_CONFIG_FILE)); - } - - private static Map getOperationStateMap() { - Map operationStateMapping = new HashMap<>(); - operationStateMapping.put(MiNiFiCommandState.FULLY_APPLIED, OperationState.FULLY_APPLIED); - operationStateMapping.put(MiNiFiCommandState.NO_OPERATION, OperationState.NO_OPERATION); - operationStateMapping.put(MiNiFiCommandState.NOT_APPLIED_WITH_RESTART, OperationState.NOT_APPLIED); - operationStateMapping.put(MiNiFiCommandState.NOT_APPLIED_WITHOUT_RESTART, OperationState.NOT_APPLIED); - return Collections.unmodifiableMap(operationStateMapping); + return flowController.getEventAccess() + .getGroupStatus(ROOT_GROUP_ID) + .getConnectionStatus() + .stream() + .map(connectionStatus -> { + FlowQueueStatus flowQueueStatus = new FlowQueueStatus(); + flowQueueStatus.setSize((long) connectionStatus.getQueuedCount()); + flowQueueStatus.setSizeMax(connectionStatus.getBackPressureObjectThreshold()); + flowQueueStatus.setDataSize(connectionStatus.getQueuedBytes()); + flowQueueStatus.setDataSizeMax(connectionStatus.getBackPressureBytesThreshold()); + return Pair.of(connectionStatus.getId(), flowQueueStatus); + }) + .collect(toMap(Pair::getKey, Pair::getValue)); } } \ No newline at end of file diff --git a/minifi/minifi-nar-bundles/minifi-framework-bundle/minifi-framework/minifi-framework-core/src/main/java/org/apache/nifi/minifi/c2/command/DefaultUpdateConfigurationStrategy.java b/minifi/minifi-nar-bundles/minifi-framework-bundle/minifi-framework/minifi-framework-core/src/main/java/org/apache/nifi/minifi/c2/command/DefaultUpdateConfigurationStrategy.java new file mode 100644 index 0000000000..1db1549d6c --- /dev/null +++ b/minifi/minifi-nar-bundles/minifi-framework-bundle/minifi-framework/minifi-framework-core/src/main/java/org/apache/nifi/minifi/c2/command/DefaultUpdateConfigurationStrategy.java @@ -0,0 +1,226 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.nifi.minifi.c2.command; + +import static java.nio.charset.StandardCharsets.UTF_8; +import static java.util.UUID.randomUUID; +import static java.util.function.Predicate.not; +import static org.apache.nifi.minifi.commons.api.MiNiFiConstants.BACKUP_EXTENSION; +import static org.apache.nifi.minifi.commons.api.MiNiFiConstants.RAW_EXTENSION; +import static org.apache.nifi.minifi.commons.util.FlowUpdateUtils.backup; +import static org.apache.nifi.minifi.commons.util.FlowUpdateUtils.persist; +import static org.apache.nifi.minifi.commons.util.FlowUpdateUtils.removeIfExists; +import static org.apache.nifi.minifi.commons.util.FlowUpdateUtils.revert; + +import java.io.IOException; +import java.nio.file.Path; +import java.util.Collection; +import java.util.List; +import java.util.Optional; +import java.util.Set; +import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; +import java.util.function.Predicate; +import java.util.function.Supplier; +import java.util.stream.Stream; +import org.apache.commons.io.FilenameUtils; +import org.apache.commons.lang3.tuple.Pair; +import org.apache.nifi.c2.client.service.operation.UpdateConfigurationStrategy; +import org.apache.nifi.components.ValidationResult; +import org.apache.nifi.components.validation.ValidationStatus; +import org.apache.nifi.controller.ComponentNode; +import org.apache.nifi.controller.FlowController; +import org.apache.nifi.controller.ProcessorNode; +import org.apache.nifi.controller.flow.FlowManager; +import org.apache.nifi.groups.ProcessGroup; +import org.apache.nifi.groups.RemoteProcessGroup; +import org.apache.nifi.minifi.commons.service.FlowEnrichService; +import org.apache.nifi.services.FlowService; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class DefaultUpdateConfigurationStrategy implements UpdateConfigurationStrategy { + + private static final Logger LOGGER = LoggerFactory.getLogger(DefaultUpdateConfigurationStrategy.class); + + private static int VALIDATION_RETRY_PAUSE_DURATION_MS = 1000; + private static int VALIDATION_MAX_RETRIES = 5; + private static int FLOW_DRAIN_RETRY_PAUSE_DURATION_MS = 1000; + private static int FLOW_DRAIN_MAX_RETRIES = 60; + + private final FlowController flowController; + private final FlowService flowService; + private final FlowEnrichService flowEnrichService; + private final Path flowConfigurationFile; + private final Path backupFlowConfigurationFile; + private final Path rawFlowConfigurationFile; + private final Path backupRawFlowConfigurationFile; + + public DefaultUpdateConfigurationStrategy(FlowController flowController, FlowService flowService, FlowEnrichService flowEnrichService, String flowConfigurationFile) { + this.flowController = flowController; + this.flowService = flowService; + this.flowEnrichService = flowEnrichService; + Path flowConfigurationFilePath = Path.of(flowConfigurationFile).toAbsolutePath(); + this.flowConfigurationFile = flowConfigurationFilePath; + this.backupFlowConfigurationFile = Path.of(flowConfigurationFilePath + BACKUP_EXTENSION); + String flowConfigurationFileBaseName = FilenameUtils.getBaseName(flowConfigurationFilePath.toString()); + this.rawFlowConfigurationFile = flowConfigurationFilePath.getParent().resolve(flowConfigurationFileBaseName + RAW_EXTENSION); + this.backupRawFlowConfigurationFile = flowConfigurationFilePath.getParent().resolve(flowConfigurationFileBaseName + BACKUP_EXTENSION + RAW_EXTENSION); + } + + @Override + public boolean update(byte[] rawFlow) { + if (LOGGER.isDebugEnabled()) { + LOGGER.debug("Attempting to update flow with content: \n{}", new String(rawFlow, UTF_8)); + } + + try { + byte[] enrichedFlowCandidate = flowEnrichService.enrichFlow(rawFlow); + backup(flowConfigurationFile, backupFlowConfigurationFile); + backup(rawFlowConfigurationFile, backupRawFlowConfigurationFile); + persist(enrichedFlowCandidate, flowConfigurationFile, true); + persist(rawFlow, rawFlowConfigurationFile, false); + reloadFlow(); + return true; + } catch (IllegalStateException e) { + LOGGER.error("Configuration update failed. Reverting and reloading previous flow", e); + revert(backupFlowConfigurationFile, flowConfigurationFile); + revert(backupRawFlowConfigurationFile, rawFlowConfigurationFile); + try { + reloadFlow(); + } catch (IOException ex) { + LOGGER.error("Unable to reload the reverted flow", e); + } + return false; + } catch (Exception e) { + LOGGER.error("Configuration update failed. Reverting to previous flow, no reload is necessary", e); + revert(backupFlowConfigurationFile, flowConfigurationFile); + revert(backupRawFlowConfigurationFile, rawFlowConfigurationFile); + return false; + } finally { + removeIfExists(backupFlowConfigurationFile); + removeIfExists(backupRawFlowConfigurationFile); + } + } + + private void reloadFlow() throws IOException { + LOGGER.info("Initiating flow reload"); + stopFlowGracefully(flowController.getFlowManager().getRootGroup()); + + flowService.load(null); + flowController.onFlowInitialized(true); + + List validationErrors = validate(flowController.getFlowManager()); + if (!validationErrors.isEmpty()) { + LOGGER.error("Validation errors found when reloading the flow: {}", validationErrors); + throw new IllegalStateException("Unable to start flow due to validation errors"); + } + + flowController.getFlowManager().getRootGroup().startProcessing(); + LOGGER.info("Flow has been reloaded successfully"); + } + + private void stopFlowGracefully(ProcessGroup rootGroup) { + LOGGER.info("Stopping flow gracefully"); + Optional drainResult = stopSourceProcessorsAndWaitFlowToDrain(rootGroup); + + rootGroup.stopProcessing(); + rootGroup.getRemoteProcessGroups().stream() + .map(RemoteProcessGroup::stopTransmitting) + .forEach(this::waitForStopOrLogTimeOut); + drainResult.ifPresentOrElse( + rootProcessGroup -> { + LOGGER.warn("Flow did not stop within graceful period. Force stopping flow and emptying queues"); + rootProcessGroup.dropAllFlowFiles(randomUUID().toString(), randomUUID().toString()); + }, + () -> LOGGER.info("Flow has been stopped gracefully")); + } + + private Optional stopSourceProcessorsAndWaitFlowToDrain(ProcessGroup rootGroup) { + rootGroup.getProcessors().stream().filter(this::isSourceNode).forEach(rootGroup::stopProcessor); + return retry(() -> rootGroup, not(ProcessGroup::isDataQueued), FLOW_DRAIN_MAX_RETRIES, FLOW_DRAIN_RETRY_PAUSE_DURATION_MS); + } + + private boolean isSourceNode(ProcessorNode processorNode) { + boolean hasNoIncomingConnection = !processorNode.hasIncomingConnection(); + + boolean allIncomingConnectionsAreLoopConnections = processorNode.getIncomingConnections() + .stream() + .allMatch(connection -> connection.getSource().equals(processorNode)); + + return hasNoIncomingConnection || allIncomingConnectionsAreLoopConnections; + } + + private void waitForStopOrLogTimeOut(Future future) { + try { + future.get(10000, TimeUnit.MICROSECONDS); + } catch (Exception e) { + LOGGER.warn("Unable to stop remote process group within defined interval", e); + } + } + + private List validate(FlowManager flowManager) { + List componentNodes = extractComponentNodes(flowManager); + + retry(() -> componentIsInValidatingState(componentNodes), List::isEmpty, VALIDATION_MAX_RETRIES, VALIDATION_RETRY_PAUSE_DURATION_MS) + .ifPresent(components -> { + LOGGER.error("The following components are still in VALIDATING state: {}", components); + throw new IllegalStateException("Maximum retry number exceeded while waiting for components to be validated"); + }); + + return componentNodes.stream() + .map(ComponentNode::getValidationErrors) + .flatMap(Collection::stream) + .toList(); + } + + private List extractComponentNodes(FlowManager flowManager) { + return Stream.of( + flowManager.getAllControllerServices(), + flowManager.getAllReportingTasks(), + Set.copyOf(flowManager.getRootGroup().findAllProcessors())) + .flatMap(Set::stream) + .toList(); + } + + private List componentIsInValidatingState(List componentNodes) { + return componentNodes.stream() + .map(componentNode -> Pair.of((ComponentNode) componentNode, componentNode.performValidation())) + .filter(pair -> pair.getRight() == ValidationStatus.VALIDATING) + .map(Pair::getLeft) + .toList(); + } + + private Optional retry(Supplier input, Predicate predicate, int maxRetries, int pauseDurationMillis) { + int retries = 0; + while (true) { + T t = input.get(); + if (predicate.test(t)) { + return Optional.empty(); + } + if (retries == maxRetries) { + return Optional.ofNullable(t); + } + retries++; + try { + Thread.sleep(pauseDurationMillis); + } catch (InterruptedException e) { + } + } + } +} diff --git a/minifi/minifi-nar-bundles/minifi-framework-bundle/minifi-framework/minifi-framework-core/src/main/java/org/apache/nifi/minifi/c2/command/TransferDebugCommandHelper.java b/minifi/minifi-nar-bundles/minifi-framework-bundle/minifi-framework/minifi-framework-core/src/main/java/org/apache/nifi/minifi/c2/command/TransferDebugCommandHelper.java index 00b923958b..20f0cb5119 100644 --- a/minifi/minifi-nar-bundles/minifi-framework-bundle/minifi-framework/minifi-framework-core/src/main/java/org/apache/nifi/minifi/c2/command/TransferDebugCommandHelper.java +++ b/minifi/minifi-nar-bundles/minifi-framework-bundle/minifi-framework/minifi-framework-core/src/main/java/org/apache/nifi/minifi/c2/command/TransferDebugCommandHelper.java @@ -29,16 +29,11 @@ import java.util.Collections; import java.util.List; import java.util.Set; import java.util.stream.Stream; +import org.apache.nifi.minifi.commons.api.MiNiFiProperties; import org.apache.nifi.util.NiFiProperties; public class TransferDebugCommandHelper { - private static final String MINIFI_CONFIG_FILE_PATH = "nifi.minifi.config.file"; - private static final String MINIFI_BOOTSTRAP_FILE_PATH = "nifi.minifi.bootstrap.file"; - private static final String MINIFI_LOG_DIRECTORY = "nifi.minifi.log.directory"; - private static final String MINIFI_APP_LOG_FILE = "nifi.minifi.app.log.file"; - private static final String MINIFI_BOOTSTRAP_LOG_FILE = "nifi.minifi.bootstrap.log.file"; - private static final Set SENSITIVE_PROPERTY_KEYWORDS = Stream.of("key:", "algorithm:", "secret.key", "sensitive.props.key", "sensitive.props.algorithm", "secret", "password", "passwd") .map(String::toLowerCase) @@ -52,10 +47,10 @@ public class TransferDebugCommandHelper { public List debugBundleFiles() { return Stream.of( - Paths.get(niFiProperties.getProperty(MINIFI_CONFIG_FILE_PATH)), - Paths.get(niFiProperties.getProperty(MINIFI_BOOTSTRAP_FILE_PATH)), - Paths.get(niFiProperties.getProperty(MINIFI_LOG_DIRECTORY), niFiProperties.getProperty(MINIFI_APP_LOG_FILE)), - Paths.get(niFiProperties.getProperty(MINIFI_LOG_DIRECTORY), niFiProperties.getProperty(MINIFI_BOOTSTRAP_LOG_FILE))) + Paths.get(niFiProperties.getProperty(MiNiFiProperties.NIFI_MINIFI_FLOW_CONFIG.getKey())), + Paths.get(niFiProperties.getProperty(MiNiFiProperties.MINIFI_BOOTSTRAP_FILE_PATH)), + Paths.get(niFiProperties.getProperty(MiNiFiProperties.MINIFI_LOG_DIRECTORY), niFiProperties.getProperty(MiNiFiProperties.MINIFI_APP_LOG_FILE)), + Paths.get(niFiProperties.getProperty(MiNiFiProperties.MINIFI_LOG_DIRECTORY), niFiProperties.getProperty(MiNiFiProperties.MINIFI_BOOTSTRAP_LOG_FILE))) .filter(Files::exists) .filter(Files::isRegularFile) .collect(toList()); diff --git a/minifi/minifi-nar-bundles/minifi-framework-bundle/minifi-framework/minifi-framework-core/src/main/java/org/apache/nifi/minifi/c2/command/UpdatePropertiesPropertyProvider.java b/minifi/minifi-nar-bundles/minifi-framework-bundle/minifi-framework/minifi-framework-core/src/main/java/org/apache/nifi/minifi/c2/command/UpdatePropertiesPropertyProvider.java index 7faece8daf..50fab4fddd 100644 --- a/minifi/minifi-nar-bundles/minifi-framework-bundle/minifi-framework/minifi-framework-core/src/main/java/org/apache/nifi/minifi/c2/command/UpdatePropertiesPropertyProvider.java +++ b/minifi/minifi-nar-bundles/minifi-framework-bundle/minifi-framework/minifi-framework-core/src/main/java/org/apache/nifi/minifi/c2/command/UpdatePropertiesPropertyProvider.java @@ -17,7 +17,7 @@ package org.apache.nifi.minifi.c2.command; -import static org.apache.nifi.minifi.MiNiFiProperties.PROPERTIES_BY_KEY; +import static org.apache.nifi.minifi.commons.api.MiNiFiProperties.sortedPropertiesByKey; import java.io.File; import java.io.FileInputStream; @@ -29,7 +29,7 @@ import java.util.Map; import java.util.Properties; import java.util.stream.Collectors; import org.apache.nifi.c2.client.service.operation.OperandPropertiesProvider; -import org.apache.nifi.minifi.MiNiFiProperties; +import org.apache.nifi.minifi.commons.api.MiNiFiProperties; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -48,7 +48,7 @@ public class UpdatePropertiesPropertyProvider implements OperandPropertiesProvid public Map getProperties() { Map bootstrapProperties = getBootstrapProperties(); - LinkedHashSet updatableProperties = PROPERTIES_BY_KEY.values() + LinkedHashSet updatableProperties = sortedPropertiesByKey().values() .stream() .filter(property -> !property.isSensitive()) .filter(MiNiFiProperties::isModifiable) diff --git a/minifi/minifi-nar-bundles/minifi-framework-bundle/minifi-framework/minifi-framework-core/src/test/java/org/apache/nifi/minifi/c2/command/DefaultUpdateConfigurationStrategyTest.java b/minifi/minifi-nar-bundles/minifi-framework-bundle/minifi-framework/minifi-framework-core/src/test/java/org/apache/nifi/minifi/c2/command/DefaultUpdateConfigurationStrategyTest.java new file mode 100644 index 0000000000..82fee33fa2 --- /dev/null +++ b/minifi/minifi-nar-bundles/minifi-framework-bundle/minifi-framework/minifi-framework-core/src/test/java/org/apache/nifi/minifi/c2/command/DefaultUpdateConfigurationStrategyTest.java @@ -0,0 +1,175 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.nifi.minifi.c2.command; + +import static java.nio.charset.StandardCharsets.UTF_8; +import static java.nio.file.Files.exists; +import static java.nio.file.Files.newOutputStream; +import static java.nio.file.StandardOpenOption.CREATE; +import static java.nio.file.StandardOpenOption.TRUNCATE_EXISTING; +import static java.nio.file.StandardOpenOption.WRITE; +import static org.apache.nifi.minifi.commons.api.MiNiFiConstants.BACKUP_EXTENSION; +import static org.apache.nifi.minifi.commons.api.MiNiFiConstants.RAW_EXTENSION; +import static org.junit.jupiter.api.Assertions.assertArrayEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.Mockito.doThrow; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.File; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.zip.GZIPInputStream; +import java.util.zip.GZIPOutputStream; +import org.apache.commons.io.FilenameUtils; +import org.apache.nifi.c2.client.service.operation.UpdateConfigurationStrategy; +import org.apache.nifi.controller.FlowController; +import org.apache.nifi.controller.flow.FlowManager; +import org.apache.nifi.groups.ProcessGroup; +import org.apache.nifi.minifi.commons.service.FlowEnrichService; +import org.apache.nifi.services.FlowService; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.junit.jupiter.api.io.TempDir; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; + +@ExtendWith(MockitoExtension.class) +public class DefaultUpdateConfigurationStrategyTest { + + private static final String FLOW_CONFIG_FILE_NAME = "flow.config.gz"; + + private static final byte[] ORIGINAL_RAW_FLOW_CONFIG_CONTENT = "original_raw_content".getBytes(UTF_8); + private static final byte[] ORIGINAL_ENRICHED_FLOW_CONFIG_CONTENT = "original_enriched_content".getBytes(UTF_8); + private static final byte[] NEW_RAW_FLOW_CONFIG_CONTENT = "new_raw_content".getBytes(UTF_8); + private static final byte[] NEW_ENRICHED_FLOW_CONFIG_CONTENT = "new_enriched_content".getBytes(UTF_8); + + @TempDir + private File tempDir; + + @Mock + private FlowController mockFlowController; + @Mock + private FlowService mockFlowService; + @Mock + private FlowEnrichService mockFlowEnrichService; + @Mock + private FlowManager mockFlowManager; + @Mock + private ProcessGroup mockProcessGroup; + + private Path flowConfigurationFile; + private Path backupFlowConfigurationFile; + private Path rawFlowConfigurationFile; + private Path backupRawFlowConfigurationFile; + + private UpdateConfigurationStrategy testUpdateConfigurationStrategy; + + @BeforeEach + public void setup() throws IOException { + flowConfigurationFile = Path.of(tempDir.getAbsolutePath(), FLOW_CONFIG_FILE_NAME).toAbsolutePath(); + backupFlowConfigurationFile = Path.of(flowConfigurationFile + BACKUP_EXTENSION); + String flowConfigurationFileBaseName = FilenameUtils.getBaseName(flowConfigurationFile.toString()); + rawFlowConfigurationFile = flowConfigurationFile.getParent().resolve(flowConfigurationFileBaseName + RAW_EXTENSION); + backupRawFlowConfigurationFile = flowConfigurationFile.getParent().resolve(flowConfigurationFileBaseName + BACKUP_EXTENSION + RAW_EXTENSION); + + testUpdateConfigurationStrategy = new DefaultUpdateConfigurationStrategy(mockFlowController, mockFlowService, mockFlowEnrichService, flowConfigurationFile.toString()); + + writeGzipFile(flowConfigurationFile, ORIGINAL_ENRICHED_FLOW_CONFIG_CONTENT); + writePlainTextFile(rawFlowConfigurationFile, ORIGINAL_RAW_FLOW_CONFIG_CONTENT); + } + + @Test + public void testFlowIsUpdatedAndBackupsAreClearedUp() throws IOException { + // given + when(mockFlowEnrichService.enrichFlow(NEW_RAW_FLOW_CONFIG_CONTENT)).thenReturn(NEW_ENRICHED_FLOW_CONFIG_CONTENT); + when(mockFlowController.getFlowManager()).thenReturn(mockFlowManager); + when(mockFlowManager.getRootGroup()).thenReturn(mockProcessGroup); + + // when + boolean result = testUpdateConfigurationStrategy.update(NEW_RAW_FLOW_CONFIG_CONTENT); + + //then + assertTrue(result); + assertTrue(exists(flowConfigurationFile)); + assertTrue(exists(rawFlowConfigurationFile)); + assertArrayEquals(NEW_ENRICHED_FLOW_CONFIG_CONTENT, readGzipFile(flowConfigurationFile)); + assertArrayEquals(NEW_RAW_FLOW_CONFIG_CONTENT, readPlainTextFile(rawFlowConfigurationFile)); + assertFalse(exists(backupFlowConfigurationFile)); + assertFalse(exists(backupRawFlowConfigurationFile)); + verify(mockFlowService, times(1)).load(null); + verify(mockFlowController, times(1)).onFlowInitialized(true); + verify(mockProcessGroup, times(1)).startProcessing(); + } + + @Test + public void testFlowIsRevertedInCaseOfAnyErrorAndBackupsAreClearedUp() throws IOException { + // given + when(mockFlowEnrichService.enrichFlow(NEW_RAW_FLOW_CONFIG_CONTENT)).thenReturn(NEW_ENRICHED_FLOW_CONFIG_CONTENT); + when(mockFlowController.getFlowManager()).thenReturn(mockFlowManager); + when(mockFlowManager.getRootGroup()).thenReturn(mockProcessGroup); + doThrow(new IOException()).when(mockFlowService).load(null); + + // when + boolean result = testUpdateConfigurationStrategy.update(NEW_RAW_FLOW_CONFIG_CONTENT); + + //then + assertFalse(result); + assertTrue(exists(flowConfigurationFile)); + assertTrue(exists(rawFlowConfigurationFile)); + assertArrayEquals(ORIGINAL_ENRICHED_FLOW_CONFIG_CONTENT, readGzipFile(flowConfigurationFile)); + assertArrayEquals(ORIGINAL_RAW_FLOW_CONFIG_CONTENT, readPlainTextFile(rawFlowConfigurationFile)); + assertFalse(exists(backupFlowConfigurationFile)); + assertFalse(exists(backupRawFlowConfigurationFile)); + verify(mockFlowService, times(1)).load(null); + verify(mockFlowController, times(0)).onFlowInitialized(true); + verify(mockProcessGroup, times(0)).startProcessing(); + } + + private void writeGzipFile(Path path, byte[] content) throws IOException { + try (ByteArrayInputStream inputStream = new ByteArrayInputStream(content); + OutputStream outputStream = new GZIPOutputStream(newOutputStream(path, WRITE, CREATE, TRUNCATE_EXISTING))) { + inputStream.transferTo(outputStream); + } + } + + private byte[] readGzipFile(Path path) throws IOException { + try (InputStream inputStream = new GZIPInputStream(Files.newInputStream(path)); + ByteArrayOutputStream outputStream = new ByteArrayOutputStream()) { + inputStream.transferTo(outputStream); + outputStream.flush(); + return outputStream.toByteArray(); + } + } + + private void writePlainTextFile(Path path, byte[] content) throws IOException { + Files.write(path, content, WRITE, CREATE, TRUNCATE_EXISTING); + } + + private byte[] readPlainTextFile(Path path) throws IOException { + return Files.readAllBytes(path); + } +} diff --git a/minifi/minifi-nar-bundles/minifi-framework-bundle/minifi-framework/minifi-framework-core/src/test/java/org/apache/nifi/minifi/c2/command/PropertiesPersisterTest.java b/minifi/minifi-nar-bundles/minifi-framework-bundle/minifi-framework/minifi-framework-core/src/test/java/org/apache/nifi/minifi/c2/command/PropertiesPersisterTest.java index e1d68df805..91961dbd56 100644 --- a/minifi/minifi-nar-bundles/minifi-framework-bundle/minifi-framework/minifi-framework-core/src/test/java/org/apache/nifi/minifi/c2/command/PropertiesPersisterTest.java +++ b/minifi/minifi-nar-bundles/minifi-framework-bundle/minifi-framework/minifi-framework-core/src/test/java/org/apache/nifi/minifi/c2/command/PropertiesPersisterTest.java @@ -17,7 +17,7 @@ package org.apache.nifi.minifi.c2.command; -import static org.apache.nifi.minifi.MiNiFiProperties.C2_ENABLE; +import static org.apache.nifi.minifi.commons.api.MiNiFiProperties.C2_ENABLE; import static org.apache.nifi.minifi.c2.command.UpdatePropertiesPropertyProvider.AVAILABLE_PROPERTIES; import static org.apache.nifi.minifi.commons.api.MiNiFiConstants.BOOTSTRAP_UPDATED_FILE_NAME; import static org.junit.jupiter.api.Assertions.assertEquals; diff --git a/minifi/minifi-nar-bundles/minifi-framework-bundle/minifi-framework/minifi-framework-core/src/test/java/org/apache/nifi/minifi/c2/command/UpdatePropertiesPropertyProviderTest.java b/minifi/minifi-nar-bundles/minifi-framework-bundle/minifi-framework/minifi-framework-core/src/test/java/org/apache/nifi/minifi/c2/command/UpdatePropertiesPropertyProviderTest.java index 6923e29734..306a184d25 100644 --- a/minifi/minifi-nar-bundles/minifi-framework-bundle/minifi-framework/minifi-framework-core/src/test/java/org/apache/nifi/minifi/c2/command/UpdatePropertiesPropertyProviderTest.java +++ b/minifi/minifi-nar-bundles/minifi-framework-bundle/minifi-framework/minifi-framework-core/src/test/java/org/apache/nifi/minifi/c2/command/UpdatePropertiesPropertyProviderTest.java @@ -17,11 +17,11 @@ package org.apache.nifi.minifi.c2.command; -import static org.apache.nifi.minifi.MiNiFiProperties.GRACEFUL_SHUTDOWN_SECOND; -import static org.apache.nifi.minifi.MiNiFiProperties.JAVA; -import static org.apache.nifi.minifi.MiNiFiProperties.NIFI_MINIFI_SECURITY_KEYSTORE_PASSWD; -import static org.apache.nifi.minifi.MiNiFiProperties.PROPERTIES_BY_KEY; +import static org.apache.nifi.minifi.commons.api.MiNiFiProperties.GRACEFUL_SHUTDOWN_SECOND; +import static org.apache.nifi.minifi.commons.api.MiNiFiProperties.JAVA; +import static org.apache.nifi.minifi.commons.api.MiNiFiProperties.NIFI_MINIFI_SECURITY_KEYSTORE_PASSWD; import static org.apache.nifi.minifi.c2.command.UpdatePropertiesPropertyProvider.AVAILABLE_PROPERTIES; +import static org.apache.nifi.minifi.commons.api.MiNiFiProperties.sortedPropertiesByKey; import static org.junit.jupiter.api.Assertions.assertEquals; import java.io.File; @@ -32,7 +32,7 @@ import java.util.LinkedHashSet; import java.util.Map; import java.util.Properties; import java.util.stream.Collectors; -import org.apache.nifi.minifi.MiNiFiProperties; +import org.apache.nifi.minifi.commons.api.MiNiFiProperties; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.io.TempDir; @@ -77,7 +77,7 @@ class UpdatePropertiesPropertyProviderTest { } private static LinkedHashSet getUpdatableProperties(Properties props) { - return PROPERTIES_BY_KEY.values() + return sortedPropertiesByKey().values() .stream() .filter(property -> !property.isSensitive()) .filter(MiNiFiProperties::isModifiable) diff --git a/minifi/minifi-nar-bundles/minifi-framework-bundle/minifi-framework/minifi-resources/pom.xml b/minifi/minifi-nar-bundles/minifi-framework-bundle/minifi-framework/minifi-resources/pom.xml index bd9c699a23..d0c0659d89 100644 --- a/minifi/minifi-nar-bundles/minifi-framework-bundle/minifi-framework/minifi-resources/pom.xml +++ b/minifi/minifi-nar-bundles/minifi-framework-bundle/minifi-framework/minifi-resources/pom.xml @@ -47,6 +47,15 @@ limitations under the License. + + org.apache.rat + apache-rat-plugin + + + **/flow.json.raw + + + diff --git a/minifi/minifi-nar-bundles/minifi-framework-bundle/minifi-framework/minifi-resources/src/main/resources/conf/bootstrap.conf b/minifi/minifi-nar-bundles/minifi-framework-bundle/minifi-framework/minifi-resources/src/main/resources/conf/bootstrap.conf index d263c9b28f..9edc0998e8 100644 --- a/minifi/minifi-nar-bundles/minifi-framework-bundle/minifi-framework/minifi-resources/src/main/resources/conf/bootstrap.conf +++ b/minifi/minifi-nar-bundles/minifi-framework-bundle/minifi-framework/minifi-resources/src/main/resources/conf/bootstrap.conf @@ -29,79 +29,6 @@ conf.dir=./conf # How long to wait after telling MiNiFi to shutdown before explicitly killing the Process graceful.shutdown.seconds=20 -# The location for the configuration file -# When running as a Windows service use the full path to the file -nifi.minifi.config=./conf/config.yml - -# Security Properties # -# These properties take precedence over any equivalent properties specified in config.yml # -nifi.minifi.security.keystore= -nifi.minifi.security.keystoreType= -nifi.minifi.security.keystorePasswd= -nifi.minifi.security.keyPasswd= -nifi.minifi.security.truststore= -nifi.minifi.security.truststoreType= -nifi.minifi.security.truststorePasswd= -nifi.minifi.security.ssl.protocol= - -nifi.minifi.sensitive.props.key= -nifi.minifi.sensitive.props.algorithm= - -# Provenance Reporting Properties # -# These properties take precedence over any equivalent properties specified in the config.yml # -nifi.minifi.provenance.reporting.comment= -nifi.minifi.provenance.reporting.scheduling.strategy= -nifi.minifi.provenance.reporting.scheduling.period= -nifi.minifi.provenance.reporting.destination.url= -nifi.minifi.provenance.reporting.input.port.name= -nifi.minifi.provenance.reporting.instance.url= -nifi.minifi.provenance.reporting.batch.size= -nifi.minifi.provenance.reporting.communications.timeout= - -# Ignore all processor SSL controller services and use parent minifi SSL instead -nifi.minifi.flow.use.parent.ssl=false - -# Notifiers to use for the associated agent, comma separated list of class names -#nifi.minifi.notifier.ingestors=org.apache.nifi.minifi.bootstrap.configuration.ingestors.RestChangeIngestor -#nifi.minifi.notifier.ingestors=org.apache.nifi.minifi.bootstrap.configuration.ingestors.PullHttpChangeIngestor - -# File change notifier configuration - -# Path of the file to monitor for changes. When these occur, the FileChangeNotifier, if configured, will begin the configuration reloading process -#nifi.minifi.notifier.ingestors.file.config.path= -# How frequently the file specified by 'nifi.minifi.notifier.file.config.path' should be evaluated for changes. -#nifi.minifi.notifier.ingestors.file.polling.period.seconds=5 - -# Rest change notifier configuration - -# Port on which the Jetty server will bind to, keep commented for a random open port -#nifi.minifi.notifier.ingestors.receive.http.port=8338 - -#Pull HTTP change notifier configuration - -# Hostname on which to pull configurations from -#nifi.minifi.notifier.ingestors.pull.http.hostname=localhost -# Port on which to pull configurations from -#nifi.minifi.notifier.ingestors.pull.http.port=4567 -# Path to pull configurations from -#nifi.minifi.notifier.ingestors.pull.http.path=/c2/config -# Query string to pull configurations with -#nifi.minifi.notifier.ingestors.pull.http.query=class=raspi3 -# Period on which to pull configurations from, defaults to 5 minutes if commented out -#nifi.minifi.notifier.ingestors.pull.http.period.ms=300000 - -# Periodic Status Reporters to use for the associated agent, comma separated list of class names -#nifi.minifi.status.reporter.components=org.apache.nifi.minifi.bootstrap.status.reporters.StatusLogger - -# Periodic Status Logger configuration - -# The FlowStatus query to submit to the MiNiFi instance -#nifi.minifi.status.reporter.log.query=instance:health,bulletins -# The log level at which the status will be logged -#nifi.minifi.status.reporter.log.level=INFO -# The period (in milliseconds) at which to log the status -#nifi.minifi.status.reporter.log.period=60000 - # Disable JSR 199 so that we can use JSP's without running a JDK java.arg.1=-Dorg.apache.jasper.compiler.disablejsr199=true @@ -119,18 +46,51 @@ java.arg.6=-Djava.protocol.handler.pkgs=sun.net.www.protocol # Sets the provider of SecureRandom to /dev/urandom to prevent blocking on VMs java.arg.7=-Djava.security.egd=file:/dev/urandom - -# The G1GC is still considered experimental but has proven to be very advantageous in providing great -# performance without significant "stop-the-world" delays. -#java.arg.13=-XX:+UseG1GC +# Garbage Collector Algorithm +java.arg.13=-XX:+UseG1GC #Set headless mode by default java.arg.14=-Djava.awt.headless=true -# MiNiFi Command & Control Configuration -# C2 Properties +# Enable Remote Debugging +#java.arg.debug=-agentlib:jdwp=transport=dt_socket,server=y,suspend=n,address=8000 + + +### MiNiFi Flow Config +nifi.minifi.flow.config=./conf/flow.json.gz +#nifi.minifi.flow.max.concurrent.threads=1 + + +### MiNiFi Security Properties +nifi.minifi.security.keystore= +nifi.minifi.security.keystoreType= +nifi.minifi.security.keystorePasswd= +nifi.minifi.security.keyPasswd= +nifi.minifi.security.truststore= +nifi.minifi.security.truststoreType= +nifi.minifi.security.truststorePasswd= +nifi.minifi.security.ssl.protocol= + +# Ignore all processor SSL controller services and use parent minifi SSL instead +nifi.minifi.flow.use.parent.ssl=false + + +### MiNiFi Sensitive Properties Settings +nifi.minifi.sensitive.props.key= +nifi.minifi.sensitive.props.algorithm= + + +### MiNiFi Command & Control (C2) Configuration # Enabling C2 Uncomment each of the following options #c2.enable=true +## heartbeat in milliseconds +#c2.agent.heartbeat.period=5000 +## define parameters about your agent +#c2.agent.class= +# Optional. Defaults to a hardware based unique identifier +#c2.agent.identifier= +# If set to false heartbeat won't contain the manifest. Defaults to true. +#c2.full.heartbeat=false ## define protocol parameters # DEPRECATED: c2.rest.url and c2.rest.url.ack are deprecated in favor of c2.rest.path.* properties and are target to be removed in future release # The absolute url of the C2 server's heartbeat endpoint, eg.: http://localhost/c2-server/api/heartbeat @@ -148,17 +108,11 @@ java.arg.14=-Djava.awt.headless=true #c2.rest.connectionTimeout=5 sec #c2.rest.readTimeout=5 sec #c2.rest.callTimeout=10 sec -## heartbeat in milliseconds -#c2.agent.heartbeat.period=5000 -## define parameters about your agent -#c2.agent.class= +# Comma separated list of HTTP headers, eg: Accept:application/json +#c2.rest.http.headers=Accept:application/json #c2.config.directory=./conf #c2.runtime.manifest.identifier=minifi #c2.runtime.type=minifi-java -# Optional. Defaults to a hardware based unique identifier -#c2.agent.identifier= -# If set to false heartbeat won't contain the manifest. Defaults to true. -#c2.full.heartbeat=false # Directory for storing assets downloaded via C2 update/asset command #c2.asset.directory=./asset ## Define TLS security properties for C2 communications @@ -169,3 +123,88 @@ java.arg.14=-Djava.awt.headless=true #c2.security.keystore.password= #c2.security.keystore.type=JKS #c2.request.compression=none + + +### MiNiFi Notifiers For Flow Updates +# If C2 is enabled these should be disabled +# Notifiers to use for the associated agent, comma separated list of class names, choose from the options below +#nifi.minifi.notifier.ingestors= + +# File change notifier configuration - org.apache.nifi.minifi.bootstrap.configuration.ingestors.FileChangeIngestor +# Path of the file to monitor for changes. When these occur, the FileChangeNotifier, if configured, will begin the configuration reloading process +#nifi.minifi.notifier.ingestors.file.config.path= +# How frequently the file specified by 'nifi.minifi.notifier.file.config.path' should be evaluated for changes. +#nifi.minifi.notifier.ingestors.file.polling.period.seconds=5 + +# Rest change notifier configuration - org.apache.nifi.minifi.bootstrap.configuration.ingestors.RestChangeIngestor +# Port on which the Jetty server will bind to, keep commented for a random open port +#nifi.minifi.notifier.ingestors.receive.http.port=8338 + +#Pull HTTP change notifier configuration - org.apache.nifi.minifi.bootstrap.configuration.ingestors.PullHttpChangeIngestor +# Hostname on which to pull configurations from +#nifi.minifi.notifier.ingestors.pull.http.hostname=localhost +# Port on which to pull configurations from +#nifi.minifi.notifier.ingestors.pull.http.port=4567 +# Path to pull configurations from +#nifi.minifi.notifier.ingestors.pull.http.path=/c2/config +# Query string to pull configurations with +#nifi.minifi.notifier.ingestors.pull.http.query=class=raspi3 +# Period on which to pull configurations from, defaults to 5 minutes if commented out +#nifi.minifi.notifier.ingestors.pull.http.period.ms=300000 +# Comma separated list of HTTP headers, eg: Accept:application/json,Accept:application/yaml +#nifi.minifi.notifier.ingestors.pull.http.headers= + + +### MiNiFi Status Reporter +# Periodic Status Reporters to use for the associated agent, comma separated list of class names +#nifi.minifi.status.reporter.components=org.apache.nifi.minifi.bootstrap.status.reporters.StatusLogger +# Periodic Status Logger configuration +# The FlowStatus query to submit to the MiNiFi instance +#nifi.minifi.status.reporter.log.query=instance:health,bulletins +# The log level at which the status will be logged +#nifi.minifi.status.reporter.log.level=INFO +# The period (in milliseconds) at which to log the status +#nifi.minifi.status.reporter.log.period=60000 + + +### MiNiFi Provenance Reporting Properties +#nifi.minifi.provenance.reporting.comment= +#nifi.minifi.provenance.reporting.scheduling.strategy= +#nifi.minifi.provenance.reporting.scheduling.period= +#nifi.minifi.provenance.reporting.destination.url= +#nifi.minifi.provenance.reporting.input.port.name= +#nifi.minifi.provenance.reporting.instance.url= +#nifi.minifi.provenance.reporting.batch.size= +#nifi.minifi.provenance.reporting.communications.timeout= +#nifi.minifi.provenance.reporting.compress.events= + + +### NiFi Framework Properties +# Core Properties +nifi.flowcontroller.graceful.shutdown.period=10 sec +nifi.flowservice.writedelay.interval=500 ms +nifi.administrative.yield.duration=30 sec +nifi.variable.registry.properties= +nifi.bored.yield.duration=10 millis +# FlowFile Repository +nifi.flowfile.repository.implementation=org.apache.nifi.controller.repository.WriteAheadFlowFileRepository +nifi.flowfile.repository.checkpoint.interval=20 secs +nifi.flowfile.repository.always.sync=false +# Content Repository +nifi.content.repository.implementation=org.apache.nifi.controller.repository.FileSystemRepository +nifi.content.claim.max.appendable.size=50 KB +nifi.content.repository.archive.max.retention.period=7 days +nifi.content.repository.archive.max.usage.percentage=50% +nifi.content.repository.archive.enabled=false +# Provenance Repository +nifi.provenance.repository.implementation=org.apache.nifi.provenance.NoOpProvenanceRepository +nifi.provenance.repository.rollover.time= +nifi.provenance.repository.index.shard.size= +nifi.provenance.repository.max.storage.size= +nifi.provenance.repository.max.storage.time= +# Components Status Repository +nifi.components.status.repository.implementation=org.apache.nifi.controller.status.history.VolatileComponentStatusRepository +nifi.components.status.snapshot.frequency=1 min +# Swap Manager +nifi.swap.manager.implementation=org.apache.nifi.controller.FileSystemSwapManager +nifi.queue.swap.threshold=20000 diff --git a/minifi/minifi-nar-bundles/minifi-framework-bundle/minifi-framework/minifi-resources/src/main/resources/conf/config.yml b/minifi/minifi-nar-bundles/minifi-framework-bundle/minifi-framework/minifi-resources/src/main/resources/conf/config.yml deleted file mode 100644 index 43d2e1af80..0000000000 --- a/minifi/minifi-nar-bundles/minifi-framework-bundle/minifi-framework/minifi-resources/src/main/resources/conf/config.yml +++ /dev/null @@ -1,62 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the \"License\"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an \"AS IS\" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -MiNiFi Config Version: 3 -Flow Controller: - name: MiNiFi Flow - comment: '' -Core Properties: - flow controller graceful shutdown period: 10 sec - flow service write delay interval: 500 ms - administrative yield duration: 30 sec - bored yield duration: 10 millis - max concurrent threads: 1 -FlowFile Repository: - partitions: 256 - checkpoint interval: 2 mins - always sync: false - Swap: - threshold: 20000 - in period: 5 sec - in threads: 1 - out period: 5 sec - out threads: 4 -Content Repository: - content claim max appendable size: 10 MB - content claim max flow files: 100 - always sync: false -Provenance Repository: - provenance rollover time: 1 min -Component Status Repository: - buffer size: 1440 - snapshot frequency: 1 min -Security Properties: - keystore: '' - keystore type: '' - keystore password: '' - key password: '' - truststore: '' - truststore type: '' - truststore password: '' - ssl protocol: '' - Sensitive Props: - key: '' - algorithm: NIFI_PBKDF2_AES_GCM_256 -Processors: [] -Process Groups: [] -Funnels: [] -Connections: [] -Remote Process Groups: [] -NiFi Properties Overrides: {} diff --git a/minifi/minifi-nar-bundles/minifi-framework-bundle/minifi-framework/minifi-resources/src/main/resources/conf/flow.json.raw b/minifi/minifi-nar-bundles/minifi-framework-bundle/minifi-framework/minifi-resources/src/main/resources/conf/flow.json.raw new file mode 100644 index 0000000000..fde8a180da --- /dev/null +++ b/minifi/minifi-nar-bundles/minifi-framework-bundle/minifi-framework/minifi-resources/src/main/resources/conf/flow.json.raw @@ -0,0 +1,38 @@ +{ + "encodingVersion": { + "majorVersion": 2, + "minorVersion": 0 + }, + "maxTimerDrivenThreadCount": 1, + "maxEventDrivenThreadCount": 1, + "registries": [], + "parameterContexts": [], + "parameterProviders": [], + "controllerServices": [], + "reportingTasks": [], + "templates": [], + "rootGroup": { + "name": "MiNiFi Flow", + "comments": "", + "position": { + "x": 0.0, + "y": 0.0 + }, + "processGroups": [], + "remoteProcessGroups": [], + "processors": [], + "inputPorts": [], + "outputPorts": [], + "connections": [], + "labels": [], + "funnels": [], + "controllerServices": [], + "variables": {}, + "defaultFlowFileExpiration": "0 sec", + "defaultBackPressureObjectThreshold": 10000, + "defaultBackPressureDataSizeThreshold": "1 GB", + "componentType": "PROCESS_GROUP", + "flowFileConcurrency": "UNBOUNDED", + "flowFileOutboundPolicy": "STREAM_WHEN_AVAILABLE" + } +} \ No newline at end of file diff --git a/minifi/minifi-nar-bundles/minifi-framework-bundle/minifi-framework/minifi-runtime/src/main/java/org/apache/nifi/minifi/MiNiFi.java b/minifi/minifi-nar-bundles/minifi-framework-bundle/minifi-framework/minifi-runtime/src/main/java/org/apache/nifi/minifi/MiNiFi.java index c34cd8d109..0c30875880 100644 --- a/minifi/minifi-nar-bundles/minifi-framework-bundle/minifi-framework/minifi-runtime/src/main/java/org/apache/nifi/minifi/MiNiFi.java +++ b/minifi/minifi-nar-bundles/minifi-framework-bundle/minifi-framework/minifi-runtime/src/main/java/org/apache/nifi/minifi/MiNiFi.java @@ -181,18 +181,15 @@ public class MiNiFi { final AtomicInteger occurrencesOutOfRange = new AtomicInteger(0); final AtomicInteger occurrences = new AtomicInteger(0); - final Runnable command = new Runnable() { - @Override - public void run() { - final long curMillis = System.currentTimeMillis(); - final long difference = curMillis - lastTriggerMillis.get(); - final long millisOff = Math.abs(difference - 2000L); - occurrences.incrementAndGet(); - if (millisOff > 500L) { - occurrencesOutOfRange.incrementAndGet(); - } - lastTriggerMillis.set(curMillis); + final Runnable command = () -> { + final long curMillis = System.currentTimeMillis(); + final long difference = curMillis - lastTriggerMillis.get(); + final long millisOff = Math.abs(difference - 2000L); + occurrences.incrementAndGet(); + if (millisOff > 500L) { + occurrencesOutOfRange.incrementAndGet(); } + lastTriggerMillis.set(curMillis); }; final ScheduledFuture future = service.scheduleWithFixedDelay(command, 2000L, 2000L, TimeUnit.MILLISECONDS); diff --git a/minifi/minifi-nar-bundles/minifi-framework-bundle/minifi-framework/minifi-server/src/main/java/org/apache/nifi/minifi/StandardMiNiFiServer.java b/minifi/minifi-nar-bundles/minifi-framework-bundle/minifi-framework/minifi-server/src/main/java/org/apache/nifi/minifi/StandardMiNiFiServer.java index b33c9e4709..4263ac809f 100644 --- a/minifi/minifi-nar-bundles/minifi-framework-bundle/minifi-framework/minifi-server/src/main/java/org/apache/nifi/minifi/StandardMiNiFiServer.java +++ b/minifi/minifi-nar-bundles/minifi-framework-bundle/minifi-framework/minifi-server/src/main/java/org/apache/nifi/minifi/StandardMiNiFiServer.java @@ -14,19 +14,28 @@ * See the License for the specific language governing permissions and * limitations under the License. */ + package org.apache.nifi.minifi; -import org.apache.nifi.headless.HeadlessNiFiServer; -import org.apache.nifi.minifi.bootstrap.BootstrapListener; -import org.apache.nifi.minifi.c2.C2NifiClientService; -import org.apache.nifi.minifi.commons.status.FlowStatusReport; -import org.apache.nifi.minifi.status.StatusConfigReporter; -import org.apache.nifi.minifi.status.StatusRequestException; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; +import static java.util.Optional.ofNullable; +import static org.apache.commons.lang3.StringUtils.EMPTY; import java.io.IOException; import java.io.UncheckedIOException; +import java.util.Optional; +import org.apache.commons.lang3.StringUtils; +import org.apache.nifi.controller.FlowSerializationStrategy; +import org.apache.nifi.headless.HeadlessNiFiServer; +import org.apache.nifi.minifi.bootstrap.BootstrapListener; +import org.apache.nifi.minifi.c2.C2NifiClientService; +import org.apache.nifi.minifi.commons.api.MiNiFiProperties; +import org.apache.nifi.minifi.commons.status.FlowStatusReport; +import org.apache.nifi.minifi.status.StatusConfigReporter; +import org.apache.nifi.minifi.status.StatusRequestException; +import org.apache.nifi.util.NiFiProperties; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + /** * */ @@ -78,16 +87,35 @@ public class StandardMiNiFiServer extends HeadlessNiFiServer implements MiNiFiSe } } + @Override + protected FlowSerializationStrategy getFlowSerializationStrategy() { + return FlowSerializationStrategy.WRITE_JSON_ONLY; + } + private void initC2() { if (Boolean.parseBoolean(getNiFiProperties().getProperty(MiNiFiProperties.C2_ENABLE.getKey(), MiNiFiProperties.C2_ENABLE.getDefaultValue()))) { - logger.info("C2 enabled, creating a C2 client instance"); - c2NifiClientService = new C2NifiClientService(getNiFiProperties(), getFlowController(), bootstrapListener); + NiFiProperties niFiProperties = getNiFiProperties(); + enabledFlowIngestors(niFiProperties).ifPresentOrElse( + flowIngestors -> { + logger.warn("Due to enabled flow ingestor(s) [{}] C2 client is not created. Please disable flow ingestors when using C2", flowIngestors); + c2NifiClientService = null; + }, + () -> { + logger.info("C2 enabled, creating a C2 client instance"); + c2NifiClientService = new C2NifiClientService(niFiProperties, getFlowController(), bootstrapListener, getFlowService()); + }); } else { logger.debug("C2 Property [{}] missing or disabled: C2 client not created", MiNiFiProperties.C2_ENABLE.getKey()); c2NifiClientService = null; } } + private Optional enabledFlowIngestors(NiFiProperties niFiProperties) { + return ofNullable(niFiProperties.getProperty(MiNiFiProperties.NIFI_MINIFI_NOTIFIER_INGESTORS.getKey(), EMPTY)) + .map(String::trim) + .filter(StringUtils::isNotBlank); + } + private void startHeartbeat() { if (c2NifiClientService != null) { c2NifiClientService.start(); @@ -106,7 +134,7 @@ public class StandardMiNiFiServer extends HeadlessNiFiServer implements MiNiFiSe bootstrapListener = new BootstrapListener(this, port); bootstrapListener.start(); - } catch(IOException e){ + } catch (IOException e) { throw new UncheckedIOException("Failed to start MiNiFi because of Bootstrap listener initialization error", e); } catch (NumberFormatException e) { throw new RuntimeException("Failed to start MiNiFi because system property '" + BOOTSTRAP_PORT_PROPERTY + "' is not a valid integer in the range 1 - 65535"); diff --git a/minifi/minifi-toolkit/minifi-toolkit-assembly/README.md b/minifi/minifi-toolkit/minifi-toolkit-assembly/README.md index fb8bfc4e1d..17fe2f564f 100644 --- a/minifi/minifi-toolkit/minifi-toolkit-assembly/README.md +++ b/minifi/minifi-toolkit/minifi-toolkit-assembly/README.md @@ -47,10 +47,18 @@ After downloading the binary and extracting it, to run the MiNiFi Toolkit Conver java org.apache.nifi.minifi.toolkit.configuration.ConfigMain options Valid commands include: - transform: Transform template xml into MiNiFi config YAML - validate: Validate config YAML + transform-yml: Transforms legacy MiNiFi flow config YAML into MiNiFi flow config JSON -Note: Currently MiNiFi does not support external Input Ports or Output Ports. Any templates that contain these will fail transformation. +## Example +- You have an older version of MiNiFi located in . +- You would like upgrade to the latest version of MiNiFi. You downloaded and extracted the latest MiNiFi into . +- Run the following command to migrate the flow and the bootstrap config +``` +./config.sh transform-yml /conf/config.yml /conf/bootstrap.conf /conf/flow.json.raw /conf/bootstrap.conf +``` + +## Note +It's not guaranteed in all circumstances that the migration will result in a correct flow. For example if a processor's configuration has changed between version, the conversion tool won't be aware of this, and will use the deprecated property names. You will need to fix such issues manually. ## Getting Help If you have questions, you can reach out to our mailing list: dev@nifi.apache.org diff --git a/minifi/minifi-toolkit/minifi-toolkit-configuration/pom.xml b/minifi/minifi-toolkit/minifi-toolkit-configuration/pom.xml index b0f2689ae2..a6269e6700 100644 --- a/minifi/minifi-toolkit/minifi-toolkit-configuration/pom.xml +++ b/minifi/minifi-toolkit/minifi-toolkit-configuration/pom.xml @@ -28,7 +28,11 @@ limitations under the License. org.apache.nifi.minifi - minifi-commons-schema + minifi-commons-api + + + org.apache.nifi.minifi + minifi-toolkit-schema ${project.version} diff --git a/minifi/minifi-toolkit/minifi-toolkit-configuration/src/main/java/org/apache/nifi/minifi/toolkit/configuration/ConfigMain.java b/minifi/minifi-toolkit/minifi-toolkit-configuration/src/main/java/org/apache/nifi/minifi/toolkit/configuration/ConfigMain.java index 1be6480a0f..91ad1114ad 100644 --- a/minifi/minifi-toolkit/minifi-toolkit-configuration/src/main/java/org/apache/nifi/minifi/toolkit/configuration/ConfigMain.java +++ b/minifi/minifi-toolkit/minifi-toolkit-configuration/src/main/java/org/apache/nifi/minifi/toolkit/configuration/ConfigMain.java @@ -17,307 +17,37 @@ package org.apache.nifi.minifi.toolkit.configuration; -import com.fasterxml.jackson.annotation.JsonInclude; -import com.fasterxml.jackson.databind.DeserializationFeature; -import com.fasterxml.jackson.databind.ObjectMapper; -import com.fasterxml.jackson.module.jaxb.JaxbAnnotationIntrospector; -import org.apache.nifi.minifi.commons.schema.ConfigSchema; -import org.apache.nifi.minifi.commons.schema.common.ConvertableSchema; -import org.apache.nifi.minifi.commons.schema.exception.SchemaLoaderException; -import org.apache.nifi.minifi.commons.schema.serialization.SchemaLoader; -import org.apache.nifi.minifi.commons.schema.serialization.SchemaSaver; -import org.apache.nifi.minifi.toolkit.configuration.dto.ConfigSchemaFunction; -import org.apache.nifi.minifi.toolkit.configuration.dto.FlowSnippetDTOEnricher; -import org.apache.nifi.minifi.toolkit.configuration.registry.NiFiRegConfigSchemaFunction; -import org.apache.nifi.minifi.toolkit.configuration.registry.VersionedProcessGroupEnricher; -import org.apache.nifi.registry.flow.VersionedFlowSnapshot; -import org.apache.nifi.web.api.dto.TemplateDTO; +import static org.apache.nifi.minifi.toolkit.configuration.json.TransformYamlCommandFactory.TRANSFORM_YML; -import javax.xml.bind.JAXBContext; -import javax.xml.bind.JAXBException; import java.io.FileInputStream; -import java.io.FileNotFoundException; import java.io.FileOutputStream; -import java.io.IOException; -import java.io.InputStream; -import java.io.OutputStream; import java.util.Map; -import java.util.TreeMap; -import java.util.function.BiFunction; import java.util.function.Function; +import org.apache.nifi.minifi.toolkit.configuration.json.TransformYamlCommandFactory; public class ConfigMain { public static final int ERR_INVALID_ARGS = 1; - public static final int ERR_UNABLE_TO_OPEN_OUTPUT = 2; public static final int ERR_UNABLE_TO_OPEN_INPUT = 3; public static final int ERR_UNABLE_TO_READ_TEMPLATE = 4; public static final int ERR_UNABLE_TO_PARSE_CONFIG = 6; - public static final int ERR_INVALID_CONFIG = 7; - public static final int ERR_UNABLE_TO_CLOSE_CONFIG = 8; public static final int ERR_UNABLE_TO_SAVE_CONFIG = 9; public static final int SUCCESS = 0; - public static final String TRANSFORM = "transform"; - public static final String TRANSFORM_VFS = "transform-vfs"; - public static final String VALIDATE = "validate"; - public static final String UPGRADE = "upgrade"; - public static final String THERE_ARE_VALIDATION_ERRORS_WITH_THE_TEMPLATE_STILL_OUTPUTTING_YAML_BUT_IT_WILL_NEED_TO_BE_EDITED = - "There are validation errors with the template, still outputting YAML but it will need to be edited."; - private final Map commandMap; private final PathInputStreamFactory pathInputStreamFactory; private final PathOutputStreamFactory pathOutputStreamFactory; - public ConfigMain() { - this(FileInputStream::new, FileOutputStream::new); - } - public ConfigMain(PathInputStreamFactory pathInputStreamFactory, PathOutputStreamFactory pathOutputStreamFactory) { this.pathInputStreamFactory = pathInputStreamFactory; this.pathOutputStreamFactory = pathOutputStreamFactory; - this.commandMap = createCommandMap(); + this.commandMap = Map.of( + TRANSFORM_YML, new TransformYamlCommandFactory(pathInputStreamFactory, pathOutputStreamFactory).create() + ); } public static void main(String[] args) { - System.exit(new ConfigMain().execute(args)); - } - - public static void printValidateUsage() { - System.out.println("Validate Usage:"); - System.out.println(); - System.out.println(" validate INPUT_FILE"); - System.out.println(); - } - - public int validate(String[] args) { - if (args.length != 2) { - printValidateUsage(); - return ERR_INVALID_ARGS; - } - try (InputStream inputStream = pathInputStreamFactory.create(args[1])) { - try { - return loadAndPrintValidationErrors(inputStream, (configSchema, valid) -> { - if (valid) { - return SUCCESS; - } else { - return ERR_INVALID_CONFIG; - } - }); - } catch (IOException | SchemaLoaderException e) { - return handleErrorLoadingConfiguration(e, ConfigMain::printValidateUsage); - } - } catch (FileNotFoundException e) { - return handleErrorOpeningInput(args[1], ConfigMain::printValidateUsage, e); - } catch (IOException e) { - handleErrorClosingInput(e); - return ERR_UNABLE_TO_CLOSE_CONFIG; - } - } - - public static void printTransformUsage() { - System.out.println("Transform Usage:"); - System.out.println(); - System.out.println(" transform INPUT_FILE OUTPUT_FILE"); - System.out.println(); - } - - public static void printUpgradeUsage() { - System.out.println("Upgrade Usage:"); - System.out.println(); - System.out.println(" upgrade INPUT_FILE OUTPUT_FILE"); - System.out.println(); - } - - public static ConfigSchema transformTemplateToSchema(InputStream source) throws JAXBException, IOException { - try { - TemplateDTO templateDTO = (TemplateDTO) JAXBContext.newInstance(TemplateDTO.class).createUnmarshaller().unmarshal(source); - - FlowSnippetDTOEnricher enricher = new FlowSnippetDTOEnricher(); - enricher.enrich(templateDTO.getSnippet(), templateDTO.getEncodingVersion()); - - ConfigSchema configSchema = new ConfigSchemaFunction().apply(templateDTO); - return configSchema; - } finally { - source.close(); - } - } - - public static ConfigSchema transformVersionedFlowSnapshotToSchema(InputStream source) throws IOException { - try { - final ObjectMapper objectMapper = new ObjectMapper(); - objectMapper.setSerializationInclusion(JsonInclude.Include.NON_NULL); - objectMapper.setAnnotationIntrospector(new JaxbAnnotationIntrospector(objectMapper.getTypeFactory())); - objectMapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false); - - final VersionedFlowSnapshot versionedFlowSnapshot = objectMapper.readValue(source, VersionedFlowSnapshot.class); - return transformVersionedFlowSnapshotToSchema(versionedFlowSnapshot); - } finally { - source.close(); - } - } - - public static ConfigSchema transformVersionedFlowSnapshotToSchema(VersionedFlowSnapshot versionedFlowSnapshot) { - VersionedProcessGroupEnricher enricher = new VersionedProcessGroupEnricher(); - enricher.enrich(versionedFlowSnapshot.getFlowContents()); - - ConfigSchema configSchema = new NiFiRegConfigSchemaFunction().apply(versionedFlowSnapshot); - return configSchema; - } - - - public int upgrade(String[] args) { - if (args.length != 3) { - printUpgradeUsage(); - return ERR_INVALID_ARGS; - } - - ConfigSchema currentSchema = null; - try (InputStream inputStream = pathInputStreamFactory.create(args[1])) { - try { - currentSchema = loadAndPrintValidationErrors(inputStream, (configSchema, valid) -> { - if (!valid) { - System.out.println(THERE_ARE_VALIDATION_ERRORS_WITH_THE_TEMPLATE_STILL_OUTPUTTING_YAML_BUT_IT_WILL_NEED_TO_BE_EDITED); - System.out.println(); - } - return configSchema; - }); - } catch (IOException | SchemaLoaderException e) { - return handleErrorLoadingConfiguration(e, ConfigMain::printUpgradeUsage); - } - } catch (FileNotFoundException e) { - return handleErrorOpeningInput(args[1], ConfigMain::printUpgradeUsage, e); - } catch (IOException e) { - handleErrorClosingInput(e); - } - - try (OutputStream fileOutputStream = pathOutputStreamFactory.create(args[2])) { - try { - SchemaSaver.saveConfigSchema(currentSchema, fileOutputStream); - } catch (IOException e) { - return handleErrorSavingCofiguration(e); - } - } catch (FileNotFoundException e) { - return handleErrorOpeningOutput(args[2], ConfigMain::printUpgradeUsage, e); - } catch (IOException e) { - handleErrorClosingOutput(e); - } - - return SUCCESS; - } - - public T loadAndPrintValidationErrors(InputStream inputStream, BiFunction resultHandler) throws IOException, SchemaLoaderException { - ConvertableSchema configSchema = SchemaLoader.loadConvertableSchemaFromYaml(inputStream); - boolean valid = true; - if (!configSchema.isValid()) { - System.out.println("Found the following errors when parsing the configuration according to its version. (" + configSchema.getVersion() + ")"); - configSchema.getValidationIssues().forEach(s -> System.out.println(s)); - System.out.println(); - valid = false; - configSchema.clearValidationIssues(); - } else { - System.out.println("No errors found when parsing configuration according to its version. (" + configSchema.getVersion() + ")"); - } - - ConfigSchema currentSchema = configSchema.convert(); - if (!currentSchema.isValid()) { - System.out.println("Found the following errors when converting configuration to latest version. (" + ConfigSchema.CONFIG_VERSION + ")"); - currentSchema.getValidationIssues().forEach(s -> System.out.println(s)); - System.out.println(); - valid = false; - } else if (configSchema.getVersion() == currentSchema.getVersion()) { - System.out.println("Configuration was already latest version (" + ConfigSchema.CONFIG_VERSION + ") so no conversion was needed."); - } else { - System.out.println("No errors found when converting configuration to latest version. (" + ConfigSchema.CONFIG_VERSION + ")"); - } - return resultHandler.apply(currentSchema, valid); - } - - public int transform(String[] args) { - if (args.length != 3) { - printTransformUsage(); - return ERR_INVALID_ARGS; - } - - ConfigSchema configSchema = null; - try (InputStream inputStream = pathInputStreamFactory.create(args[1])) { - try { - // both transform commands call this method, so determine which transform is being done - if (TRANSFORM_VFS.equals(args[0])) { - configSchema = transformVersionedFlowSnapshotToSchema(inputStream); - } else { - configSchema = transformTemplateToSchema(inputStream); - } - - if (!configSchema.isValid()) { - System.out.println(THERE_ARE_VALIDATION_ERRORS_WITH_THE_TEMPLATE_STILL_OUTPUTTING_YAML_BUT_IT_WILL_NEED_TO_BE_EDITED); - configSchema.getValidationIssues().forEach(System.out::println); - System.out.println(); - } else { - System.out.println("No validation errors found in converted configuration."); - } - } catch (JAXBException e) { - System.out.println("Error reading template. (" + e + ")"); - System.out.println(); - printTransformUsage(); - return ERR_UNABLE_TO_READ_TEMPLATE; - } - } catch (FileNotFoundException e) { - return handleErrorOpeningInput(args[1], ConfigMain::printTransformUsage, e); - } catch (IOException e) { - handleErrorClosingInput(e); - } - - try (OutputStream fileOutputStream = pathOutputStreamFactory.create(args[2])) { - try { - SchemaSaver.saveConfigSchema(configSchema, fileOutputStream); - } catch (IOException e) { - return handleErrorSavingCofiguration(e); - } - } catch (FileNotFoundException e) { - return handleErrorOpeningOutput(args[2], ConfigMain::printTransformUsage, e); - } catch (IOException e) { - handleErrorClosingOutput(e); - } - - return SUCCESS; - } - - protected void handleErrorClosingOutput(IOException e) { - System.out.println("Error closing output. (" + e + ")"); - System.out.println(); - } - - protected void handleErrorClosingInput(IOException e) { - System.out.println("Error closing input. (" + e + ")"); - System.out.println(); - } - - protected int handleErrorOpeningInput(String fileName, Runnable usagePrinter, FileNotFoundException e) { - System.out.println("Unable to open file " + fileName + " for reading. (" + e + ")"); - System.out.println(); - usagePrinter.run(); - return ERR_UNABLE_TO_OPEN_INPUT; - } - - protected int handleErrorOpeningOutput(String fileName, Runnable usagePrinter, FileNotFoundException e) { - System.out.println("Unable to open file " + fileName + " for writing. (" + e + ")"); - System.out.println(); - usagePrinter.run(); - return ERR_UNABLE_TO_OPEN_OUTPUT; - } - - protected int handleErrorLoadingConfiguration(Exception e, Runnable usagePrinter) { - System.out.println("Unable to load configuration. (" + e + ")"); - System.out.println(); - usagePrinter.run(); - return ERR_UNABLE_TO_PARSE_CONFIG; - } - - protected int handleErrorSavingCofiguration(IOException e) { - System.out.println("Unable to save configuration: " + e); - System.out.println(); - return ERR_UNABLE_TO_SAVE_CONFIG; + System.exit(new ConfigMain(FileInputStream::new, FileOutputStream::new).execute(args)); } public int execute(String[] args) { @@ -328,23 +58,15 @@ public class ConfigMain { return commandMap.get(args[0].toLowerCase()).function.apply(args); } - public Map createCommandMap() { - Map result = new TreeMap<>(); - result.put(TRANSFORM, new Command(this::transform, "Transform template xml into MiNiFi config YAML")); - result.put(TRANSFORM_VFS, new Command(this::transform, "Transform VersionedFlowSnapshot JSON into MiNiFi config YAML")); - result.put(VALIDATE, new Command(this::validate, "Validate config YAML")); - result.put(UPGRADE, new Command(this::upgrade, "Upgrade config YAML to current version (" + ConfigSchema.CONFIG_VERSION + ")")); - return result; - } - - public void printUsage() { + private void printUsage() { System.out.println("Usage:"); System.out.println(); System.out.println("Valid commands include:"); commandMap.forEach((s, command) -> System.out.println(s + ": " + command.description)); } - public class Command { + + public static class Command { private final Function function; private final String description; diff --git a/minifi/minifi-c2/minifi-c2-provider/minifi-c2-provider-nifi-rest/src/main/java/org/apache/nifi/minifi/c2/provider/nifi/rest/TemplatesIteratorException.java b/minifi/minifi-toolkit/minifi-toolkit-configuration/src/main/java/org/apache/nifi/minifi/toolkit/configuration/ConfigTransformException.java similarity index 69% rename from minifi/minifi-c2/minifi-c2-provider/minifi-c2-provider-nifi-rest/src/main/java/org/apache/nifi/minifi/c2/provider/nifi/rest/TemplatesIteratorException.java rename to minifi/minifi-toolkit/minifi-toolkit-configuration/src/main/java/org/apache/nifi/minifi/toolkit/configuration/ConfigTransformException.java index af22474f2d..0e30102b63 100644 --- a/minifi/minifi-c2/minifi-c2-provider/minifi-c2-provider-nifi-rest/src/main/java/org/apache/nifi/minifi/c2/provider/nifi/rest/TemplatesIteratorException.java +++ b/minifi/minifi-toolkit/minifi-toolkit-configuration/src/main/java/org/apache/nifi/minifi/toolkit/configuration/ConfigTransformException.java @@ -15,17 +15,18 @@ * limitations under the License. */ -package org.apache.nifi.minifi.c2.provider.nifi.rest; +package org.apache.nifi.minifi.toolkit.configuration; -import java.io.IOException; +public class ConfigTransformException extends Exception { -public class TemplatesIteratorException extends RuntimeException { - public TemplatesIteratorException(IOException cause) { - super(cause); + private final int errorCode; + + public ConfigTransformException(String message, int errorCode, Throwable throwable) { + super(message, throwable); + this.errorCode = errorCode; } - @Override - public synchronized IOException getCause() { - return (IOException) super.getCause(); + public int getErrorCode() { + return errorCode; } -} +} \ No newline at end of file diff --git a/minifi/minifi-toolkit/minifi-toolkit-configuration/src/main/java/org/apache/nifi/minifi/toolkit/configuration/PathInputStreamFactory.java b/minifi/minifi-toolkit/minifi-toolkit-configuration/src/main/java/org/apache/nifi/minifi/toolkit/configuration/PathInputStreamFactory.java index e88715716d..518a2812b1 100644 --- a/minifi/minifi-toolkit/minifi-toolkit-configuration/src/main/java/org/apache/nifi/minifi/toolkit/configuration/PathInputStreamFactory.java +++ b/minifi/minifi-toolkit/minifi-toolkit-configuration/src/main/java/org/apache/nifi/minifi/toolkit/configuration/PathInputStreamFactory.java @@ -20,6 +20,7 @@ package org.apache.nifi.minifi.toolkit.configuration; import java.io.FileNotFoundException; import java.io.InputStream; +@FunctionalInterface public interface PathInputStreamFactory { InputStream create(String path) throws FileNotFoundException; } diff --git a/minifi/minifi-toolkit/minifi-toolkit-configuration/src/main/java/org/apache/nifi/minifi/toolkit/configuration/PathOutputStreamFactory.java b/minifi/minifi-toolkit/minifi-toolkit-configuration/src/main/java/org/apache/nifi/minifi/toolkit/configuration/PathOutputStreamFactory.java index ea13376296..800024a7c3 100644 --- a/minifi/minifi-toolkit/minifi-toolkit-configuration/src/main/java/org/apache/nifi/minifi/toolkit/configuration/PathOutputStreamFactory.java +++ b/minifi/minifi-toolkit/minifi-toolkit-configuration/src/main/java/org/apache/nifi/minifi/toolkit/configuration/PathOutputStreamFactory.java @@ -20,6 +20,7 @@ package org.apache.nifi.minifi.toolkit.configuration; import java.io.FileNotFoundException; import java.io.OutputStream; +@FunctionalInterface public interface PathOutputStreamFactory { OutputStream create(String path) throws FileNotFoundException; } diff --git a/minifi/minifi-toolkit/minifi-toolkit-configuration/src/main/java/org/apache/nifi/minifi/toolkit/configuration/dto/ConfigSchemaFunction.java b/minifi/minifi-toolkit/minifi-toolkit-configuration/src/main/java/org/apache/nifi/minifi/toolkit/configuration/dto/ConfigSchemaFunction.java deleted file mode 100644 index 221c4aa1a1..0000000000 --- a/minifi/minifi-toolkit/minifi-toolkit-configuration/src/main/java/org/apache/nifi/minifi/toolkit/configuration/dto/ConfigSchemaFunction.java +++ /dev/null @@ -1,147 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.nifi.minifi.toolkit.configuration.dto; - -import org.apache.nifi.minifi.commons.schema.ConfigSchema; -import org.apache.nifi.minifi.commons.schema.ConnectionSchema; -import org.apache.nifi.minifi.commons.schema.ControllerServiceSchema; -import org.apache.nifi.minifi.commons.schema.FunnelSchema; -import org.apache.nifi.minifi.commons.schema.PortSchema; -import org.apache.nifi.minifi.commons.schema.ProcessGroupSchema; -import org.apache.nifi.minifi.commons.schema.ProcessorSchema; -import org.apache.nifi.minifi.commons.schema.RemoteProcessGroupSchema; -import org.apache.nifi.minifi.commons.schema.common.CollectionUtil; -import org.apache.nifi.minifi.commons.schema.common.CommonPropertyKeys; -import org.apache.nifi.minifi.commons.schema.common.StringUtil; -import org.apache.nifi.web.api.dto.FlowSnippetDTO; -import org.apache.nifi.web.api.dto.TemplateDTO; - -import java.util.Comparator; -import java.util.HashMap; -import java.util.Map; -import java.util.function.Function; -import java.util.stream.Collectors; - -import static org.apache.nifi.minifi.commons.schema.common.CollectionUtil.nullToEmpty; -import static org.apache.nifi.minifi.commons.schema.common.CommonPropertyKeys.ID_KEY; -import static org.apache.nifi.minifi.commons.schema.common.CommonPropertyKeys.INPUT_PORTS_KEY; -import static org.apache.nifi.minifi.commons.schema.common.CommonPropertyKeys.NAME_KEY; -import static org.apache.nifi.minifi.commons.schema.common.CommonPropertyKeys.OUTPUT_PORTS_KEY; - -public class ConfigSchemaFunction implements Function { - private final FlowControllerSchemaFunction flowControllerSchemaFunction; - private final ProcessorSchemaFunction processorSchemaFunction; - private final ControllerServiceSchemaFunction controllerServiceSchemaFunction; - private final ConnectionSchemaFunction connectionSchemaFunction; - private final FunnelSchemaFunction funnelSchemaFunction; - private final RemoteProcessGroupSchemaFunction remoteProcessGroupSchemaFunction; - private final PortSchemaFunction inputPortSchemaFunction; - private final PortSchemaFunction outputPortSchemaFunction; - - public ConfigSchemaFunction() { - this(new FlowControllerSchemaFunction(), new ProcessorSchemaFunction(), new ConnectionSchemaFunction(), new FunnelSchemaFunction(), new RemoteProcessGroupSchemaFunction( - new RemotePortSchemaFunction()), new PortSchemaFunction(INPUT_PORTS_KEY), new PortSchemaFunction(OUTPUT_PORTS_KEY), new ControllerServiceSchemaFunction()); - } - - public ConfigSchemaFunction(FlowControllerSchemaFunction flowControllerSchemaFunction, ProcessorSchemaFunction processorSchemaFunction, ConnectionSchemaFunction connectionSchemaFunction, - FunnelSchemaFunction funnelSchemaFunction, RemoteProcessGroupSchemaFunction remoteProcessGroupSchemaFunction, PortSchemaFunction inputPortSchemaFunction, - PortSchemaFunction outputPortSchemaFunction, ControllerServiceSchemaFunction controllerServiceSchemaFunction) { - this.flowControllerSchemaFunction = flowControllerSchemaFunction; - this.processorSchemaFunction = processorSchemaFunction; - this.connectionSchemaFunction = connectionSchemaFunction; - this.funnelSchemaFunction = funnelSchemaFunction; - this.remoteProcessGroupSchemaFunction = remoteProcessGroupSchemaFunction; - this.inputPortSchemaFunction = inputPortSchemaFunction; - this.outputPortSchemaFunction = outputPortSchemaFunction; - this.controllerServiceSchemaFunction = controllerServiceSchemaFunction; - } - - @Override - public ConfigSchema apply(TemplateDTO templateDTO) { - Map map = new HashMap<>(); - - map.put(CommonPropertyKeys.FLOW_CONTROLLER_PROPS_KEY, flowControllerSchemaFunction.apply(templateDTO).toMap()); - - FlowSnippetDTO snippet = templateDTO.getSnippet(); - - addSnippet(map, snippet); - - return new ConfigSchema(map); - } - - protected void addSnippet(Map map, FlowSnippetDTO snippet) { - addSnippet(map, null, null, snippet); - } - - protected Map addSnippet(Map map, String id, String name, FlowSnippetDTO snippet) { - if (!StringUtil.isNullOrEmpty(id)) { - map.put(ID_KEY, id); - } - - if (!StringUtil.isNullOrEmpty(name)) { - map.put(NAME_KEY, name); - } - - map.put(CommonPropertyKeys.PROCESSORS_KEY, nullToEmpty(snippet.getProcessors()).stream() - .map(processorSchemaFunction) - .sorted(Comparator.comparing(ProcessorSchema::getName)) - .map(ProcessorSchema::toMap) - .collect(Collectors.toList())); - - map.put(CommonPropertyKeys.CONTROLLER_SERVICES_KEY, nullToEmpty(snippet.getControllerServices()).stream() - .map(controllerServiceSchemaFunction) - .sorted(Comparator.comparing(ControllerServiceSchema::getName)) - .map(ControllerServiceSchema::toMap) - .collect(Collectors.toList())); - - map.put(CommonPropertyKeys.CONNECTIONS_KEY, nullToEmpty(snippet.getConnections()).stream() - .map(connectionSchemaFunction) - .sorted(Comparator.comparing(ConnectionSchema::getName)) - .map(ConnectionSchema::toMap) - .collect(Collectors.toList())); - - map.put(CommonPropertyKeys.FUNNELS_KEY, CollectionUtil.nullToEmpty(snippet.getFunnels()).stream() - .map(funnelSchemaFunction) - .sorted(Comparator.comparing(FunnelSchema::getId)) - .map(FunnelSchema::toMap) - .collect(Collectors.toList())); - - map.put(CommonPropertyKeys.REMOTE_PROCESS_GROUPS_KEY, nullToEmpty(snippet.getRemoteProcessGroups()).stream() - .map(remoteProcessGroupSchemaFunction) - .sorted(Comparator.comparing(RemoteProcessGroupSchema::getName)) - .map(RemoteProcessGroupSchema::toMap) - .collect(Collectors.toList())); - - map.put(INPUT_PORTS_KEY, nullToEmpty(snippet.getInputPorts()).stream() - .map(inputPortSchemaFunction) - .sorted(Comparator.comparing(PortSchema::getName)) - .map(PortSchema::toMap) - .collect(Collectors.toList())); - - map.put(OUTPUT_PORTS_KEY, nullToEmpty(snippet.getOutputPorts()).stream() - .map(outputPortSchemaFunction) - .sorted(Comparator.comparing(PortSchema::getName)) - .map(PortSchema::toMap) - .collect(Collectors.toList())); - - map.put(ProcessGroupSchema.PROCESS_GROUPS_KEY, nullToEmpty(snippet.getProcessGroups()).stream() - .map(p -> addSnippet(new HashMap<>(), p.getId(), p.getName(), p.getContents())).collect(Collectors.toList())); - - return map; - } -} diff --git a/minifi/minifi-toolkit/minifi-toolkit-configuration/src/main/java/org/apache/nifi/minifi/toolkit/configuration/dto/ConnectionSchemaFunction.java b/minifi/minifi-toolkit/minifi-toolkit-configuration/src/main/java/org/apache/nifi/minifi/toolkit/configuration/dto/ConnectionSchemaFunction.java deleted file mode 100644 index 7acab41554..0000000000 --- a/minifi/minifi-toolkit/minifi-toolkit-configuration/src/main/java/org/apache/nifi/minifi/toolkit/configuration/dto/ConnectionSchemaFunction.java +++ /dev/null @@ -1,63 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.nifi.minifi.toolkit.configuration.dto; - -import org.apache.nifi.connectable.ConnectableType; -import org.apache.nifi.minifi.commons.schema.ConnectionSchema; -import org.apache.nifi.web.api.dto.ConnectionDTO; - -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.function.Function; -import java.util.stream.Collectors; - -import static org.apache.nifi.minifi.commons.schema.common.CollectionUtil.nullToEmpty; -import static org.apache.nifi.minifi.commons.schema.common.CommonPropertyKeys.CONNECTIONS_KEY; -import static org.apache.nifi.minifi.commons.schema.common.CommonPropertyKeys.ID_KEY; -import static org.apache.nifi.minifi.commons.schema.common.CommonPropertyKeys.NAME_KEY; - -public class ConnectionSchemaFunction implements Function { - @Override - public ConnectionSchema apply(ConnectionDTO connectionDTO) { - Map map = new HashMap<>(); - map.put(ID_KEY, connectionDTO.getId()); - map.put(NAME_KEY, connectionDTO.getName()); - map.put(ConnectionSchema.SOURCE_ID_KEY, connectionDTO.getSource().getId()); - Set selectedRelationships = nullToEmpty(connectionDTO.getSelectedRelationships()); - map.put(ConnectionSchema.SOURCE_RELATIONSHIP_NAMES_KEY, selectedRelationships.stream().sorted().collect(Collectors.toList())); - map.put(ConnectionSchema.DESTINATION_ID_KEY, connectionDTO.getDestination().getId()); - - map.put(ConnectionSchema.MAX_WORK_QUEUE_SIZE_KEY, connectionDTO.getBackPressureObjectThreshold()); - map.put(ConnectionSchema.MAX_WORK_QUEUE_DATA_SIZE_KEY, connectionDTO.getBackPressureDataSizeThreshold()); - map.put(ConnectionSchema.FLOWFILE_EXPIRATION__KEY, connectionDTO.getFlowFileExpiration()); - List queuePrioritizers = nullToEmpty(connectionDTO.getPrioritizers()); - if (queuePrioritizers.size() > 0) { - map.put(ConnectionSchema.QUEUE_PRIORITIZER_CLASS_KEY, queuePrioritizers.get(0)); - } - ConnectionSchema connectionSchema = new ConnectionSchema(map); - if (ConnectableType.FUNNEL.name().equals(connectionDTO.getSource().getType())) { - connectionSchema.addValidationIssue("Connection " + connectionDTO.getName() + " has type " + ConnectableType.FUNNEL.name() + " which is not supported by MiNiFi"); - } - if (queuePrioritizers.size() > 1) { - connectionSchema.addValidationIssue(ConnectionSchema.QUEUE_PRIORITIZER_CLASS_KEY, CONNECTIONS_KEY, " has more than one queue prioritizer"); - } - return connectionSchema; - } -} diff --git a/minifi/minifi-toolkit/minifi-toolkit-configuration/src/main/java/org/apache/nifi/minifi/toolkit/configuration/dto/ControllerServiceSchemaFunction.java b/minifi/minifi-toolkit/minifi-toolkit-configuration/src/main/java/org/apache/nifi/minifi/toolkit/configuration/dto/ControllerServiceSchemaFunction.java deleted file mode 100644 index c76751cbaa..0000000000 --- a/minifi/minifi-toolkit/minifi-toolkit-configuration/src/main/java/org/apache/nifi/minifi/toolkit/configuration/dto/ControllerServiceSchemaFunction.java +++ /dev/null @@ -1,51 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.nifi.minifi.toolkit.configuration.dto; - -import org.apache.nifi.minifi.commons.schema.ControllerServiceSchema; -import org.apache.nifi.web.api.dto.ControllerServiceDTO; - -import java.util.HashMap; -import java.util.Map; -import java.util.function.Function; - -import static org.apache.nifi.minifi.commons.schema.common.CollectionUtil.nullToEmpty; -import static org.apache.nifi.minifi.commons.schema.common.CommonPropertyKeys.ANNOTATION_DATA_KEY; -import static org.apache.nifi.minifi.commons.schema.common.CommonPropertyKeys.ID_KEY; -import static org.apache.nifi.minifi.commons.schema.common.CommonPropertyKeys.NAME_KEY; -import static org.apache.nifi.minifi.commons.schema.common.CommonPropertyKeys.PROPERTIES_KEY; -import static org.apache.nifi.minifi.commons.schema.common.CommonPropertyKeys.TYPE_KEY; - -public class ControllerServiceSchemaFunction implements Function { - @Override - public ControllerServiceSchema apply(ControllerServiceDTO controllerServiceDTO) { - Map map = new HashMap<>(); - map.put(NAME_KEY, controllerServiceDTO.getName()); - map.put(ID_KEY, controllerServiceDTO.getId()); - map.put(TYPE_KEY, controllerServiceDTO.getType()); - - map.put(PROPERTIES_KEY, new HashMap<>(nullToEmpty(controllerServiceDTO.getProperties()))); - - String annotationData = controllerServiceDTO.getAnnotationData(); - if(annotationData != null && !annotationData.isEmpty()) { - map.put(ANNOTATION_DATA_KEY, annotationData); - } - - return new ControllerServiceSchema(map); - } -} diff --git a/minifi/minifi-toolkit/minifi-toolkit-configuration/src/main/java/org/apache/nifi/minifi/toolkit/configuration/dto/FlowControllerSchemaFunction.java b/minifi/minifi-toolkit/minifi-toolkit-configuration/src/main/java/org/apache/nifi/minifi/toolkit/configuration/dto/FlowControllerSchemaFunction.java deleted file mode 100644 index d2ca1578f2..0000000000 --- a/minifi/minifi-toolkit/minifi-toolkit-configuration/src/main/java/org/apache/nifi/minifi/toolkit/configuration/dto/FlowControllerSchemaFunction.java +++ /dev/null @@ -1,36 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.nifi.minifi.toolkit.configuration.dto; - -import org.apache.nifi.minifi.commons.schema.FlowControllerSchema; -import org.apache.nifi.minifi.commons.schema.common.CommonPropertyKeys; -import org.apache.nifi.web.api.dto.TemplateDTO; - -import java.util.HashMap; -import java.util.Map; -import java.util.function.Function; - -public class FlowControllerSchemaFunction implements Function { - @Override - public FlowControllerSchema apply(TemplateDTO templateDTO) { - Map map = new HashMap<>(); - map.put(CommonPropertyKeys.NAME_KEY, templateDTO.getName()); - map.put(CommonPropertyKeys.COMMENT_KEY, templateDTO.getDescription()); - return new FlowControllerSchema(map); - } -} diff --git a/minifi/minifi-toolkit/minifi-toolkit-configuration/src/main/java/org/apache/nifi/minifi/toolkit/configuration/dto/FlowSnippetDTOEnricher.java b/minifi/minifi-toolkit/minifi-toolkit-configuration/src/main/java/org/apache/nifi/minifi/toolkit/configuration/dto/FlowSnippetDTOEnricher.java deleted file mode 100644 index 4795e12dd0..0000000000 --- a/minifi/minifi-toolkit/minifi-toolkit-configuration/src/main/java/org/apache/nifi/minifi/toolkit/configuration/dto/FlowSnippetDTOEnricher.java +++ /dev/null @@ -1,174 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.nifi.minifi.toolkit.configuration.dto; - -import org.apache.commons.lang3.StringUtils; -import org.apache.nifi.connectable.ConnectableType; -import org.apache.nifi.minifi.commons.schema.common.StringUtil; -import org.apache.nifi.web.api.dto.ComponentDTO; -import org.apache.nifi.web.api.dto.ConnectableDTO; -import org.apache.nifi.web.api.dto.ConnectionDTO; -import org.apache.nifi.web.api.dto.FlowSnippetDTO; -import org.apache.nifi.web.api.dto.PortDTO; -import org.apache.nifi.web.api.dto.ProcessGroupDTO; -import org.apache.nifi.web.api.dto.ProcessorDTO; -import org.apache.nifi.web.api.dto.RemoteProcessGroupContentsDTO; -import org.apache.nifi.web.api.dto.RemoteProcessGroupDTO; -import org.apache.nifi.web.api.dto.RemoteProcessGroupPortDTO; - -import java.util.ArrayList; -import java.util.Collection; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Objects; -import java.util.Optional; -import java.util.Set; -import java.util.function.Function; -import java.util.stream.Collectors; -import java.util.stream.Stream; - -import static org.apache.nifi.minifi.commons.schema.common.CollectionUtil.nullToEmpty; - -public class FlowSnippetDTOEnricher { - - public void enrich(FlowSnippetDTO flowSnippetDTO, final String encodingVersion) { - List allFlowSnippets = getAllFlowSnippets(flowSnippetDTO); - - Set remoteProcessGroups = getAll(allFlowSnippets, FlowSnippetDTO::getRemoteProcessGroups).collect(Collectors.toSet()); - - Map connectableNameMap = getAll(allFlowSnippets, FlowSnippetDTO::getProcessors).collect(Collectors.toMap(ComponentDTO::getId, ProcessorDTO::getName)); - Map rpgIdToTargetIdMap = new HashMap<>(); - - for (RemoteProcessGroupDTO remoteProcessGroupDTO : remoteProcessGroups) { - final RemoteProcessGroupContentsDTO contents = remoteProcessGroupDTO.getContents(); - final Set rpgInputPortDtos = nullToEmpty(contents.getInputPorts()); - final Set rpgOutputPortDtos = nullToEmpty(contents.getOutputPorts()); - - /* - * Templates created prior to version 1.0 of NiFi did not have an encoding-version specified - * There are no material changes to the flow representation that affect MiNiFi prior to the 1.2 encoding-version introduced in NiFi 1.5, so we avoid evaluating - */ - if (encodingVersion != null) { - switch (encodingVersion) { - // Perform a no-op for the first two versions of templates - case "1.0": - case "1.1": - break; - // Starting in 1.2, the way in which RPGs are mapped has changed - default: - // Map all port DTOs to their respective targetIds - rpgIdToTargetIdMap.putAll( - Stream.concat(rpgInputPortDtos.stream(), rpgOutputPortDtos.stream()) - .collect(Collectors.toMap(RemoteProcessGroupPortDTO::getId, RemoteProcessGroupPortDTO::getTargetId))); - break; - } - } - - addConnectables(connectableNameMap, rpgInputPortDtos, RemoteProcessGroupPortDTO::getId, RemoteProcessGroupPortDTO::getId); - addConnectables(connectableNameMap, rpgOutputPortDtos, RemoteProcessGroupPortDTO::getId, RemoteProcessGroupPortDTO::getId); - } - - - addConnectables(connectableNameMap, getAll(allFlowSnippets, FlowSnippetDTO::getInputPorts).collect(Collectors.toList()), PortDTO::getId, PortDTO::getName); - addConnectables(connectableNameMap, getAll(allFlowSnippets, FlowSnippetDTO::getOutputPorts).collect(Collectors.toList()), PortDTO::getId, PortDTO::getName); - - final Set connections = getAll(allFlowSnippets, FlowSnippetDTO::getConnections).collect(Collectors.toSet()); - - // Enrich connection endpoints using known names and overriding with targetIds for remote ports - for (ConnectionDTO connection : connections) { - setName(connectableNameMap, connection.getSource(), rpgIdToTargetIdMap); - setName(connectableNameMap, connection.getDestination(), rpgIdToTargetIdMap); - } - - // Override any ids that are for Remote Ports to use their target Ids where available - connections.stream() - .flatMap(connectionDTO -> Stream.of(connectionDTO.getSource(), connectionDTO.getDestination())) - .filter(connectable -> connectable.getType().equals(ConnectableType.REMOTE_OUTPUT_PORT.toString()) || connectable.getType().equals(ConnectableType.REMOTE_INPUT_PORT.toString())) - .forEach(connectable -> connectable.setId(Optional.ofNullable(rpgIdToTargetIdMap.get(connectable.getId())).orElse(connectable.getId()))); - - // Establish unique names for connections - for (ConnectionDTO connection : connections) { - if (StringUtil.isNullOrEmpty(connection.getName())) { - StringBuilder name = new StringBuilder(); - ConnectableDTO connectionSource = connection.getSource(); - name.append(determineValueForConnectable(connectionSource, rpgIdToTargetIdMap)); - - name.append("/"); - if (connection.getSelectedRelationships() != null && connection.getSelectedRelationships().size() > 0) { - name.append(connection.getSelectedRelationships().iterator().next()); - } - - name.append("/"); - ConnectableDTO connectionDestination = connection.getDestination(); - name.append(determineValueForConnectable(connectionDestination, rpgIdToTargetIdMap)); - - connection.setName(name.toString()); - } - } - nullToEmpty(flowSnippetDTO.getProcessGroups()).stream().map(ProcessGroupDTO::getContents).forEach(snippetDTO -> enrich(snippetDTO, encodingVersion)); - } - - private static String determineValueForConnectable(ConnectableDTO connectable, Map idOverrideMap) { - String connectionName = ""; - if (connectable != null) { - connectionName = connectable.getName(); - // If no name is specified, determine the appropriate id to use, preferring any overrides specified - if (StringUtils.isBlank(connectionName)) { - connectionName = idOverrideMap.containsKey(connectable.getId()) ? idOverrideMap.get(connectable.getId()) : connectable.getId(); - } - } - return connectionName; - } - - private static Stream getAll(List allFlowSnippets, Function> accessor) { - return allFlowSnippets.stream().flatMap(f -> accessor.apply(f).stream()).filter(Objects::nonNull); - } - - private static List getAllFlowSnippets(FlowSnippetDTO flowSnippetDTO) { - List result = new ArrayList<>(); - getAllFlowSnippets(flowSnippetDTO, result); - return result; - } - - private static void getAllFlowSnippets(FlowSnippetDTO flowSnippetDTO, List result) { - result.add(flowSnippetDTO); - nullToEmpty(flowSnippetDTO.getProcessGroups()).stream().map(ProcessGroupDTO::getContents).forEach(f -> getAllFlowSnippets(f, result)); - } - - private static void setName(Map connectableNameMap, ConnectableDTO connectableDTO, Map nameOverrides) { - if (connectableDTO != null) { - final String name = connectableNameMap.get(connectableDTO.getId()); - if (name != null) { - connectableDTO.setName(Optional.ofNullable(nameOverrides.get(connectableDTO.getId())).orElse(name)); - } - } - } - - private static void addConnectables(Map connectableNameMap, Collection hasIdAndNames, Function idGetter, Function nameGetter) { - if (hasIdAndNames != null) { - for (T hasIdAndName : hasIdAndNames) { - String id = idGetter.apply(hasIdAndName); - String name = nameGetter.apply(hasIdAndName); - if (!StringUtil.isNullOrEmpty(name)) { - connectableNameMap.put(id, name); - } - } - } - } - -} diff --git a/minifi/minifi-toolkit/minifi-toolkit-configuration/src/main/java/org/apache/nifi/minifi/toolkit/configuration/dto/FunnelSchemaFunction.java b/minifi/minifi-toolkit/minifi-toolkit-configuration/src/main/java/org/apache/nifi/minifi/toolkit/configuration/dto/FunnelSchemaFunction.java deleted file mode 100644 index a03b105030..0000000000 --- a/minifi/minifi-toolkit/minifi-toolkit-configuration/src/main/java/org/apache/nifi/minifi/toolkit/configuration/dto/FunnelSchemaFunction.java +++ /dev/null @@ -1,38 +0,0 @@ -/* - * - * * Licensed to the Apache Software Foundation (ASF) under one or more - * * contributor license agreements. See the NOTICE file distributed with - * * this work for additional information regarding copyright ownership. - * * The ASF licenses this file to You under the Apache License, Version 2.0 - * * (the "License"); you may not use this file except in compliance with - * * the License. You may obtain a copy of the License at - * * - * * http://www.apache.org/licenses/LICENSE-2.0 - * * - * * Unless required by applicable law or agreed to in writing, software - * * distributed under the License is distributed on an "AS IS" BASIS, - * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * * See the License for the specific language governing permissions and - * * limitations under the License. - * - */ - -package org.apache.nifi.minifi.toolkit.configuration.dto; - -import org.apache.nifi.minifi.commons.schema.FunnelSchema; -import org.apache.nifi.web.api.dto.FunnelDTO; - -import java.util.HashMap; -import java.util.Map; -import java.util.function.Function; - -import static org.apache.nifi.minifi.commons.schema.common.CommonPropertyKeys.ID_KEY; - -public class FunnelSchemaFunction implements Function { - @Override - public FunnelSchema apply(FunnelDTO funnelDTO) { - Map map = new HashMap<>(); - map.put(ID_KEY, funnelDTO.getId()); - return new FunnelSchema(map); - } -} \ No newline at end of file diff --git a/minifi/minifi-toolkit/minifi-toolkit-configuration/src/main/java/org/apache/nifi/minifi/toolkit/configuration/dto/PortSchemaFunction.java b/minifi/minifi-toolkit/minifi-toolkit-configuration/src/main/java/org/apache/nifi/minifi/toolkit/configuration/dto/PortSchemaFunction.java deleted file mode 100644 index 29efac3c85..0000000000 --- a/minifi/minifi-toolkit/minifi-toolkit-configuration/src/main/java/org/apache/nifi/minifi/toolkit/configuration/dto/PortSchemaFunction.java +++ /dev/null @@ -1,46 +0,0 @@ -/* - * - * * Licensed to the Apache Software Foundation (ASF) under one or more - * * contributor license agreements. See the NOTICE file distributed with - * * this work for additional information regarding copyright ownership. - * * The ASF licenses this file to You under the Apache License, Version 2.0 - * * (the "License"); you may not use this file except in compliance with - * * the License. You may obtain a copy of the License at - * * - * * http://www.apache.org/licenses/LICENSE-2.0 - * * - * * Unless required by applicable law or agreed to in writing, software - * * distributed under the License is distributed on an "AS IS" BASIS, - * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * * See the License for the specific language governing permissions and - * * limitations under the License. - * - */ - -package org.apache.nifi.minifi.toolkit.configuration.dto; - -import org.apache.nifi.minifi.commons.schema.PortSchema; -import org.apache.nifi.web.api.dto.PortDTO; - -import java.util.HashMap; -import java.util.Map; -import java.util.function.Function; - -import static org.apache.nifi.minifi.commons.schema.common.CommonPropertyKeys.ID_KEY; -import static org.apache.nifi.minifi.commons.schema.common.CommonPropertyKeys.NAME_KEY; - -public class PortSchemaFunction implements Function { - private final String wrapperName; - - public PortSchemaFunction(String wrapperName) { - this.wrapperName = wrapperName; - } - - @Override - public PortSchema apply(PortDTO portDTO) { - Map map = new HashMap<>(); - map.put(ID_KEY, portDTO.getId()); - map.put(NAME_KEY, portDTO.getName()); - return new PortSchema(map, wrapperName); - } -} diff --git a/minifi/minifi-toolkit/minifi-toolkit-configuration/src/main/java/org/apache/nifi/minifi/toolkit/configuration/dto/ProcessorSchemaFunction.java b/minifi/minifi-toolkit/minifi-toolkit-configuration/src/main/java/org/apache/nifi/minifi/toolkit/configuration/dto/ProcessorSchemaFunction.java deleted file mode 100644 index 2efaca8fdb..0000000000 --- a/minifi/minifi-toolkit/minifi-toolkit-configuration/src/main/java/org/apache/nifi/minifi/toolkit/configuration/dto/ProcessorSchemaFunction.java +++ /dev/null @@ -1,72 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.nifi.minifi.toolkit.configuration.dto; - -import org.apache.nifi.minifi.commons.schema.ProcessorSchema; -import org.apache.nifi.minifi.commons.schema.common.CommonPropertyKeys; -import org.apache.nifi.web.api.dto.ProcessorConfigDTO; -import org.apache.nifi.web.api.dto.ProcessorDTO; -import org.apache.nifi.web.api.dto.RelationshipDTO; - -import java.util.HashMap; -import java.util.Map; -import java.util.function.Function; -import java.util.stream.Collectors; - -import static org.apache.nifi.minifi.commons.schema.common.CollectionUtil.nullToEmpty; -import static org.apache.nifi.minifi.commons.schema.common.CommonPropertyKeys.ANNOTATION_DATA_KEY; -import static org.apache.nifi.minifi.commons.schema.common.CommonPropertyKeys.CLASS_KEY; -import static org.apache.nifi.minifi.commons.schema.common.CommonPropertyKeys.ID_KEY; -import static org.apache.nifi.minifi.commons.schema.common.CommonPropertyKeys.NAME_KEY; -import static org.apache.nifi.minifi.commons.schema.common.CommonPropertyKeys.PROPERTIES_KEY; -import static org.apache.nifi.minifi.commons.schema.common.CommonPropertyKeys.SCHEDULING_PERIOD_KEY; -import static org.apache.nifi.minifi.commons.schema.common.CommonPropertyKeys.SCHEDULING_STRATEGY_KEY; - -public class ProcessorSchemaFunction implements Function { - @Override - public ProcessorSchema apply(ProcessorDTO processorDTO) { - ProcessorConfigDTO processorDTOConfig = processorDTO.getConfig(); - - Map map = new HashMap<>(); - map.put(NAME_KEY, processorDTO.getName()); - map.put(ID_KEY, processorDTO.getId()); - map.put(CLASS_KEY, processorDTO.getType()); - map.put(SCHEDULING_STRATEGY_KEY, processorDTOConfig.getSchedulingStrategy()); - map.put(SCHEDULING_PERIOD_KEY, processorDTOConfig.getSchedulingPeriod()); - - map.put(CommonPropertyKeys.MAX_CONCURRENT_TASKS_KEY, processorDTOConfig.getConcurrentlySchedulableTaskCount()); - map.put(ProcessorSchema.PENALIZATION_PERIOD_KEY, processorDTOConfig.getPenaltyDuration()); - map.put(CommonPropertyKeys.YIELD_PERIOD_KEY, processorDTOConfig.getYieldDuration()); - Long runDurationMillis = processorDTOConfig.getRunDurationMillis(); - if (runDurationMillis != null) { - map.put(ProcessorSchema.RUN_DURATION_NANOS_KEY, runDurationMillis * 1000); - } - map.put(ProcessorSchema.AUTO_TERMINATED_RELATIONSHIPS_LIST_KEY, nullToEmpty(processorDTO.getRelationships()).stream() - .filter(RelationshipDTO::isAutoTerminate) - .map(RelationshipDTO::getName) - .collect(Collectors.toList())); - map.put(PROPERTIES_KEY, new HashMap<>(nullToEmpty(processorDTOConfig.getProperties()))); - - String annotationData = processorDTOConfig.getAnnotationData(); - if(annotationData != null && !annotationData.isEmpty()) { - map.put(ANNOTATION_DATA_KEY, annotationData); - } - - return new ProcessorSchema(map); - } -} diff --git a/minifi/minifi-toolkit/minifi-toolkit-configuration/src/main/java/org/apache/nifi/minifi/toolkit/configuration/dto/RemotePortSchemaFunction.java b/minifi/minifi-toolkit/minifi-toolkit-configuration/src/main/java/org/apache/nifi/minifi/toolkit/configuration/dto/RemotePortSchemaFunction.java deleted file mode 100644 index 889d7860df..0000000000 --- a/minifi/minifi-toolkit/minifi-toolkit-configuration/src/main/java/org/apache/nifi/minifi/toolkit/configuration/dto/RemotePortSchemaFunction.java +++ /dev/null @@ -1,46 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.nifi.minifi.toolkit.configuration.dto; - -import org.apache.commons.lang3.StringUtils; -import org.apache.nifi.minifi.commons.schema.RemotePortSchema; -import org.apache.nifi.minifi.commons.schema.common.CommonPropertyKeys; -import org.apache.nifi.web.api.dto.RemoteProcessGroupPortDTO; - -import java.util.HashMap; -import java.util.Map; -import java.util.function.Function; - -import static org.apache.nifi.minifi.commons.schema.common.CommonPropertyKeys.ID_KEY; -import static org.apache.nifi.minifi.commons.schema.common.CommonPropertyKeys.NAME_KEY; - -public class RemotePortSchemaFunction implements Function { - @Override - public RemotePortSchema apply(RemoteProcessGroupPortDTO remoteProcessGroupPortDTO) { - Map map = new HashMap<>(); - // If a targetId is specified, it takes precedence over the original id - final String targetId = remoteProcessGroupPortDTO.getTargetId(); - map.put(ID_KEY, StringUtils.isNotBlank(targetId) ? targetId : remoteProcessGroupPortDTO.getId()); - map.put(NAME_KEY, remoteProcessGroupPortDTO.getName()); - - map.put(CommonPropertyKeys.COMMENT_KEY, remoteProcessGroupPortDTO.getComments()); - map.put(CommonPropertyKeys.MAX_CONCURRENT_TASKS_KEY, remoteProcessGroupPortDTO.getConcurrentlySchedulableTaskCount()); - map.put(CommonPropertyKeys.USE_COMPRESSION_KEY, remoteProcessGroupPortDTO.getUseCompression()); - return new RemotePortSchema(map); - } -} diff --git a/minifi/minifi-toolkit/minifi-toolkit-configuration/src/main/java/org/apache/nifi/minifi/toolkit/configuration/dto/RemoteProcessGroupSchemaFunction.java b/minifi/minifi-toolkit/minifi-toolkit-configuration/src/main/java/org/apache/nifi/minifi/toolkit/configuration/dto/RemoteProcessGroupSchemaFunction.java deleted file mode 100644 index 03fbf5ec0c..0000000000 --- a/minifi/minifi-toolkit/minifi-toolkit-configuration/src/main/java/org/apache/nifi/minifi/toolkit/configuration/dto/RemoteProcessGroupSchemaFunction.java +++ /dev/null @@ -1,82 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.nifi.minifi.toolkit.configuration.dto; - -import org.apache.commons.lang3.StringUtils; -import org.apache.nifi.minifi.commons.schema.RemotePortSchema; -import org.apache.nifi.minifi.commons.schema.RemoteProcessGroupSchema; -import org.apache.nifi.minifi.commons.schema.common.CommonPropertyKeys; -import org.apache.nifi.web.api.dto.RemoteProcessGroupContentsDTO; -import org.apache.nifi.web.api.dto.RemoteProcessGroupDTO; -import org.apache.nifi.web.api.dto.RemoteProcessGroupPortDTO; - -import java.util.HashMap; -import java.util.Map; -import java.util.Set; -import java.util.function.Function; -import java.util.stream.Collectors; - -public class RemoteProcessGroupSchemaFunction implements Function { - private final RemotePortSchemaFunction remotePortSchemaFunction; - - public RemoteProcessGroupSchemaFunction(RemotePortSchemaFunction remotePortSchemaFunction) { - this.remotePortSchemaFunction = remotePortSchemaFunction; - } - - @Override - public RemoteProcessGroupSchema apply(RemoteProcessGroupDTO remoteProcessGroupDTO) { - Map map = new HashMap<>(); - map.put(CommonPropertyKeys.ID_KEY, remoteProcessGroupDTO.getId()); - map.put(CommonPropertyKeys.NAME_KEY, remoteProcessGroupDTO.getName()); - - - // Prefer the targetUris if populated, otherwise, default to using the singular targetUri - final String targetUris = remoteProcessGroupDTO.getTargetUris(); - map.put(RemoteProcessGroupSchema.URL_KEY, - StringUtils.isNotBlank(targetUris) ? targetUris : remoteProcessGroupDTO.getTargetUri()); - - RemoteProcessGroupContentsDTO contents = remoteProcessGroupDTO.getContents(); - if (contents != null) { - Set inputPorts = contents.getInputPorts(); - if (inputPorts != null) { - map.put(CommonPropertyKeys.INPUT_PORTS_KEY, inputPorts.stream() - .map(remotePortSchemaFunction) - .map(RemotePortSchema::toMap) - .collect(Collectors.toList())); - } - Set outputPorts = contents.getOutputPorts(); - if (outputPorts != null) { - map.put(CommonPropertyKeys.OUTPUT_PORTS_KEY, outputPorts.stream() - .map(remotePortSchemaFunction) - .map(RemotePortSchema::toMap) - .collect(Collectors.toList())); - } - } - - map.put(CommonPropertyKeys.COMMENT_KEY, remoteProcessGroupDTO.getComments()); - map.put(RemoteProcessGroupSchema.TIMEOUT_KEY, remoteProcessGroupDTO.getCommunicationsTimeout()); - map.put(CommonPropertyKeys.YIELD_PERIOD_KEY, remoteProcessGroupDTO.getYieldDuration()); - map.put(RemoteProcessGroupSchema.TRANSPORT_PROTOCOL_KEY, remoteProcessGroupDTO.getTransportProtocol()); - map.put(RemoteProcessGroupSchema.PROXY_HOST_KEY, remoteProcessGroupDTO.getProxyHost()); - map.put(RemoteProcessGroupSchema.PROXY_PORT_KEY, remoteProcessGroupDTO.getProxyPort()); - map.put(RemoteProcessGroupSchema.PROXY_USER_KEY, remoteProcessGroupDTO.getProxyUser()); - map.put(RemoteProcessGroupSchema.PROXY_PASSWORD_KEY, remoteProcessGroupDTO.getProxyPassword()); - map.put(RemoteProcessGroupSchema.LOCAL_NETWORK_INTERFACE_KEY, remoteProcessGroupDTO.getLocalNetworkInterface()); - return new RemoteProcessGroupSchema(map); - } -} diff --git a/minifi/minifi-toolkit/minifi-toolkit-configuration/src/main/java/org/apache/nifi/minifi/toolkit/configuration/json/ComponentPropertyProvider.java b/minifi/minifi-toolkit/minifi-toolkit-configuration/src/main/java/org/apache/nifi/minifi/toolkit/configuration/json/ComponentPropertyProvider.java new file mode 100644 index 0000000000..9bd5f7d6c5 --- /dev/null +++ b/minifi/minifi-toolkit/minifi-toolkit-configuration/src/main/java/org/apache/nifi/minifi/toolkit/configuration/json/ComponentPropertyProvider.java @@ -0,0 +1,164 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.nifi.minifi.toolkit.configuration.json; + +import static java.util.Objects.nonNull; +import static java.util.Optional.ofNullable; +import static java.util.stream.Collectors.toMap; +import static java.util.stream.Stream.concat; + +import java.util.List; +import java.util.Map; +import java.util.function.Function; +import java.util.stream.Stream; +import org.apache.commons.lang3.tuple.Pair; +import org.apache.commons.lang3.tuple.Triple; +import org.apache.nifi.flow.ConnectableComponentType; +import org.apache.nifi.minifi.toolkit.schema.ConfigSchema; +import org.apache.nifi.minifi.toolkit.schema.ProcessGroupSchema; +import org.apache.nifi.minifi.toolkit.schema.RemoteProcessGroupSchema; +import org.apache.nifi.minifi.toolkit.schema.common.BaseSchemaWithId; + +/** + * Helper class to support flow conversion from YAML to JSON format + * When creating connection objects in the JSON definition, the component's type and parentId are required + * However these attributes are not available in the YAML definition on those objects. + * We iterate through the YAML definition, and store the related data in HashMaps, so it can be queried during + * the connection build easily + */ +public class ComponentPropertyProvider { + + private static final Map>> PG_COMPONENT_CONNECTABLE_TYPE_WITH_ACCESSOR = Map.of( + ConnectableComponentType.PROCESSOR, ProcessGroupSchema::getProcessors, + ConnectableComponentType.INPUT_PORT, ProcessGroupSchema::getInputPortSchemas, + ConnectableComponentType.OUTPUT_PORT, ProcessGroupSchema::getOutputPortSchemas, + ConnectableComponentType.FUNNEL, ProcessGroupSchema::getFunnels + ); + private static final Map>> RPG_COMPONENT_CONNECTABLE_TYPE_WITH_ACCESSOR = Map.of( + ConnectableComponentType.REMOTE_INPUT_PORT, RemoteProcessGroupSchema::getInputPorts, + ConnectableComponentType.REMOTE_OUTPUT_PORT, RemoteProcessGroupSchema::getOutputPorts + ); + + private final Map componentIdToParentIdMap; + private final Map componentTypeMap; + + public ComponentPropertyProvider(ConfigSchema configSchema) { + this.componentIdToParentIdMap = createComponentIdToParentIdMap(configSchema); + this.componentTypeMap = createComponentTypeMap(configSchema); + } + + public String parentId(String componentId) { + return componentIdToParentIdMap.get(componentId); + } + + public ConnectableComponentType connectableComponentType(String componentId) { + return componentTypeMap.get(componentId); + } + + private Map createComponentIdToParentIdMap(ConfigSchema configSchema) { + return concat( + fetchControllerServicesProperties(configSchema.getProcessGroupSchema()), + concat( + fetchProcessGroupComponentProperties(configSchema.getProcessGroupSchema()), + fetchRemoteProcessGroupComponentsProperties(configSchema.getProcessGroupSchema())) + .map(triple -> Pair.of(triple.getLeft(), triple.getMiddle()))) + .collect(toMap(Pair::getLeft, Pair::getRight)); + } + + private Map createComponentTypeMap(ConfigSchema configSchema) { + return concat( + fetchProcessGroupComponentProperties(configSchema.getProcessGroupSchema()), + fetchRemoteProcessGroupComponentsProperties(configSchema.getProcessGroupSchema())) + .filter(triple -> nonNull(triple.getRight())) + .collect(toMap(Triple::getLeft, Triple::getRight)); + } + + private Stream> fetchProcessGroupComponentProperties(ProcessGroupSchema processGroupSchema) { + return + concat( + concat( + // adding child process groups to component property map + ofNullable(processGroupSchema.getProcessGroupSchemas()).orElse(List.of()) + .stream() + .map(ProcessGroupSchema::getId) + .map(childProcessGroupId -> Triple.of(childProcessGroupId, processGroupSchema.getId(), null)), + // adding components in process group to component property map + PG_COMPONENT_CONNECTABLE_TYPE_WITH_ACCESSOR.entrySet() + .stream() + .flatMap(schemaAccessorAndComponentType -> + getProcessGroupComponentsProperties(processGroupSchema, schemaAccessorAndComponentType.getValue(), schemaAccessorAndComponentType.getKey())) + ), + // recursively processing child process groups + ofNullable(processGroupSchema.getProcessGroupSchemas()).orElse(List.of()) + .stream() + .flatMap(this::fetchProcessGroupComponentProperties) + ); + } + + private Stream> getProcessGroupComponentsProperties(ProcessGroupSchema processGroupSchema, + Function> schemaAccessor, + ConnectableComponentType componentType) { + return ofNullable(schemaAccessor.apply(processGroupSchema)).orElse(List.of()) + .stream() + .map(component -> Triple.of(component.getId(), processGroupSchema.getId(), componentType)); + } + + private Stream> fetchRemoteProcessGroupComponentsProperties(ProcessGroupSchema processGroupSchema) { + return concat( + concat( + // adding remote process groups to component property map + ofNullable(processGroupSchema.getRemoteProcessGroups()).orElse(List.of()) + .stream() + .map(RemoteProcessGroupSchema::getId) + .map(remoteProcessGroupId -> Triple.of(remoteProcessGroupId, processGroupSchema.getId(), null)), + // adding components from all remote process groups in the process group to component property map + ofNullable(processGroupSchema.getRemoteProcessGroups()).orElse(List.of()) + .stream() + .flatMap(remoteProcessGroupSchema -> RPG_COMPONENT_CONNECTABLE_TYPE_WITH_ACCESSOR.entrySet() + .stream() + .flatMap(schemaAccessorAndComponentType -> getRemoteProcessGroupComponentsProperties( + remoteProcessGroupSchema, schemaAccessorAndComponentType.getValue(), schemaAccessorAndComponentType.getKey()))) + ), + // recursively processing child process groups + ofNullable(processGroupSchema.getProcessGroupSchemas()).orElse(List.of()) + .stream() + .flatMap(this::fetchRemoteProcessGroupComponentsProperties) + ); + } + + private Stream> getRemoteProcessGroupComponentsProperties(RemoteProcessGroupSchema remoteProcessGroupSchema, + Function> schemaAccessor, + ConnectableComponentType componentType) { + return ofNullable(schemaAccessor.apply(remoteProcessGroupSchema)).orElse(List.of()) + .stream() + .map(component -> Triple.of(component.getId(), remoteProcessGroupSchema.getId(), componentType)); + } + + private Stream> fetchControllerServicesProperties(ProcessGroupSchema processGroupSchema) { + return concat( + // adding controller services id and parent id for this process group + ofNullable(processGroupSchema.getControllerServices()).orElse(List.of()) + .stream() + .map(controllerServiceSchema -> Pair.of(controllerServiceSchema.getId(), processGroupSchema.getId())), + // recursively processing child process groups + ofNullable(processGroupSchema.getProcessGroupSchemas()).orElse(List.of()) + .stream() + .flatMap(this::fetchControllerServicesProperties) + ); + } +} diff --git a/minifi/minifi-toolkit/minifi-toolkit-configuration/src/main/java/org/apache/nifi/minifi/toolkit/configuration/json/ConfigSchemaToVersionedDataFlowTransformer.java b/minifi/minifi-toolkit/minifi-toolkit-configuration/src/main/java/org/apache/nifi/minifi/toolkit/configuration/json/ConfigSchemaToVersionedDataFlowTransformer.java new file mode 100644 index 0000000000..829962885c --- /dev/null +++ b/minifi/minifi-toolkit/minifi-toolkit-configuration/src/main/java/org/apache/nifi/minifi/toolkit/configuration/json/ConfigSchemaToVersionedDataFlowTransformer.java @@ -0,0 +1,467 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.nifi.minifi.toolkit.configuration.json; + +import static java.lang.Boolean.TRUE; +import static java.util.Map.entry; +import static java.util.Optional.ofNullable; +import static java.util.UUID.randomUUID; +import static java.util.concurrent.TimeUnit.NANOSECONDS; +import static java.util.stream.Collectors.toList; +import static java.util.stream.Collectors.toMap; +import static java.util.stream.Collectors.toSet; +import static org.apache.commons.lang3.StringUtils.EMPTY; +import static org.apache.nifi.minifi.commons.api.MiNiFiProperties.NIFI_MINIFI_FLOW_MAX_CONCURRENT_THREADS; +import static org.apache.nifi.util.NiFiProperties.ADMINISTRATIVE_YIELD_DURATION; +import static org.apache.nifi.util.NiFiProperties.BORED_YIELD_DURATION; +import static org.apache.nifi.util.NiFiProperties.COMPONENT_STATUS_SNAPSHOT_FREQUENCY; +import static org.apache.nifi.util.NiFiProperties.CONTENT_ARCHIVE_ENABLED; +import static org.apache.nifi.util.NiFiProperties.CONTENT_ARCHIVE_MAX_RETENTION_PERIOD; +import static org.apache.nifi.util.NiFiProperties.CONTENT_ARCHIVE_MAX_USAGE_PERCENTAGE; +import static org.apache.nifi.util.NiFiProperties.CONTENT_REPOSITORY_IMPLEMENTATION; +import static org.apache.nifi.util.NiFiProperties.FLOWFILE_REPOSITORY_ALWAYS_SYNC; +import static org.apache.nifi.util.NiFiProperties.FLOWFILE_REPOSITORY_CHECKPOINT_INTERVAL; +import static org.apache.nifi.util.NiFiProperties.FLOWFILE_REPOSITORY_IMPLEMENTATION; +import static org.apache.nifi.util.NiFiProperties.FLOW_CONTROLLER_GRACEFUL_SHUTDOWN_PERIOD; +import static org.apache.nifi.util.NiFiProperties.MAX_APPENDABLE_CLAIM_SIZE; +import static org.apache.nifi.util.NiFiProperties.PROVENANCE_INDEX_SHARD_SIZE; +import static org.apache.nifi.util.NiFiProperties.PROVENANCE_MAX_STORAGE_SIZE; +import static org.apache.nifi.util.NiFiProperties.PROVENANCE_MAX_STORAGE_TIME; +import static org.apache.nifi.util.NiFiProperties.PROVENANCE_REPO_IMPLEMENTATION_CLASS; +import static org.apache.nifi.util.NiFiProperties.PROVENANCE_ROLLOVER_TIME; +import static org.apache.nifi.util.NiFiProperties.QUEUE_SWAP_THRESHOLD; +import static org.apache.nifi.util.NiFiProperties.VARIABLE_REGISTRY_PROPERTIES; +import static org.apache.nifi.util.NiFiProperties.WRITE_DELAY_INTERVAL; + +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Set; +import java.util.function.Function; +import java.util.function.Supplier; +import java.util.stream.Collector; +import java.util.stream.Stream; +import org.apache.commons.lang3.StringUtils; +import org.apache.nifi.controller.flow.VersionedDataflow; +import org.apache.nifi.controller.flow.VersionedFlowEncodingVersion; +import org.apache.nifi.flow.Bundle; +import org.apache.nifi.flow.ComponentType; +import org.apache.nifi.flow.ConnectableComponent; +import org.apache.nifi.flow.PortType; +import org.apache.nifi.flow.Position; +import org.apache.nifi.flow.ScheduledState; +import org.apache.nifi.flow.VersionedComponent; +import org.apache.nifi.flow.VersionedConnection; +import org.apache.nifi.flow.VersionedControllerService; +import org.apache.nifi.flow.VersionedFunnel; +import org.apache.nifi.flow.VersionedPort; +import org.apache.nifi.flow.VersionedProcessGroup; +import org.apache.nifi.flow.VersionedProcessor; +import org.apache.nifi.flow.VersionedRemoteGroupPort; +import org.apache.nifi.flow.VersionedRemoteProcessGroup; +import org.apache.nifi.flow.VersionedReportingTask; +import org.apache.nifi.logging.LogLevel; +import org.apache.nifi.minifi.toolkit.schema.ComponentStatusRepositorySchema; +import org.apache.nifi.minifi.toolkit.schema.ConfigSchema; +import org.apache.nifi.minifi.toolkit.schema.ConnectionSchema; +import org.apache.nifi.minifi.toolkit.schema.ContentRepositorySchema; +import org.apache.nifi.minifi.toolkit.schema.ControllerServiceSchema; +import org.apache.nifi.minifi.toolkit.schema.CorePropertiesSchema; +import org.apache.nifi.minifi.toolkit.schema.FlowFileRepositorySchema; +import org.apache.nifi.minifi.toolkit.schema.FunnelSchema; +import org.apache.nifi.minifi.toolkit.schema.PortSchema; +import org.apache.nifi.minifi.toolkit.schema.ProcessGroupSchema; +import org.apache.nifi.minifi.toolkit.schema.ProcessorSchema; +import org.apache.nifi.minifi.toolkit.schema.ProvenanceRepositorySchema; +import org.apache.nifi.minifi.toolkit.schema.RemotePortSchema; +import org.apache.nifi.minifi.toolkit.schema.RemoteProcessGroupSchema; +import org.apache.nifi.minifi.toolkit.schema.ReportingSchema; +import org.apache.nifi.minifi.toolkit.schema.SwapSchema; +import org.apache.nifi.minifi.toolkit.schema.common.BaseSchemaWithIdAndName; +import org.apache.nifi.scheduling.ExecutionNode; + +public class ConfigSchemaToVersionedDataFlowTransformer { + + private static final String DEFAULT_FLOW_FILE_EXPIRATION = "0 sec"; + private static final String DEFAULT_BACK_PRESSURE_DATA_SIZE_THRESHOLD = "1 GB"; + private static final String FLOW_FILE_CONCURRENCY = "UNBOUNDED"; + private static final String FLOW_FILE_OUTBOUND_POLICY = "STREAM_WHEN_AVAILABLE"; + private static final long DEFAULT_BACK_PRESSURE_OBJECT_THRESHOLD = 10000L; + private static final Position DEFAULT_POSITION = new Position(0, 0); + + private final ConfigSchema configSchema; + private final ComponentPropertyProvider componentPropertyProvider; + + public ConfigSchemaToVersionedDataFlowTransformer(ConfigSchema configSchema) { + this.configSchema = configSchema; + this.componentPropertyProvider = new ComponentPropertyProvider(configSchema); + } + + public Map extractProperties() { + CorePropertiesSchema coreProperties = configSchema.getCoreProperties(); + FlowFileRepositorySchema flowFileRepositoryProperties = configSchema.getFlowfileRepositoryProperties(); + ContentRepositorySchema contentRepositoryProperties = configSchema.getContentRepositoryProperties(); + ProvenanceRepositorySchema provenanceRepositoryProperties = configSchema.getProvenanceRepositorySchema(); + ComponentStatusRepositorySchema componentStatusRepositoryProperties = configSchema.getComponentStatusRepositoryProperties(); + SwapSchema swapProperties = configSchema.getFlowfileRepositoryProperties().getSwapProperties(); + + return Stream.concat( + Stream.of( + entry(NIFI_MINIFI_FLOW_MAX_CONCURRENT_THREADS.getKey(), coreProperties.getMaxConcurrentThreads().toString()), + entry(FLOW_CONTROLLER_GRACEFUL_SHUTDOWN_PERIOD, coreProperties.getFlowControllerGracefulShutdownPeriod()), + entry(WRITE_DELAY_INTERVAL, coreProperties.getFlowServiceWriteDelayInterval()), + entry(ADMINISTRATIVE_YIELD_DURATION, coreProperties.getAdministrativeYieldDuration()), + entry(BORED_YIELD_DURATION, coreProperties.getBoredYieldDuration()), + entry(VARIABLE_REGISTRY_PROPERTIES, coreProperties.getVariableRegistryProperties()), + entry(FLOWFILE_REPOSITORY_IMPLEMENTATION, flowFileRepositoryProperties.getFlowFileRepository()), + entry(FLOWFILE_REPOSITORY_CHECKPOINT_INTERVAL, flowFileRepositoryProperties.getCheckpointInterval()), + entry(FLOWFILE_REPOSITORY_ALWAYS_SYNC, Boolean.toString(flowFileRepositoryProperties.getAlwaysSync())), + entry(CONTENT_REPOSITORY_IMPLEMENTATION, contentRepositoryProperties.getContentRepository()), + entry(MAX_APPENDABLE_CLAIM_SIZE, contentRepositoryProperties.getContentClaimMaxAppendableSize()), + entry(CONTENT_ARCHIVE_MAX_RETENTION_PERIOD, contentRepositoryProperties.getContentRepoArchiveMaxRetentionPeriod()), + entry(CONTENT_ARCHIVE_MAX_USAGE_PERCENTAGE, contentRepositoryProperties.getContentRepoArchiveMaxUsagePercentage()), + entry(CONTENT_ARCHIVE_ENABLED, Boolean.toString(contentRepositoryProperties.getContentRepoArchiveEnabled())), + entry(PROVENANCE_REPO_IMPLEMENTATION_CLASS, provenanceRepositoryProperties.getProvenanceRepository()), + entry(PROVENANCE_ROLLOVER_TIME, provenanceRepositoryProperties.getProvenanceRepoRolloverTimeKey()), + entry(PROVENANCE_INDEX_SHARD_SIZE, provenanceRepositoryProperties.getProvenanceRepoIndexShardSize()), + entry(PROVENANCE_MAX_STORAGE_SIZE, provenanceRepositoryProperties.getProvenanceRepoMaxStorageSize()), + entry(PROVENANCE_MAX_STORAGE_TIME, provenanceRepositoryProperties.getProvenanceRepoMaxStorageTime()), + entry(COMPONENT_STATUS_SNAPSHOT_FREQUENCY, componentStatusRepositoryProperties.getSnapshotFrequency()), + entry(QUEUE_SWAP_THRESHOLD, swapProperties.getThreshold().toString()) + ), + ofNullable(configSchema.getNifiPropertiesOverrides()).map(Map::entrySet).orElse(Set.of()).stream() + ) + .collect(toMap(Map.Entry::getKey, Map.Entry::getValue)); + } + + public VersionedDataflow convert() { + VersionedDataflow versionedDataflow = new VersionedDataflow(); + versionedDataflow.setEncodingVersion(new VersionedFlowEncodingVersion(2, 0)); + versionedDataflow.setMaxTimerDrivenThreadCount(configSchema.getCoreProperties().getMaxConcurrentThreads().intValue()); + + versionedDataflow.setRegistries(List.of()); + versionedDataflow.setParameterContexts(List.of()); + versionedDataflow.setParameterProviders(List.of()); + versionedDataflow.setControllerServices(List.of()); + versionedDataflow.setTemplates(Set.of()); + versionedDataflow.setReportingTasks( + convertComponents(configSchema::getReportingTasksSchema, this::toVersionedReportingTask, toList())); + + VersionedProcessGroup versionedProcessGroup = new VersionedProcessGroup(); + versionedProcessGroup.setDefaultFlowFileExpiration(DEFAULT_FLOW_FILE_EXPIRATION); + versionedProcessGroup.setDefaultBackPressureObjectThreshold(DEFAULT_BACK_PRESSURE_OBJECT_THRESHOLD); + versionedProcessGroup.setDefaultBackPressureDataSizeThreshold(DEFAULT_BACK_PRESSURE_DATA_SIZE_THRESHOLD); + versionedProcessGroup.setFlowFileConcurrency(FLOW_FILE_CONCURRENCY); + versionedProcessGroup.setFlowFileOutboundPolicy(FLOW_FILE_OUTBOUND_POLICY); + + convertProcessGroup(configSchema.getProcessGroupSchema(), versionedProcessGroup); + + // we need to set the instance ids of the components in the end, as at the time of creating the connection the instance id is not available + Map idToInstanceIdMapOfConnectableComponents = getIdToInstanceIdMapOfConnectableComponents(versionedProcessGroup); + setConnectableComponentsInstanceId(versionedProcessGroup, idToInstanceIdMapOfConnectableComponents); + + versionedDataflow.setRootGroup(versionedProcessGroup); + + return versionedDataflow; + } + + private Map getIdToInstanceIdMapOfConnectableComponents(VersionedProcessGroup versionedProcessGroup) { + Map thisProcessGroupIdToInstanceIdMaps = Stream.of( + ofNullable(versionedProcessGroup.getProcessors()).orElse(Set.of()), + ofNullable(versionedProcessGroup.getInputPorts()).orElse(Set.of()), + ofNullable(versionedProcessGroup.getOutputPorts()).orElse(Set.of()), + ofNullable(versionedProcessGroup.getFunnels()).orElse(Set.of()), + ofNullable(versionedProcessGroup.getRemoteProcessGroups()).orElse(Set.of()) + .stream() + .map(VersionedRemoteProcessGroup::getInputPorts) + .flatMap(Set::stream) + .collect(toSet()), + ofNullable(versionedProcessGroup.getRemoteProcessGroups()).orElse(Set.of()) + .stream() + .map(VersionedRemoteProcessGroup::getOutputPorts) + .flatMap(Set::stream) + .collect(toSet()) + ) + .flatMap(Set::stream) + .collect(toMap(VersionedComponent::getIdentifier, VersionedComponent::getInstanceIdentifier)); + + Stream> childProcessGroupsIdToInstanceIdMaps = ofNullable(versionedProcessGroup.getProcessGroups()).orElse(Set.of()) + .stream() + .map(this::getIdToInstanceIdMapOfConnectableComponents); + + return Stream.concat( + Stream.of(thisProcessGroupIdToInstanceIdMaps), + childProcessGroupsIdToInstanceIdMaps) + .map(Map::entrySet) + .flatMap(Set::stream) + .collect(toMap(Map.Entry::getKey, Map.Entry::getValue)); + } + + private void setConnectableComponentsInstanceId(VersionedProcessGroup versionedProcessGroup, Map idToInstanceIdMapOfConnectableComponents) { + ofNullable(versionedProcessGroup.getConnections()).orElse(Set.of()) + .forEach(connection -> { + ConnectableComponent source = connection.getSource(); + source.setInstanceIdentifier(idToInstanceIdMapOfConnectableComponents.get(source.getId())); + ConnectableComponent destination = connection.getDestination(); + System.err.println(destination.getType() + " - " + destination.getId() + " - " + idToInstanceIdMapOfConnectableComponents.get(destination.getId())); + destination.setInstanceIdentifier(idToInstanceIdMapOfConnectableComponents.get(destination.getId())); + }); + ofNullable(versionedProcessGroup.getProcessGroups()).orElse(Set.of()) + .forEach(childProcessGroup -> setConnectableComponentsInstanceId(childProcessGroup, idToInstanceIdMapOfConnectableComponents)); + } + + private void convertProcessGroup(ProcessGroupSchema processGroupSchema, VersionedProcessGroup processGroup) { + processGroup.setIdentifier(processGroupSchema.getId()); + processGroup.setInstanceIdentifier(processGroupSchema.getId()); + processGroup.setName(getNameOrId(processGroupSchema)); + processGroup.setComments(processGroupSchema.getComment()); + processGroup.setPosition(DEFAULT_POSITION); + processGroup.setProcessors( + convertComponents(processGroupSchema::getProcessors, this::toVersionedProcessor, toSet())); + processGroup.setControllerServices( + convertComponents(processGroupSchema::getControllerServices, this::toVersionedControllerService, toSet())); + processGroup.setConnections( + convertComponents(processGroupSchema::getConnections, this::toVersionedConnection, toSet())); + processGroup.setFunnels( + convertComponents(processGroupSchema::getFunnels, this::toVersionedFunnel, toSet())); + processGroup.setRemoteProcessGroups( + convertComponents(processGroupSchema::getRemoteProcessGroups, this::toRemoteProcessGroup, toSet())); + processGroup.setInputPorts( + convertComponents(processGroupSchema::getInputPortSchemas, this::toInputPort, toSet())); + processGroup.setOutputPorts( + convertComponents(processGroupSchema::getOutputPortSchemas, this::toOutputPort, toSet())); + processGroup.setProcessGroups( + convertComponents(processGroupSchema::getProcessGroupSchemas, this::toVersionedProcessGroup, toSet())); + } + + private V convertComponents(Supplier> convertibles, Function converter, Collector collector) { + return ofNullable(convertibles.get()).orElse(List.of()) + .stream() + .map(converter) + .collect(collector); + } + + private VersionedReportingTask toVersionedReportingTask(ReportingSchema reportingSchema) { + VersionedReportingTask reportingTask = new VersionedReportingTask(); + reportingTask.setIdentifier(reportingSchema.getId()); + reportingTask.setInstanceIdentifier(randomUUID().toString()); + reportingTask.setName(getNameOrId(reportingSchema)); + reportingTask.setComments(reportingSchema.getComment()); + reportingTask.setType(reportingSchema.getReportingClass()); + reportingTask.setBundle(bundleFor(reportingSchema.getReportingClass())); + reportingTask.setSchedulingStrategy(reportingSchema.getSchedulingStrategy()); + reportingTask.setSchedulingPeriod(reportingSchema.getSchedulingPeriod()); + reportingTask.setProperties(toStringStringProperties(reportingSchema.getProperties())); + reportingTask.setComponentType(ComponentType.REPORTING_TASK); + reportingTask.setScheduledState(ScheduledState.RUNNING); + reportingTask.setPropertyDescriptors(Map.of()); + reportingTask.setPosition(DEFAULT_POSITION); + return reportingTask; + } + + private VersionedProcessor toVersionedProcessor(ProcessorSchema processorSchema) { + VersionedProcessor processor = new VersionedProcessor(); + processor.setIdentifier(processorSchema.getId()); + processor.setInstanceIdentifier(randomUUID().toString()); + processor.setGroupIdentifier(componentPropertyProvider.parentId(processorSchema.getId())); + processor.setName(getNameOrId(processorSchema)); + processor.setType(processorSchema.getProcessorClass()); + processor.setBundle(bundleFor(processorSchema.getProcessorClass())); + processor.setConcurrentlySchedulableTaskCount(processorSchema.getMaxConcurrentTasks().intValue()); + processor.setSchedulingStrategy(processorSchema.getSchedulingStrategy()); + processor.setSchedulingPeriod(processorSchema.getSchedulingPeriod()); + processor.setPenaltyDuration(processorSchema.getPenalizationPeriod()); + processor.setYieldDuration(processorSchema.getYieldPeriod()); + processor.setRunDurationMillis(NANOSECONDS.toMicros(processorSchema.getRunDurationNanos().longValue())); + processor.setAutoTerminatedRelationships(Set.copyOf(processorSchema.getAutoTerminatedRelationshipsList())); + processor.setProperties(toStringStringProperties(processorSchema.getProperties())); + processor.setAnnotationData(processorSchema.getAnnotationData()); + processor.setComponentType(ComponentType.PROCESSOR); + processor.setScheduledState(ScheduledState.RUNNING); + processor.setPropertyDescriptors(Map.of()); + processor.setBulletinLevel(LogLevel.WARN.name()); + processor.setPosition(DEFAULT_POSITION); + processor.setExecutionNode(ExecutionNode.ALL.name()); + return processor; + } + + private VersionedControllerService toVersionedControllerService(ControllerServiceSchema controllerServiceSchema) { + VersionedControllerService controllerService = new VersionedControllerService(); + controllerService.setIdentifier(controllerServiceSchema.getId()); + controllerService.setInstanceIdentifier(randomUUID().toString()); + controllerService.setGroupIdentifier(componentPropertyProvider.parentId(controllerServiceSchema.getId())); + controllerService.setName(getNameOrId(controllerServiceSchema)); + controllerService.setType(controllerServiceSchema.getServiceClass()); + controllerService.setBundle(bundleFor(controllerServiceSchema.getServiceClass())); + controllerService.setProperties(toStringStringProperties(controllerServiceSchema.getProperties())); + controllerService.setAnnotationData(controllerServiceSchema.getAnnotationData()); + controllerService.setComponentType(ComponentType.CONTROLLER_SERVICE); + controllerService.setScheduledState(ScheduledState.RUNNING); + controllerService.setPropertyDescriptors(Map.of()); + controllerService.setPosition(DEFAULT_POSITION); + return controllerService; + } + + private VersionedConnection toVersionedConnection(ConnectionSchema connectionSchema) { + VersionedConnection connection = new VersionedConnection(); + connection.setIdentifier(connectionSchema.getId()); + connection.setInstanceIdentifier(randomUUID().toString()); + connection.setName(getNameOrId(connectionSchema)); + connection.setComponentType(ComponentType.CONNECTION); + connection.setSource(connectableComponent(connectionSchema.getSourceId())); + connection.setDestination(connectableComponent(connectionSchema.getDestinationId())); + connection.setSelectedRelationships(Set.copyOf(connectionSchema.getSourceRelationshipNames())); + connection.setBackPressureDataSizeThreshold(connectionSchema.getMaxWorkQueueDataSize()); + connection.setBackPressureObjectThreshold(connectionSchema.getMaxWorkQueueSize().longValue()); + connection.setFlowFileExpiration(connectionSchema.getFlowfileExpiration()); + connection.setPrioritizers(List.of(connectionSchema.getQueuePrioritizerClass()).stream().filter(StringUtils::isNotBlank).collect(toList())); + connection.setPosition(DEFAULT_POSITION); + connection.setLabelIndex(0); + connection.setzIndex(0L); + return connection; + } + + private ConnectableComponent connectableComponent(String componentId) { + ConnectableComponent component = new ConnectableComponent(); + component.setId(componentId); + component.setGroupId(componentPropertyProvider.parentId(componentId)); + component.setType(componentPropertyProvider.connectableComponentType(componentId)); + return component; + } + + private VersionedFunnel toVersionedFunnel(FunnelSchema funnelSchema) { + VersionedFunnel funnel = new VersionedFunnel(); + funnel.setIdentifier(funnelSchema.getId()); + funnel.setInstanceIdentifier(randomUUID().toString()); + funnel.setGroupIdentifier(componentPropertyProvider.parentId(funnelSchema.getId())); + funnel.setName(funnelSchema.getWrapperName()); + funnel.setComponentType(ComponentType.FUNNEL); + funnel.setPosition(DEFAULT_POSITION); + return funnel; + } + + private VersionedRemoteProcessGroup toRemoteProcessGroup(RemoteProcessGroupSchema remoteProcessGroupSchema) { + VersionedRemoteProcessGroup remoteProcessGroup = new VersionedRemoteProcessGroup(); + remoteProcessGroup.setIdentifier(remoteProcessGroupSchema.getId()); + remoteProcessGroup.setInstanceIdentifier(randomUUID().toString()); + remoteProcessGroup.setGroupIdentifier(componentPropertyProvider.parentId(remoteProcessGroupSchema.getId())); + remoteProcessGroup.setName(getNameOrId(remoteProcessGroupSchema)); + remoteProcessGroup.setComponentType(ComponentType.REMOTE_PROCESS_GROUP); + remoteProcessGroup.setTargetUris(remoteProcessGroupSchema.getUrls()); + remoteProcessGroup.setComments(remoteProcessGroupSchema.getComment()); + remoteProcessGroup.setCommunicationsTimeout(remoteProcessGroupSchema.getTimeout()); + remoteProcessGroup.setYieldDuration(remoteProcessGroupSchema.getYieldPeriod()); + remoteProcessGroup.setTransportProtocol(remoteProcessGroupSchema.getTransportProtocol()); + ofNullable(remoteProcessGroupSchema.getLocalNetworkInterface()) + .filter(StringUtils::isNotBlank) + .ifPresent(remoteProcessGroup::setLocalNetworkInterface); + ofNullable(remoteProcessGroupSchema.getProxyHost()) + .filter(StringUtils::isNotBlank) + .ifPresent(remoteProcessGroup::setProxyHost); + ofNullable(remoteProcessGroupSchema.getProxyPort()) + .filter(Objects::nonNull) + .ifPresent(remoteProcessGroup::setProxyPort); + ofNullable(remoteProcessGroupSchema.getProxyUser()) + .filter(StringUtils::isNotBlank) + .ifPresent(remoteProcessGroup::setProxyUser); + ofNullable(remoteProcessGroupSchema.getProxyPassword()) + .filter(StringUtils::isNotBlank) + .ifPresent(remoteProcessGroup::setProxyPassword); + remoteProcessGroup.setPosition(DEFAULT_POSITION); + + remoteProcessGroup.setInputPorts( + convertComponents(remoteProcessGroupSchema::getInputPorts, this::toRemoteInputPort, toSet())); + remoteProcessGroup.setOutputPorts( + convertComponents(remoteProcessGroupSchema::getOutputPorts, this::toRemoteOutputPort, toSet())); + + return remoteProcessGroup; + } + + private VersionedRemoteGroupPort toRemoteInputPort(RemotePortSchema portSchema) { + return toVersionedRemoteGroupPort(portSchema, ComponentType.REMOTE_INPUT_PORT); + } + + private VersionedRemoteGroupPort toRemoteOutputPort(RemotePortSchema portSchema) { + return toVersionedRemoteGroupPort(portSchema, ComponentType.REMOTE_OUTPUT_PORT); + } + + private VersionedRemoteGroupPort toVersionedRemoteGroupPort(RemotePortSchema portSchema, ComponentType portType) { + VersionedRemoteGroupPort port = new VersionedRemoteGroupPort(); + port.setIdentifier(randomUUID().toString()); + port.setInstanceIdentifier(portSchema.getId()); + port.setTargetId(portSchema.getId()); + port.setGroupIdentifier(componentPropertyProvider.parentId(portSchema.getId())); + port.setName(getNameOrId(portSchema)); + port.setComponentType(portType); + port.setScheduledState(ScheduledState.RUNNING); + port.setComments(portSchema.getComment()); + port.setConcurrentlySchedulableTaskCount(portSchema.getMax_concurrent_tasks().intValue()); + port.setUseCompression(portSchema.getUseCompression()); + port.setPosition(DEFAULT_POSITION); + return port; + } + + private VersionedPort toInputPort(PortSchema portSchema) { + return toVersionedPort(portSchema, PortType.INPUT_PORT); + } + + private VersionedPort toOutputPort(PortSchema portSchema) { + return toVersionedPort(portSchema, PortType.OUTPUT_PORT); + } + + private VersionedPort toVersionedPort(PortSchema portSchema, PortType portType) { + VersionedPort port = new VersionedPort(); + port.setIdentifier(portSchema.getId()); + port.setInstanceIdentifier(randomUUID().toString()); + port.setGroupIdentifier(componentPropertyProvider.parentId(portSchema.getId())); + port.setName(getNameOrId(portSchema)); + port.setComponentType(portType == PortType.INPUT_PORT ? ComponentType.INPUT_PORT : ComponentType.OUTPUT_PORT); + port.setScheduledState(ScheduledState.RUNNING); + port.setType(portType); + port.setAllowRemoteAccess(TRUE); + port.setPosition(DEFAULT_POSITION); + return port; + } + + private VersionedProcessGroup toVersionedProcessGroup(ProcessGroupSchema childProcessGroupSchema) { + VersionedProcessGroup childProcessGroup = new VersionedProcessGroup(); + childProcessGroup.setGroupIdentifier(componentPropertyProvider.parentId(childProcessGroupSchema.getId())); + + convertProcessGroup(childProcessGroupSchema, childProcessGroup); + + return childProcessGroup; + } + + private Map toStringStringProperties(Map stringObjectProperties) { + return stringObjectProperties.entrySet() + .stream() + .collect(toMap(Map.Entry::getKey, entry -> ofNullable(entry.getValue()).map(Object::toString).orElse(EMPTY))); + } + + private Bundle bundleFor(String type) { + Bundle bundle = new Bundle(); + bundle.setGroup(EMPTY); + bundle.setArtifact(type); + bundle.setVersion(EMPTY); + return bundle; + } + + private String getNameOrId(BaseSchemaWithIdAndName schema) { + return ofNullable(schema.getName()).filter(StringUtils::isNotBlank).orElse(schema.getId()); + } +} \ No newline at end of file diff --git a/minifi/minifi-toolkit/minifi-toolkit-configuration/src/main/java/org/apache/nifi/minifi/toolkit/configuration/json/TransformYamlCommandFactory.java b/minifi/minifi-toolkit/minifi-toolkit-configuration/src/main/java/org/apache/nifi/minifi/toolkit/configuration/json/TransformYamlCommandFactory.java new file mode 100644 index 0000000000..c8dc45411c --- /dev/null +++ b/minifi/minifi-toolkit/minifi-toolkit-configuration/src/main/java/org/apache/nifi/minifi/toolkit/configuration/json/TransformYamlCommandFactory.java @@ -0,0 +1,170 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.nifi.minifi.toolkit.configuration.json; + +import static java.lang.System.lineSeparator; +import static java.nio.charset.StandardCharsets.UTF_8; +import static java.util.stream.Collectors.joining; +import static java.util.stream.Collectors.toMap; +import static org.apache.commons.io.IOUtils.write; + +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.databind.DeserializationFeature; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.module.jaxb.JaxbAnnotationIntrospector; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.util.Map; +import java.util.Map.Entry; +import java.util.Properties; +import org.apache.nifi.controller.flow.VersionedDataflow; +import org.apache.nifi.minifi.toolkit.configuration.ConfigMain; +import org.apache.nifi.minifi.toolkit.configuration.ConfigTransformException; +import org.apache.nifi.minifi.toolkit.configuration.PathInputStreamFactory; +import org.apache.nifi.minifi.toolkit.configuration.PathOutputStreamFactory; +import org.apache.nifi.minifi.toolkit.schema.ConfigSchema; +import org.apache.nifi.minifi.toolkit.schema.common.ConvertableSchema; +import org.apache.nifi.minifi.toolkit.schema.common.Schema; +import org.apache.nifi.minifi.toolkit.schema.exception.SchemaInstantiatonException; +import org.apache.nifi.minifi.toolkit.schema.exception.SchemaLoaderException; +import org.apache.nifi.minifi.toolkit.schema.serialization.SchemaLoader; + +public class TransformYamlCommandFactory { + + public static final String TRANSFORM_YML = "transform-yml"; + + private static final String COMMAND_DESCRIPTION = "Transform MiNiFi config YAML into NiFi flow JSON format"; + private static final String PROPERTY_KEY_VALUE_DELIMITER = "="; + + private final PathInputStreamFactory pathInputStreamFactory; + private final PathOutputStreamFactory pathOutputStreamFactory; + + public TransformYamlCommandFactory(PathInputStreamFactory pathInputStreamFactory, PathOutputStreamFactory pathOutputStreamFactory) { + this.pathInputStreamFactory = pathInputStreamFactory; + this.pathOutputStreamFactory = pathOutputStreamFactory; + } + + public ConfigMain.Command create() { + return new ConfigMain.Command(this::transformYamlToJson, COMMAND_DESCRIPTION); + } + + private int transformYamlToJson(String[] args) { + if (args.length != 5) { + printTransformYmlUsage(); + return ConfigMain.ERR_INVALID_ARGS; + } + + String sourceMiNiFiConfigYamlPath = args[1]; + String sourceBootstrapConfigPath = args[2]; + String targetFlowJsonPath = args[3]; + String targetBootstrapConfigPath = args[4]; + + try { + ConfigSchema configSchema = readMiNiFiConfig(sourceMiNiFiConfigYamlPath); + + Properties sourceBootstrapProperties = loadProperties(sourceBootstrapConfigPath); + Properties targetBootstrapProperties = loadProperties(targetBootstrapConfigPath); + + ConfigSchemaToVersionedDataFlowTransformer transformer = new ConfigSchemaToVersionedDataFlowTransformer(configSchema); + VersionedDataflow convertedFlow = transformer.convert(); + + Map extractedProperties = transformer.extractProperties(); + targetBootstrapProperties.putAll(sourceBootstrapProperties.entrySet().stream().collect(toMap(Entry::getKey, Entry::getValue))); + targetBootstrapProperties.putAll(extractedProperties); + + persistFlowJson(convertedFlow, targetFlowJsonPath); + persistProperties(targetBootstrapProperties, targetBootstrapConfigPath); + } catch (ConfigTransformException e) { + System.out.println("Unable to convert MiNiFi config YAML to flow JSON: " + e); + return e.getErrorCode(); + } + + return ConfigMain.SUCCESS; + } + + private void printTransformYmlUsage() { + System.out.println("Transform YML Usage:"); + System.out.println(); + System.out.println(" transform-yml SOURCE_MINIFI_CONFIG_YAML_FILE SOURCE_BOOTSTRAP_PROPERTIES_FILE TARGET_FLOW_JSON_FILE TARGET_BOOTSTRAP_PROPERTIES_FILE"); + System.out.println(); + } + + private ConfigSchema readMiNiFiConfig(String miNiFiConfigPath) throws ConfigTransformException { + try (InputStream inputStream = pathInputStreamFactory.create(miNiFiConfigPath)) { + ConvertableSchema convertableSchema = throwIfInvalid(SchemaLoader.loadConvertableSchemaFromYaml(inputStream)); + return throwIfInvalid(convertableSchema.convert()); + } catch (IOException e) { + throw new ConfigTransformException("Error when read MiNiFi config file", ConfigMain.ERR_UNABLE_TO_READ_TEMPLATE, e); + } catch (SchemaLoaderException e) { + throw new ConfigTransformException("Error when parsing MiNiFi config file", ConfigMain.ERR_UNABLE_TO_PARSE_CONFIG, e); + } + } + + private T throwIfInvalid(T schema) throws SchemaLoaderException { + if (!schema.isValid()) { + throw new SchemaInstantiatonException("Failed to transform config file due to:[" + + schema.getValidationIssues().stream().sorted().collect(joining("], [")) + "]"); + } + return schema; + } + + private Properties loadProperties(String propertiesFilePath) throws ConfigTransformException { + try (InputStream inputStream = pathInputStreamFactory.create(propertiesFilePath)) { + Properties properties = new Properties(); + properties.load(inputStream); + return properties; + } catch (IOException e) { + throw new ConfigTransformException("Error when read properties file: " + propertiesFilePath, ConfigMain.ERR_UNABLE_TO_OPEN_INPUT, e); + } + } + + private void persistFlowJson(VersionedDataflow flow, String flowJsonPath) throws ConfigTransformException { + try (OutputStream outputStream = pathOutputStreamFactory.create(flowJsonPath)) { + ObjectMapper objectMapper = createObjectMapper(); + objectMapper.writeValue(outputStream, flow); + } catch (IOException e) { + throw new ConfigTransformException("Error when persisting flow JSON file: " + flowJsonPath, ConfigMain.ERR_UNABLE_TO_SAVE_CONFIG, e); + } + } + + private ObjectMapper createObjectMapper() { + ObjectMapper objectMapper = new ObjectMapper(); + objectMapper.setDefaultPropertyInclusion(JsonInclude.Value.construct(JsonInclude.Include.NON_NULL, JsonInclude.Include.NON_NULL)); + objectMapper.setSerializationInclusion(JsonInclude.Include.NON_NULL); + objectMapper.setAnnotationIntrospector(new JaxbAnnotationIntrospector(objectMapper.getTypeFactory())); + objectMapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false); + return objectMapper; + } + + private void persistProperties(Properties properties, String bootstrapPropertiesPath) throws ConfigTransformException { + try (OutputStream outputStream = pathOutputStreamFactory.create(bootstrapPropertiesPath)) { + write( + properties.entrySet() + .stream() + .map(entry -> entry.getKey() + PROPERTY_KEY_VALUE_DELIMITER + entry.getValue()) + .sorted() + .collect(joining(lineSeparator())), + outputStream, + UTF_8 + ); + } catch (IOException e) { + throw new ConfigTransformException("Error when persisting bootstrap properties file: " + bootstrapPropertiesPath, ConfigMain.ERR_UNABLE_TO_SAVE_CONFIG, e); + } + } +} diff --git a/minifi/minifi-toolkit/minifi-toolkit-configuration/src/main/java/org/apache/nifi/minifi/toolkit/configuration/registry/NiFiRegConfigSchemaFunction.java b/minifi/minifi-toolkit/minifi-toolkit-configuration/src/main/java/org/apache/nifi/minifi/toolkit/configuration/registry/NiFiRegConfigSchemaFunction.java deleted file mode 100644 index 9ab26e8789..0000000000 --- a/minifi/minifi-toolkit/minifi-toolkit-configuration/src/main/java/org/apache/nifi/minifi/toolkit/configuration/registry/NiFiRegConfigSchemaFunction.java +++ /dev/null @@ -1,158 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.nifi.minifi.toolkit.configuration.registry; - -import org.apache.nifi.minifi.commons.schema.ConfigSchema; -import org.apache.nifi.minifi.commons.schema.ConnectionSchema; -import org.apache.nifi.minifi.commons.schema.ControllerServiceSchema; -import org.apache.nifi.minifi.commons.schema.FunnelSchema; -import org.apache.nifi.minifi.commons.schema.PortSchema; -import org.apache.nifi.minifi.commons.schema.ProcessGroupSchema; -import org.apache.nifi.minifi.commons.schema.ProcessorSchema; -import org.apache.nifi.minifi.commons.schema.RemoteProcessGroupSchema; -import org.apache.nifi.minifi.commons.schema.common.CollectionUtil; -import org.apache.nifi.minifi.commons.schema.common.CommonPropertyKeys; -import org.apache.nifi.minifi.commons.schema.common.StringUtil; -import org.apache.nifi.registry.flow.VersionedFlowSnapshot; -import org.apache.nifi.flow.VersionedProcessGroup; - -import java.util.Comparator; -import java.util.HashMap; -import java.util.Map; -import java.util.function.Function; -import java.util.stream.Collectors; - -import static org.apache.nifi.minifi.commons.schema.common.CollectionUtil.nullToEmpty; -import static org.apache.nifi.minifi.commons.schema.common.CommonPropertyKeys.ID_KEY; -import static org.apache.nifi.minifi.commons.schema.common.CommonPropertyKeys.INPUT_PORTS_KEY; -import static org.apache.nifi.minifi.commons.schema.common.CommonPropertyKeys.NAME_KEY; -import static org.apache.nifi.minifi.commons.schema.common.CommonPropertyKeys.OUTPUT_PORTS_KEY; - -public class NiFiRegConfigSchemaFunction implements Function { - - private final NiFiRegFlowControllerSchemaFunction flowControllerSchemaFunction; - private final NiFiRegProcessorSchemaFunction processorSchemaFunction; - private final NiFiRegControllerServiceSchemaFunction controllerServiceSchemaFunction; - private final NiFiRegConnectionSchemaFunction connectionSchemaFunction; - private final NiFiRegFunnelSchemaFunction funnelSchemaFunction; - private final NiFiRegRemoteProcessGroupSchemaFunction remoteProcessGroupSchemaFunction; - private final NiFiRegPortSchemaFunction inputPortSchemaFunction; - private final NiFiRegPortSchemaFunction outputPortSchemaFunction; - - public NiFiRegConfigSchemaFunction() { - this( - new NiFiRegFlowControllerSchemaFunction(), - new NiFiRegProcessorSchemaFunction(), - new NiFiRegControllerServiceSchemaFunction(), - new NiFiRegConnectionSchemaFunction(), - new NiFiRegFunnelSchemaFunction(), - new NiFiRegRemoteProcessGroupSchemaFunction(new NiFiRegRemotePortSchemaFunction()), - new NiFiRegPortSchemaFunction(INPUT_PORTS_KEY), - new NiFiRegPortSchemaFunction(OUTPUT_PORTS_KEY) - ); - } - - public NiFiRegConfigSchemaFunction(final NiFiRegFlowControllerSchemaFunction flowControllerSchemaFunction, - final NiFiRegProcessorSchemaFunction processorSchemaFunction, - final NiFiRegControllerServiceSchemaFunction controllerServiceSchemaFunction, - final NiFiRegConnectionSchemaFunction connectionSchemaFunction, - final NiFiRegFunnelSchemaFunction funnelSchemaFunction, - final NiFiRegRemoteProcessGroupSchemaFunction remoteProcessGroupSchemaFunction, - final NiFiRegPortSchemaFunction inputPortSchemaFunction, - final NiFiRegPortSchemaFunction outputPortSchemaFunction) { - this.flowControllerSchemaFunction = flowControllerSchemaFunction; - this.processorSchemaFunction = processorSchemaFunction; - this.controllerServiceSchemaFunction = controllerServiceSchemaFunction; - this.connectionSchemaFunction = connectionSchemaFunction; - this.funnelSchemaFunction = funnelSchemaFunction; - this.remoteProcessGroupSchemaFunction = remoteProcessGroupSchemaFunction; - this.inputPortSchemaFunction = inputPortSchemaFunction; - this.outputPortSchemaFunction = outputPortSchemaFunction; - } - - @Override - public ConfigSchema apply(final VersionedFlowSnapshot versionedFlowSnapshot) { - Map map = new HashMap<>(); - map.put(CommonPropertyKeys.FLOW_CONTROLLER_PROPS_KEY, flowControllerSchemaFunction.apply(versionedFlowSnapshot).toMap()); - - VersionedProcessGroup versionedProcessGroup = versionedFlowSnapshot.getFlowContents(); - addVersionedProcessGroup(map, versionedProcessGroup); - - return new ConfigSchema(map); - } - - protected void addVersionedProcessGroup(Map map, VersionedProcessGroup versionedProcessGroup) { - addVersionedProcessGroup(map, null, null, versionedProcessGroup); - } - - protected Map addVersionedProcessGroup(Map map, String id, String name, VersionedProcessGroup versionedProcessGroup) { - if (!StringUtil.isNullOrEmpty(id)) { - map.put(ID_KEY, id); - } - - if (!StringUtil.isNullOrEmpty(name)) { - map.put(NAME_KEY, name); - } - - map.put(CommonPropertyKeys.PROCESSORS_KEY, nullToEmpty(versionedProcessGroup.getProcessors()).stream() - .map(processorSchemaFunction) - .sorted(Comparator.comparing(ProcessorSchema::getName)) - .map(ProcessorSchema::toMap) - .collect(Collectors.toList())); - - map.put(CommonPropertyKeys.CONTROLLER_SERVICES_KEY, nullToEmpty(versionedProcessGroup.getControllerServices()).stream() - .map(controllerServiceSchemaFunction) - .sorted(Comparator.comparing(ControllerServiceSchema::getName)) - .map(ControllerServiceSchema::toMap) - .collect(Collectors.toList())); - - map.put(CommonPropertyKeys.CONNECTIONS_KEY, nullToEmpty(versionedProcessGroup.getConnections()).stream() - .map(connectionSchemaFunction) - .sorted(Comparator.comparing(ConnectionSchema::getName)) - .map(ConnectionSchema::toMap) - .collect(Collectors.toList())); - - map.put(CommonPropertyKeys.FUNNELS_KEY, CollectionUtil.nullToEmpty(versionedProcessGroup.getFunnels()).stream() - .map(funnelSchemaFunction) - .sorted(Comparator.comparing(FunnelSchema::getId)) - .map(FunnelSchema::toMap) - .collect(Collectors.toList())); - - map.put(CommonPropertyKeys.REMOTE_PROCESS_GROUPS_KEY, nullToEmpty(versionedProcessGroup.getRemoteProcessGroups()).stream() - .map(remoteProcessGroupSchemaFunction) - .sorted(Comparator.comparing(RemoteProcessGroupSchema::getName)) - .map(RemoteProcessGroupSchema::toMap) - .collect(Collectors.toList())); - - map.put(INPUT_PORTS_KEY, nullToEmpty(versionedProcessGroup.getInputPorts()).stream() - .map(inputPortSchemaFunction) - .sorted(Comparator.comparing(PortSchema::getName)) - .map(PortSchema::toMap) - .collect(Collectors.toList())); - - map.put(OUTPUT_PORTS_KEY, nullToEmpty(versionedProcessGroup.getOutputPorts()).stream() - .map(outputPortSchemaFunction) - .sorted(Comparator.comparing(PortSchema::getName)) - .map(PortSchema::toMap) - .collect(Collectors.toList())); - - map.put(ProcessGroupSchema.PROCESS_GROUPS_KEY, nullToEmpty(versionedProcessGroup.getProcessGroups()).stream() - .map(p -> addVersionedProcessGroup(new HashMap<>(), p.getIdentifier(), p.getName(), p)).collect(Collectors.toList())); - - return map; - } -} diff --git a/minifi/minifi-toolkit/minifi-toolkit-configuration/src/main/java/org/apache/nifi/minifi/toolkit/configuration/registry/NiFiRegConnectionSchemaFunction.java b/minifi/minifi-toolkit/minifi-toolkit-configuration/src/main/java/org/apache/nifi/minifi/toolkit/configuration/registry/NiFiRegConnectionSchemaFunction.java deleted file mode 100644 index 8256e58a08..0000000000 --- a/minifi/minifi-toolkit/minifi-toolkit-configuration/src/main/java/org/apache/nifi/minifi/toolkit/configuration/registry/NiFiRegConnectionSchemaFunction.java +++ /dev/null @@ -1,63 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.nifi.minifi.toolkit.configuration.registry; - -import org.apache.nifi.connectable.ConnectableType; -import org.apache.nifi.minifi.commons.schema.ConnectionSchema; -import org.apache.nifi.flow.VersionedConnection; - -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.function.Function; -import java.util.stream.Collectors; - -import static org.apache.nifi.minifi.commons.schema.common.CollectionUtil.nullToEmpty; -import static org.apache.nifi.minifi.commons.schema.common.CommonPropertyKeys.CONNECTIONS_KEY; -import static org.apache.nifi.minifi.commons.schema.common.CommonPropertyKeys.ID_KEY; -import static org.apache.nifi.minifi.commons.schema.common.CommonPropertyKeys.NAME_KEY; - -public class NiFiRegConnectionSchemaFunction implements Function { - - @Override - public ConnectionSchema apply(final VersionedConnection versionedConnection) { - Map map = new HashMap<>(); - map.put(ID_KEY, versionedConnection.getIdentifier()); - map.put(NAME_KEY, versionedConnection.getName()); - map.put(ConnectionSchema.SOURCE_ID_KEY, versionedConnection.getSource().getId()); - Set selectedRelationships = nullToEmpty(versionedConnection.getSelectedRelationships()); - map.put(ConnectionSchema.SOURCE_RELATIONSHIP_NAMES_KEY, selectedRelationships.stream().sorted().collect(Collectors.toList())); - map.put(ConnectionSchema.DESTINATION_ID_KEY, versionedConnection.getDestination().getId()); - - map.put(ConnectionSchema.MAX_WORK_QUEUE_SIZE_KEY, versionedConnection.getBackPressureObjectThreshold()); - map.put(ConnectionSchema.MAX_WORK_QUEUE_DATA_SIZE_KEY, versionedConnection.getBackPressureDataSizeThreshold()); - map.put(ConnectionSchema.FLOWFILE_EXPIRATION__KEY, versionedConnection.getFlowFileExpiration()); - List queuePrioritizers = nullToEmpty(versionedConnection.getPrioritizers()); - if (queuePrioritizers.size() > 0) { - map.put(ConnectionSchema.QUEUE_PRIORITIZER_CLASS_KEY, queuePrioritizers.get(0)); - } - ConnectionSchema connectionSchema = new ConnectionSchema(map); - if (ConnectableType.FUNNEL.name().equals(versionedConnection.getSource().getType())) { - connectionSchema.addValidationIssue("Connection " + versionedConnection.getName() + " has type " + ConnectableType.FUNNEL.name() + " which is not supported by MiNiFi"); - } - if (queuePrioritizers.size() > 1) { - connectionSchema.addValidationIssue(ConnectionSchema.QUEUE_PRIORITIZER_CLASS_KEY, CONNECTIONS_KEY, " has more than one queue prioritizer"); - } - return connectionSchema; - } -} diff --git a/minifi/minifi-toolkit/minifi-toolkit-configuration/src/main/java/org/apache/nifi/minifi/toolkit/configuration/registry/NiFiRegControllerServiceSchemaFunction.java b/minifi/minifi-toolkit/minifi-toolkit-configuration/src/main/java/org/apache/nifi/minifi/toolkit/configuration/registry/NiFiRegControllerServiceSchemaFunction.java deleted file mode 100644 index 99924754dd..0000000000 --- a/minifi/minifi-toolkit/minifi-toolkit-configuration/src/main/java/org/apache/nifi/minifi/toolkit/configuration/registry/NiFiRegControllerServiceSchemaFunction.java +++ /dev/null @@ -1,51 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.nifi.minifi.toolkit.configuration.registry; - -import org.apache.nifi.minifi.commons.schema.ControllerServiceSchema; -import org.apache.nifi.flow.VersionedControllerService; - -import java.util.HashMap; -import java.util.Map; -import java.util.function.Function; - -import static org.apache.nifi.minifi.commons.schema.common.CollectionUtil.nullToEmpty; -import static org.apache.nifi.minifi.commons.schema.common.CommonPropertyKeys.ANNOTATION_DATA_KEY; -import static org.apache.nifi.minifi.commons.schema.common.CommonPropertyKeys.ID_KEY; -import static org.apache.nifi.minifi.commons.schema.common.CommonPropertyKeys.NAME_KEY; -import static org.apache.nifi.minifi.commons.schema.common.CommonPropertyKeys.PROPERTIES_KEY; -import static org.apache.nifi.minifi.commons.schema.common.CommonPropertyKeys.TYPE_KEY; - -public class NiFiRegControllerServiceSchemaFunction implements Function { - - @Override - public ControllerServiceSchema apply(final VersionedControllerService versionedControllerService) { - Map map = new HashMap<>(); - map.put(NAME_KEY, versionedControllerService.getName()); - map.put(ID_KEY, versionedControllerService.getIdentifier()); - map.put(TYPE_KEY, versionedControllerService.getType()); - - map.put(PROPERTIES_KEY, new HashMap<>(nullToEmpty(versionedControllerService.getProperties()))); - - String annotationData = versionedControllerService.getAnnotationData(); - if(annotationData != null && !annotationData.isEmpty()) { - map.put(ANNOTATION_DATA_KEY, annotationData); - } - - return new ControllerServiceSchema(map); - } -} diff --git a/minifi/minifi-toolkit/minifi-toolkit-configuration/src/main/java/org/apache/nifi/minifi/toolkit/configuration/registry/NiFiRegFlowControllerSchemaFunction.java b/minifi/minifi-toolkit/minifi-toolkit-configuration/src/main/java/org/apache/nifi/minifi/toolkit/configuration/registry/NiFiRegFlowControllerSchemaFunction.java deleted file mode 100644 index d1caef98f8..0000000000 --- a/minifi/minifi-toolkit/minifi-toolkit-configuration/src/main/java/org/apache/nifi/minifi/toolkit/configuration/registry/NiFiRegFlowControllerSchemaFunction.java +++ /dev/null @@ -1,51 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.nifi.minifi.toolkit.configuration.registry; - -import org.apache.nifi.minifi.commons.schema.FlowControllerSchema; -import org.apache.nifi.minifi.commons.schema.common.CommonPropertyKeys; -import org.apache.nifi.registry.flow.VersionedFlowSnapshot; - -import java.util.HashMap; -import java.util.Map; -import java.util.function.Function; - -public class NiFiRegFlowControllerSchemaFunction implements Function { - - @Override - public FlowControllerSchema apply(final VersionedFlowSnapshot versionedFlowSnapshot) { - - // If the VersionedFlowSnapshot came directly from NiFi Registry without modification, as would be the - // case with C2 server, then we should have a non-null VersionedFlow, but if we're using a snapshot that - // was export from another tool like the CLI, the flow may be null'd out, so fall back to root group. - - String name; - String description; - if (versionedFlowSnapshot.getFlow() == null) { - name = versionedFlowSnapshot.getFlowContents().getName(); - description = versionedFlowSnapshot.getFlowContents().getComments(); - } else { - name = versionedFlowSnapshot.getFlow().getName(); - description = versionedFlowSnapshot.getFlow().getDescription(); - } - - Map map = new HashMap<>(); - map.put(CommonPropertyKeys.NAME_KEY, name); - map.put(CommonPropertyKeys.COMMENT_KEY, description); - return new FlowControllerSchema(map); - } -} diff --git a/minifi/minifi-toolkit/minifi-toolkit-configuration/src/main/java/org/apache/nifi/minifi/toolkit/configuration/registry/NiFiRegFunnelSchemaFunction.java b/minifi/minifi-toolkit/minifi-toolkit-configuration/src/main/java/org/apache/nifi/minifi/toolkit/configuration/registry/NiFiRegFunnelSchemaFunction.java deleted file mode 100644 index b279d5dcf0..0000000000 --- a/minifi/minifi-toolkit/minifi-toolkit-configuration/src/main/java/org/apache/nifi/minifi/toolkit/configuration/registry/NiFiRegFunnelSchemaFunction.java +++ /dev/null @@ -1,35 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.nifi.minifi.toolkit.configuration.registry; - -import org.apache.nifi.minifi.commons.schema.FunnelSchema; -import org.apache.nifi.flow.VersionedFunnel; - -import java.util.HashMap; -import java.util.Map; -import java.util.function.Function; - -import static org.apache.nifi.minifi.commons.schema.common.CommonPropertyKeys.ID_KEY; - -public class NiFiRegFunnelSchemaFunction implements Function { - @Override - public FunnelSchema apply(VersionedFunnel versionedFunnel) { - Map map = new HashMap<>(); - map.put(ID_KEY, versionedFunnel.getIdentifier()); - return new FunnelSchema(map); - } -} diff --git a/minifi/minifi-toolkit/minifi-toolkit-configuration/src/main/java/org/apache/nifi/minifi/toolkit/configuration/registry/NiFiRegPortSchemaFunction.java b/minifi/minifi-toolkit/minifi-toolkit-configuration/src/main/java/org/apache/nifi/minifi/toolkit/configuration/registry/NiFiRegPortSchemaFunction.java deleted file mode 100644 index a817457dd2..0000000000 --- a/minifi/minifi-toolkit/minifi-toolkit-configuration/src/main/java/org/apache/nifi/minifi/toolkit/configuration/registry/NiFiRegPortSchemaFunction.java +++ /dev/null @@ -1,43 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.nifi.minifi.toolkit.configuration.registry; - -import org.apache.nifi.minifi.commons.schema.PortSchema; -import org.apache.nifi.flow.VersionedPort; - -import java.util.HashMap; -import java.util.Map; -import java.util.function.Function; - -import static org.apache.nifi.minifi.commons.schema.common.CommonPropertyKeys.ID_KEY; -import static org.apache.nifi.minifi.commons.schema.common.CommonPropertyKeys.NAME_KEY; - -public class NiFiRegPortSchemaFunction implements Function { - private final String wrapperName; - - public NiFiRegPortSchemaFunction(String wrapperName) { - this.wrapperName = wrapperName; - } - - @Override - public PortSchema apply(VersionedPort versionedPort) { - Map map = new HashMap<>(); - map.put(ID_KEY, versionedPort.getIdentifier()); - map.put(NAME_KEY, versionedPort.getName()); - return new PortSchema(map, wrapperName); - } -} diff --git a/minifi/minifi-toolkit/minifi-toolkit-configuration/src/main/java/org/apache/nifi/minifi/toolkit/configuration/registry/NiFiRegProcessorSchemaFunction.java b/minifi/minifi-toolkit/minifi-toolkit-configuration/src/main/java/org/apache/nifi/minifi/toolkit/configuration/registry/NiFiRegProcessorSchemaFunction.java deleted file mode 100644 index 53f3ac1da4..0000000000 --- a/minifi/minifi-toolkit/minifi-toolkit-configuration/src/main/java/org/apache/nifi/minifi/toolkit/configuration/registry/NiFiRegProcessorSchemaFunction.java +++ /dev/null @@ -1,68 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.nifi.minifi.toolkit.configuration.registry; - -import org.apache.nifi.minifi.commons.schema.ProcessorSchema; -import org.apache.nifi.minifi.commons.schema.common.CommonPropertyKeys; -import org.apache.nifi.flow.VersionedProcessor; - -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.function.Function; - -import static org.apache.nifi.minifi.commons.schema.common.CollectionUtil.nullToEmpty; -import static org.apache.nifi.minifi.commons.schema.common.CommonPropertyKeys.ANNOTATION_DATA_KEY; -import static org.apache.nifi.minifi.commons.schema.common.CommonPropertyKeys.CLASS_KEY; -import static org.apache.nifi.minifi.commons.schema.common.CommonPropertyKeys.ID_KEY; -import static org.apache.nifi.minifi.commons.schema.common.CommonPropertyKeys.NAME_KEY; -import static org.apache.nifi.minifi.commons.schema.common.CommonPropertyKeys.PROPERTIES_KEY; -import static org.apache.nifi.minifi.commons.schema.common.CommonPropertyKeys.SCHEDULING_PERIOD_KEY; -import static org.apache.nifi.minifi.commons.schema.common.CommonPropertyKeys.SCHEDULING_STRATEGY_KEY; - -public class NiFiRegProcessorSchemaFunction implements Function { - @Override - public ProcessorSchema apply(final VersionedProcessor versionedProcessor) { - Map map = new HashMap<>(); - map.put(NAME_KEY, versionedProcessor.getName()); - map.put(ID_KEY, versionedProcessor.getIdentifier()); - map.put(CLASS_KEY, versionedProcessor.getType()); - map.put(SCHEDULING_STRATEGY_KEY, versionedProcessor.getSchedulingStrategy()); - map.put(SCHEDULING_PERIOD_KEY, versionedProcessor.getSchedulingPeriod()); - - map.put(CommonPropertyKeys.MAX_CONCURRENT_TASKS_KEY, versionedProcessor.getConcurrentlySchedulableTaskCount()); - map.put(ProcessorSchema.PENALIZATION_PERIOD_KEY, versionedProcessor.getPenaltyDuration()); - map.put(CommonPropertyKeys.YIELD_PERIOD_KEY, versionedProcessor.getYieldDuration()); - Long runDurationMillis = versionedProcessor.getRunDurationMillis(); - if (runDurationMillis != null) { - map.put(ProcessorSchema.RUN_DURATION_NANOS_KEY, runDurationMillis * 1000); - } - - final List autoTerminateRelationships = new ArrayList<>(nullToEmpty(versionedProcessor.getAutoTerminatedRelationships())); - map.put(ProcessorSchema.AUTO_TERMINATED_RELATIONSHIPS_LIST_KEY, autoTerminateRelationships); - - map.put(PROPERTIES_KEY, new HashMap<>(nullToEmpty(versionedProcessor.getProperties()))); - - String annotationData = versionedProcessor.getAnnotationData(); - if(annotationData != null && !annotationData.isEmpty()) { - map.put(ANNOTATION_DATA_KEY, annotationData); - } - - return new ProcessorSchema(map); - } -} diff --git a/minifi/minifi-toolkit/minifi-toolkit-configuration/src/main/java/org/apache/nifi/minifi/toolkit/configuration/registry/NiFiRegRemotePortSchemaFunction.java b/minifi/minifi-toolkit/minifi-toolkit-configuration/src/main/java/org/apache/nifi/minifi/toolkit/configuration/registry/NiFiRegRemotePortSchemaFunction.java deleted file mode 100644 index da22b74ab1..0000000000 --- a/minifi/minifi-toolkit/minifi-toolkit-configuration/src/main/java/org/apache/nifi/minifi/toolkit/configuration/registry/NiFiRegRemotePortSchemaFunction.java +++ /dev/null @@ -1,45 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.nifi.minifi.toolkit.configuration.registry; - -import org.apache.commons.lang3.StringUtils; -import org.apache.nifi.minifi.commons.schema.RemotePortSchema; -import org.apache.nifi.minifi.commons.schema.common.CommonPropertyKeys; -import org.apache.nifi.flow.VersionedRemoteGroupPort; - -import java.util.HashMap; -import java.util.Map; -import java.util.function.Function; - -import static org.apache.nifi.minifi.commons.schema.common.CommonPropertyKeys.ID_KEY; -import static org.apache.nifi.minifi.commons.schema.common.CommonPropertyKeys.NAME_KEY; - -public class NiFiRegRemotePortSchemaFunction implements Function { - @Override - public RemotePortSchema apply(VersionedRemoteGroupPort versionedRemoteGroupPort) { - Map map = new HashMap<>(); - // If a targetId is specified, it takes precedence over the original id - final String targetId = versionedRemoteGroupPort.getTargetId(); - map.put(ID_KEY, StringUtils.isNotBlank(targetId) ? targetId : versionedRemoteGroupPort.getIdentifier()); - map.put(NAME_KEY, versionedRemoteGroupPort.getName()); - - map.put(CommonPropertyKeys.COMMENT_KEY, versionedRemoteGroupPort.getComments()); - map.put(CommonPropertyKeys.MAX_CONCURRENT_TASKS_KEY, versionedRemoteGroupPort.getConcurrentlySchedulableTaskCount()); - map.put(CommonPropertyKeys.USE_COMPRESSION_KEY, versionedRemoteGroupPort.isUseCompression()); - return new RemotePortSchema(map); - } -} diff --git a/minifi/minifi-toolkit/minifi-toolkit-configuration/src/main/java/org/apache/nifi/minifi/toolkit/configuration/registry/NiFiRegRemoteProcessGroupSchemaFunction.java b/minifi/minifi-toolkit/minifi-toolkit-configuration/src/main/java/org/apache/nifi/minifi/toolkit/configuration/registry/NiFiRegRemoteProcessGroupSchemaFunction.java deleted file mode 100644 index a6f3bc9335..0000000000 --- a/minifi/minifi-toolkit/minifi-toolkit-configuration/src/main/java/org/apache/nifi/minifi/toolkit/configuration/registry/NiFiRegRemoteProcessGroupSchemaFunction.java +++ /dev/null @@ -1,76 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.nifi.minifi.toolkit.configuration.registry; - -import org.apache.nifi.minifi.commons.schema.RemotePortSchema; -import org.apache.nifi.minifi.commons.schema.RemoteProcessGroupSchema; -import org.apache.nifi.minifi.commons.schema.common.CommonPropertyKeys; -import org.apache.nifi.flow.VersionedRemoteGroupPort; -import org.apache.nifi.flow.VersionedRemoteProcessGroup; - -import java.util.HashMap; -import java.util.Map; -import java.util.Set; -import java.util.function.Function; -import java.util.stream.Collectors; - -public class NiFiRegRemoteProcessGroupSchemaFunction implements Function { - - private final NiFiRegRemotePortSchemaFunction remotePortSchemaFunction; - - public NiFiRegRemoteProcessGroupSchemaFunction(NiFiRegRemotePortSchemaFunction remotePortSchemaFunction) { - this.remotePortSchemaFunction = remotePortSchemaFunction; - } - @Override - public RemoteProcessGroupSchema apply(VersionedRemoteProcessGroup versionedRemoteProcessGroup) { - Map map = new HashMap<>(); - map.put(CommonPropertyKeys.ID_KEY, versionedRemoteProcessGroup.getIdentifier()); - map.put(CommonPropertyKeys.NAME_KEY, versionedRemoteProcessGroup.getName()); - map.put(RemoteProcessGroupSchema.URL_KEY, versionedRemoteProcessGroup.getTargetUris()); - - Set inputPorts = versionedRemoteProcessGroup.getInputPorts(); - if (inputPorts != null) { - map.put(CommonPropertyKeys.INPUT_PORTS_KEY, inputPorts.stream() - .map(remotePortSchemaFunction) - .map(RemotePortSchema::toMap) - .collect(Collectors.toList())); - } - - Set outputPorts = versionedRemoteProcessGroup.getOutputPorts(); - if (outputPorts != null) { - map.put(CommonPropertyKeys.OUTPUT_PORTS_KEY, outputPorts.stream() - .map(remotePortSchemaFunction) - .map(RemotePortSchema::toMap) - .collect(Collectors.toList())); - } - - - map.put(CommonPropertyKeys.COMMENT_KEY, versionedRemoteProcessGroup.getComments()); - map.put(RemoteProcessGroupSchema.TIMEOUT_KEY, versionedRemoteProcessGroup.getCommunicationsTimeout()); - map.put(CommonPropertyKeys.YIELD_PERIOD_KEY, versionedRemoteProcessGroup.getYieldDuration()); - map.put(RemoteProcessGroupSchema.TRANSPORT_PROTOCOL_KEY, versionedRemoteProcessGroup.getTransportProtocol()); - map.put(RemoteProcessGroupSchema.PROXY_HOST_KEY, versionedRemoteProcessGroup.getProxyHost()); - map.put(RemoteProcessGroupSchema.PROXY_PORT_KEY, versionedRemoteProcessGroup.getProxyPort()); - map.put(RemoteProcessGroupSchema.PROXY_USER_KEY, versionedRemoteProcessGroup.getProxyUser()); - - // TODO - we don't have this in registry data model, most likely templates blank it out too? - //map.put(RemoteProcessGroupSchema.PROXY_PASSWORD_KEY, versionedRemoteProcessGroup.getProxyPassword()); - - map.put(RemoteProcessGroupSchema.LOCAL_NETWORK_INTERFACE_KEY, versionedRemoteProcessGroup.getLocalNetworkInterface()); - return new RemoteProcessGroupSchema(map); - } -} diff --git a/minifi/minifi-toolkit/minifi-toolkit-configuration/src/main/java/org/apache/nifi/minifi/toolkit/configuration/registry/VersionedProcessGroupEnricher.java b/minifi/minifi-toolkit/minifi-toolkit-configuration/src/main/java/org/apache/nifi/minifi/toolkit/configuration/registry/VersionedProcessGroupEnricher.java deleted file mode 100644 index eee8ddf7b9..0000000000 --- a/minifi/minifi-toolkit/minifi-toolkit-configuration/src/main/java/org/apache/nifi/minifi/toolkit/configuration/registry/VersionedProcessGroupEnricher.java +++ /dev/null @@ -1,159 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.nifi.minifi.toolkit.configuration.registry; - -import org.apache.commons.lang3.StringUtils; -import org.apache.nifi.minifi.commons.schema.common.StringUtil; -import org.apache.nifi.flow.ConnectableComponentType; -import org.apache.nifi.flow.ConnectableComponent; -import org.apache.nifi.flow.VersionedComponent; -import org.apache.nifi.flow.VersionedConnection; -import org.apache.nifi.flow.VersionedPort; -import org.apache.nifi.flow.VersionedProcessGroup; -import org.apache.nifi.flow.VersionedRemoteGroupPort; -import org.apache.nifi.flow.VersionedRemoteProcessGroup; - -import java.util.ArrayList; -import java.util.Collection; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Objects; -import java.util.Optional; -import java.util.Set; -import java.util.function.Function; -import java.util.stream.Collectors; -import java.util.stream.Stream; - -import static org.apache.nifi.minifi.commons.schema.common.CollectionUtil.nullToEmpty; - -public class VersionedProcessGroupEnricher { - - public void enrich(final VersionedProcessGroup versionedProcessGroup) { - List allVersionedProcessGroups = getAllVersionedProcessGroups(versionedProcessGroup); - - Set remoteProcessGroups = getAll(allVersionedProcessGroups, VersionedProcessGroup::getRemoteProcessGroups).collect(Collectors.toSet()); - - Map connectableNameMap = getAll(allVersionedProcessGroups, VersionedProcessGroup::getProcessors) - .collect(Collectors.toMap(VersionedComponent::getIdentifier, VersionedComponent::getName)); - - Map rpgIdToTargetIdMap = new HashMap<>(); - - for (VersionedRemoteProcessGroup remoteProcessGroup : remoteProcessGroups) { - final Set rpgInputPorts = nullToEmpty(remoteProcessGroup.getInputPorts()); - final Set rpgOutputPorts = nullToEmpty(remoteProcessGroup.getOutputPorts()); - - // Map all port DTOs to their respective targetIds - rpgIdToTargetIdMap.putAll( - Stream.concat(rpgInputPorts.stream(), rpgOutputPorts.stream()) - .collect(Collectors.toMap(VersionedRemoteGroupPort::getIdentifier, VersionedRemoteGroupPort::getTargetId))); - - addConnectables(connectableNameMap, rpgInputPorts, VersionedRemoteGroupPort::getIdentifier, VersionedRemoteGroupPort::getIdentifier); - addConnectables(connectableNameMap, rpgOutputPorts, VersionedRemoteGroupPort::getIdentifier, VersionedRemoteGroupPort::getIdentifier); - } - - addConnectables(connectableNameMap, getAll(allVersionedProcessGroups, VersionedProcessGroup::getInputPorts) - .collect(Collectors.toList()), VersionedPort::getIdentifier, VersionedPort::getName); - - addConnectables(connectableNameMap, getAll(allVersionedProcessGroups, VersionedProcessGroup::getOutputPorts) - .collect(Collectors.toList()), VersionedPort::getIdentifier, VersionedPort::getName); - - final Set connections = getAll(allVersionedProcessGroups, VersionedProcessGroup::getConnections).collect(Collectors.toSet()); - - // Enrich connection endpoints using known names and overriding with targetIds for remote ports - for (VersionedConnection connection : connections) { - setName(connectableNameMap, connection.getSource(), rpgIdToTargetIdMap); - setName(connectableNameMap, connection.getDestination(), rpgIdToTargetIdMap); - } - - // Override any ids that are for Remote Ports to use their target Ids where available - connections.stream() - .flatMap(connectionDTO -> Stream.of(connectionDTO.getSource(), connectionDTO.getDestination())) - .filter(connectable -> (connectable.getType() == ConnectableComponentType.REMOTE_OUTPUT_PORT || connectable.getType() == ConnectableComponentType.REMOTE_INPUT_PORT)) - .forEach(connectable -> connectable.setId(Optional.ofNullable(rpgIdToTargetIdMap.get(connectable.getId())).orElse(connectable.getId()))); - - // Establish unique names for connections - for (VersionedConnection connection : connections) { - if (StringUtil.isNullOrEmpty(connection.getName())) { - StringBuilder name = new StringBuilder(); - ConnectableComponent connectionSource = connection.getSource(); - name.append(determineValueForConnectable(connectionSource, rpgIdToTargetIdMap)); - - name.append("/"); - if (connection.getSelectedRelationships() != null && connection.getSelectedRelationships().size() > 0) { - name.append(connection.getSelectedRelationships().iterator().next()); - } - - name.append("/"); - ConnectableComponent connectionDestination = connection.getDestination(); - name.append(determineValueForConnectable(connectionDestination, rpgIdToTargetIdMap)); - - connection.setName(name.toString()); - } - } - nullToEmpty(versionedProcessGroup.getProcessGroups()).stream().forEach(pg -> enrich(pg)); - } - - private static String determineValueForConnectable(ConnectableComponent connectable, Map idOverrideMap) { - String connectionName = ""; - if (connectable != null) { - connectionName = connectable.getName(); - // If no name is specified, determine the appropriate id to use, preferring any overrides specified - if (StringUtils.isBlank(connectionName)) { - connectionName = idOverrideMap.containsKey(connectable.getId()) ? idOverrideMap.get(connectable.getId()) : connectable.getId(); - } - } - return connectionName; - } - - private static Stream getAll(List allVersionedProcessGroups, Function> accessor) { - return allVersionedProcessGroups.stream().flatMap(f -> accessor.apply(f).stream()).filter(Objects::nonNull); - } - - private static List getAllVersionedProcessGroups(VersionedProcessGroup versionedProcessGroup) { - List result = new ArrayList<>(); - getAllVersionedProcessGroups(versionedProcessGroup, result); - return result; - } - - private static void getAllVersionedProcessGroups(VersionedProcessGroup versionedProcessGroup, List result) { - result.add(versionedProcessGroup); - nullToEmpty(versionedProcessGroup.getProcessGroups()).stream().forEach(f -> getAllVersionedProcessGroups(f, result)); - } - - private static void setName(Map connectableNameMap, ConnectableComponent connectable, Map nameOverrides) { - if (connectable != null) { - final String name = connectableNameMap.get(connectable.getId()); - if (name != null) { - connectable.setName(Optional.ofNullable(nameOverrides.get(connectable.getId())).orElse(name)); - } - } - } - - private static void addConnectables(Map connectableNameMap, Collection hasIdAndNames, Function idGetter, Function nameGetter) { - if (hasIdAndNames != null) { - for (T hasIdAndName : hasIdAndNames) { - String id = idGetter.apply(hasIdAndName); - String name = nameGetter.apply(hasIdAndName); - if (!StringUtil.isNullOrEmpty(name)) { - connectableNameMap.put(id, name); - } - } - } - } - -} diff --git a/minifi/minifi-toolkit/minifi-toolkit-configuration/src/test/java/org/apache/nifi/minifi/toolkit/configuration/ConfigMainTest.java b/minifi/minifi-toolkit/minifi-toolkit-configuration/src/test/java/org/apache/nifi/minifi/toolkit/configuration/ConfigMainTest.java deleted file mode 100644 index ca521641d3..0000000000 --- a/minifi/minifi-toolkit/minifi-toolkit-configuration/src/test/java/org/apache/nifi/minifi/toolkit/configuration/ConfigMainTest.java +++ /dev/null @@ -1,487 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.nifi.minifi.toolkit.configuration; - -import org.apache.nifi.controller.repository.io.LimitedInputStream; -import org.apache.nifi.minifi.commons.schema.ConfigSchema; -import org.apache.nifi.minifi.commons.schema.ConnectionSchema; -import org.apache.nifi.minifi.commons.schema.ProcessorSchema; -import org.apache.nifi.minifi.commons.schema.RemotePortSchema; -import org.apache.nifi.minifi.commons.schema.RemoteProcessGroupSchema; -import org.apache.nifi.minifi.commons.schema.common.ConvertableSchema; -import org.apache.nifi.minifi.commons.schema.serialization.SchemaLoader; -import org.apache.nifi.minifi.commons.schema.exception.SchemaLoaderException; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.extension.ExtendWith; -import org.mockito.Mock; -import org.mockito.junit.jupiter.MockitoExtension; - -import javax.xml.bind.JAXBException; -import java.io.ByteArrayInputStream; -import java.io.ByteArrayOutputStream; -import java.io.FileNotFoundException; -import java.io.IOException; -import java.io.InputStream; -import java.io.OutputStream; -import java.nio.charset.StandardCharsets; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.Set; - -import static org.apache.nifi.minifi.toolkit.configuration.ConfigMain.SUCCESS; -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertTrue; -import static org.junit.jupiter.api.Assertions.fail; -import static org.mockito.Mockito.when; - -@ExtendWith(MockitoExtension.class) -public class ConfigMainTest { - @Mock - PathInputStreamFactory pathInputStreamFactory; - - @Mock - PathOutputStreamFactory pathOutputStreamFactory; - - ConfigMain configMain; - - String testInput; - - String testOutput; - - @BeforeEach - public void setup() { - configMain = new ConfigMain(pathInputStreamFactory, pathOutputStreamFactory); - testInput = "testInput"; - testOutput = "testOutput"; - } - - @Test - public void testExecuteNoArgs() { - assertEquals(ConfigMain.ERR_INVALID_ARGS, configMain.execute(new String[0])); - } - - @Test - public void testExecuteInvalidCommand() { - assertEquals(ConfigMain.ERR_INVALID_ARGS, configMain.execute(new String[]{"badCommand"})); - } - - @Test - public void testValidateInvalidCommand() { - assertEquals(ConfigMain.ERR_INVALID_ARGS, configMain.execute(new String[]{ConfigMain.VALIDATE})); - } - - @Test - public void testValidateErrorOpeningInput() throws FileNotFoundException { - when(pathInputStreamFactory.create(testInput)).thenThrow(new FileNotFoundException()); - assertEquals(ConfigMain.ERR_UNABLE_TO_OPEN_INPUT, configMain.execute(new String[]{ConfigMain.VALIDATE, testInput})); - } - - @Test - public void testValidateUnableToParseConfig() throws FileNotFoundException { - when(pathInputStreamFactory.create(testInput)).thenReturn(new ByteArrayInputStream("!@#$%^&".getBytes(StandardCharsets.UTF_8))); - assertEquals(ConfigMain.ERR_UNABLE_TO_PARSE_CONFIG, configMain.execute(new String[]{ConfigMain.VALIDATE, testInput})); - } - - @Test - public void testValidateInvalidConfig() throws FileNotFoundException { - when(pathInputStreamFactory.create(testInput)).thenAnswer(invocation -> - ConfigMainTest.class.getClassLoader().getResourceAsStream("config-malformed-field.yml")); - assertEquals(ConfigMain.ERR_INVALID_CONFIG, configMain.execute(new String[]{ConfigMain.VALIDATE, testInput})); - } - - @Test - public void testTransformInvalidCommand() { - assertEquals(ConfigMain.ERR_INVALID_ARGS, configMain.execute(new String[]{ConfigMain.TRANSFORM})); - } - - @Test - public void testValidateSuccess() throws FileNotFoundException { - when(pathInputStreamFactory.create(testInput)).thenAnswer(invocation -> - ConfigMainTest.class.getClassLoader().getResourceAsStream("config.yml")); - assertEquals(SUCCESS, configMain.execute(new String[]{ConfigMain.VALIDATE, testInput})); - } - - @Test - public void testValidateV1Success() throws FileNotFoundException { - when(pathInputStreamFactory.create(testInput)).thenAnswer(invocation -> - ConfigMainTest.class.getClassLoader().getResourceAsStream("config-v1.yml")); - assertEquals(SUCCESS, configMain.execute(new String[]{ConfigMain.VALIDATE, testInput})); - } - - @Test - public void testTransformErrorOpeningInput() throws FileNotFoundException { - when(pathInputStreamFactory.create(testInput)).thenThrow(new FileNotFoundException()); - assertEquals(ConfigMain.ERR_UNABLE_TO_OPEN_INPUT, configMain.execute(new String[]{ConfigMain.TRANSFORM, testInput, testOutput})); - } - - @Test - public void testTransformErrorOpeningOutput() throws FileNotFoundException { - when(pathInputStreamFactory.create(testInput)).thenAnswer(invocation -> - ConfigMainTest.class.getClassLoader().getResourceAsStream("CsvToJson.xml")); - when(pathOutputStreamFactory.create(testOutput)).thenThrow(new FileNotFoundException()); - assertEquals(ConfigMain.ERR_UNABLE_TO_OPEN_OUTPUT, configMain.execute(new String[]{ConfigMain.TRANSFORM, testInput, testOutput})); - } - - @Test - public void testTransformErrorReadingTemplate() throws FileNotFoundException { - when(pathInputStreamFactory.create(testInput)).thenAnswer(invocation -> new ByteArrayInputStream("malformed xml".getBytes(StandardCharsets.UTF_8))); - assertEquals(ConfigMain.ERR_UNABLE_TO_READ_TEMPLATE, configMain.execute(new String[]{ConfigMain.TRANSFORM, testInput, testOutput})); - } - - @Test - public void testTransformErrorTransformingTemplate() throws FileNotFoundException { - when(pathInputStreamFactory.create(testInput)).thenAnswer(invocation -> - new LimitedInputStream(ConfigMainTest.class.getClassLoader().getResourceAsStream("TemplateWithFunnel.xml"), 25)); - assertEquals(ConfigMain.ERR_UNABLE_TO_READ_TEMPLATE, configMain.execute(new String[]{ConfigMain.TRANSFORM, testInput, testOutput})); - } - - @Test - public void testTransformSuccess() throws FileNotFoundException { - when(pathInputStreamFactory.create(testInput)).thenAnswer(invocation -> - ConfigMainTest.class.getClassLoader().getResourceAsStream("CsvToJson.xml")); - when(pathOutputStreamFactory.create(testOutput)).thenAnswer(invocation -> new ByteArrayOutputStream()); - assertEquals(SUCCESS, configMain.execute(new String[]{ConfigMain.TRANSFORM, testInput, testOutput})); - } - - @Test - public void testTransformRoundTripCsvToJson() throws IOException, JAXBException, SchemaLoaderException { - transformRoundTrip("CsvToJson"); - } - - @Test - public void testTransformRoundTripTemplateNoEncodingVersion() throws IOException, JAXBException, SchemaLoaderException { - transformRoundTrip("NoTemplateEncodingVersion"); - } - - @Test - public void testTransformRoundTrip15RPGHandling() throws IOException, JAXBException, SchemaLoaderException { - transformRoundTrip("1.5_RPG_Handling"); - } - - @Test - public void testTransformRoundTrip13TemplateEncoding() throws IOException, JAXBException, SchemaLoaderException { - transformRoundTrip("MINIFI-521_1.3_TemplateEncoding"); - } - - @Test - public void testTransformRoundTripDecompression() throws IOException, JAXBException, SchemaLoaderException { - transformRoundTrip("DecompressionCircularFlow"); - } - - @Test - public void testTransformRoundTripInvokeHttp() throws IOException, JAXBException, SchemaLoaderException { - transformRoundTrip("InvokeHttpMiNiFiTemplateTest"); - } - - @Test - public void testTransformRoundTripReplaceText() throws IOException, JAXBException, SchemaLoaderException { - transformRoundTrip("ReplaceTextExpressionLanguageCSVReformatting"); - } - - @Test - public void testTransformRoundTripStressTestFramework() throws IOException, JAXBException, SchemaLoaderException { - transformRoundTrip("StressTestFramework"); - } - - @Test - public void testTransformRoundTripStressTestFrameworkFunnel() throws IOException, JAXBException, SchemaLoaderException { - transformRoundTrip("StressTestFrameworkFunnel"); - } - - @Test - public void testTransformRoundTripMultipleRelationships() throws IOException, JAXBException, SchemaLoaderException { - transformRoundTrip("MultipleRelationships"); - } - - @Test - public void testTransformRoundTripProcessGroupsAndRemoteProcessGroups() throws IOException, JAXBException, SchemaLoaderException { - transformRoundTrip("ProcessGroupsAndRemoteProcessGroups"); - } - - @Test - public void testTransformRoundTripSimpleTailFileToRPG() throws IOException, JAXBException, SchemaLoaderException { - transformRoundTrip("SimpleTailFileToRPG"); - } - - @Test - public void testTransformRoundTripSimpleRPGToLogAttributes() throws IOException, JAXBException, SchemaLoaderException { - transformRoundTrip("SimpleRPGToLogAttributes"); - } - - @Test - public void testTransformRoundTripNestedControllerServices() throws IOException, JAXBException, SchemaLoaderException { - transformRoundTrip("NestedControllerServices"); - } - - @Test - public void testTransformRoundTripMultipleUriRPG() throws IOException, JAXBException, SchemaLoaderException { - transformRoundTrip("MultipleUriRPG"); - } - - @Test - public void testSuccessTransformProcessGroup() throws IOException, JAXBException { - ConfigMain.transformTemplateToSchema(getClass().getClassLoader().getResourceAsStream("TemplateWithProcessGroup.xml")).toMap(); - } - - @Test - public void testSuccessTransformInputPort() throws IOException, JAXBException { - ConfigMain.transformTemplateToSchema(getClass().getClassLoader().getResourceAsStream("TemplateWithOutputPort.xml")).toMap(); - } - - @Test - public void testSuccessTransformOutputPort() throws IOException, JAXBException { - ConfigMain.transformTemplateToSchema(getClass().getClassLoader().getResourceAsStream("TemplateWithInputPort.xml")).toMap(); - } - - @Test - public void testSuccessTransformFunnel() throws IOException, JAXBException { - ConfigMain.transformTemplateToSchema(getClass().getClassLoader().getResourceAsStream("TemplateWithFunnel.xml")).toMap(); - } - - @Test - public void testUpgradeInputFileNotFoundException() throws FileNotFoundException { - when(pathInputStreamFactory.create(testInput)).thenThrow(new FileNotFoundException()); - assertEquals(ConfigMain.ERR_UNABLE_TO_OPEN_INPUT, configMain.execute(new String[]{ConfigMain.UPGRADE, testInput, testOutput})); - } - - @Test - public void testUpgradeCantLoadSchema() throws FileNotFoundException { - when(pathInputStreamFactory.create(testInput)).thenReturn(new InputStream() { - @Override - public int read() throws IOException { - throw new IOException(); - } - }); - assertEquals(ConfigMain.ERR_UNABLE_TO_PARSE_CONFIG, configMain.execute(new String[]{ConfigMain.UPGRADE, testInput, testOutput})); - } - - @Test - public void testUpgradeOutputFileNotFoundException() throws FileNotFoundException { - when(pathInputStreamFactory.create(testInput)).thenReturn(getClass().getClassLoader().getResourceAsStream("CsvToJson-v1.yml")); - when(pathOutputStreamFactory.create(testOutput)).thenThrow(new FileNotFoundException()); - assertEquals(ConfigMain.ERR_UNABLE_TO_OPEN_OUTPUT, configMain.execute(new String[]{ConfigMain.UPGRADE, testInput, testOutput})); - } - - @Test - public void testUpgradeCantSaveSchema() throws FileNotFoundException { - when(pathInputStreamFactory.create(testInput)).thenReturn(getClass().getClassLoader().getResourceAsStream("CsvToJson-v1.yml")); - when(pathOutputStreamFactory.create(testOutput)).thenReturn(new OutputStream() { - @Override - public void write(int b) throws IOException { - throw new IOException(); - } - }); - assertEquals(ConfigMain.ERR_UNABLE_TO_SAVE_CONFIG, configMain.execute(new String[]{ConfigMain.UPGRADE, testInput, testOutput})); - } - - @Test - public void testUpgradeInvalidArgs() { - assertEquals(ConfigMain.ERR_INVALID_ARGS, configMain.execute(new String[]{ConfigMain.UPGRADE})); - } - - @Test - public void testTransformVersionedFlowSnapshotSimple() throws IOException, SchemaLoaderException { - transformVsfRoundTrip("VersionedFlowSnapshot-Simple"); - } - - @Test - public void testSuccessTransformDualRPGs() throws IOException, JAXBException { - ConfigSchema configSchema = ConfigMain.transformTemplateToSchema(getClass().getClassLoader().getResourceAsStream("MINIFI-496/dual_rpgs.xml")); - assertTrue(configSchema.isValid()); - } - - private void transformRoundTrip(String name) throws JAXBException, IOException, SchemaLoaderException { - Map templateMap = ConfigMain.transformTemplateToSchema(getClass().getClassLoader().getResourceAsStream(name + ".xml")).toMap(); - Map yamlMap = SchemaLoader.loadYamlAsMap(getClass().getClassLoader().getResourceAsStream(name + ".yml")); - assertNoMapDifferences(templateMap, yamlMap); - testV2YmlIfPresent(name, yamlMap); - testV1YmlIfPresent(name, yamlMap); - } - - private void transformVsfRoundTrip(String name) throws IOException, SchemaLoaderException { - Map templateMap = ConfigMain.transformVersionedFlowSnapshotToSchema(getClass().getClassLoader().getResourceAsStream(name + ".json")).toMap(); - Map yamlMap = SchemaLoader.loadYamlAsMap(getClass().getClassLoader().getResourceAsStream(name + ".yml")); - assertNoMapDifferences(templateMap, yamlMap); - testV2YmlIfPresent(name, yamlMap); - testV1YmlIfPresent(name, yamlMap); - } - - private InputStream upgradeAndReturn(String name) throws FileNotFoundException { - InputStream yamlV1Stream = getClass().getClassLoader().getResourceAsStream(name); - if (yamlV1Stream == null) { - return null; - } - when(pathInputStreamFactory.create(testInput)).thenReturn(yamlV1Stream); - ByteArrayOutputStream outputStream = new ByteArrayOutputStream(); - when(pathOutputStreamFactory.create(testOutput)).thenReturn(outputStream); - assertEquals(SUCCESS, configMain.execute(new String[]{"upgrade", testInput, testOutput})); - return new ByteArrayInputStream(outputStream.toByteArray()); - } - - private void testV2YmlIfPresent(String name, Map yamlMap) throws IOException, SchemaLoaderException { - InputStream upgradedInputStream = upgradeAndReturn(name + "-v2.yml"); - if (upgradedInputStream != null) { - ConvertableSchema configSchemaConvertableSchema = SchemaLoader.loadConvertableSchemaFromYaml(upgradedInputStream); - ConfigSchema configSchemaUpgradedFromV2 = configSchemaConvertableSchema.convert(); - - ConfigSchema configSchemaFromCurrent = new ConfigSchema(yamlMap); - ConfigSchema.getAllProcessGroups(configSchemaFromCurrent.getProcessGroupSchema()).stream().flatMap(p -> p.getRemoteProcessGroups().stream()).forEach(r -> { - clearV3orLaterProperties(r); - }); - - assertNoMapDifferences(configSchemaUpgradedFromV2.toMap(), configSchemaFromCurrent.toMap()); - } - } - - private void clearV3orLaterProperties(RemoteProcessGroupSchema remoteProcessGroupSchema) { - remoteProcessGroupSchema.setProxyHost(RemoteProcessGroupSchema.DEFAULT_PROXY_HOST); - remoteProcessGroupSchema.setProxyPort(RemoteProcessGroupSchema.DEFAULT_PROXY_PORT); - remoteProcessGroupSchema.setProxyUser(RemoteProcessGroupSchema.DEFAULT_PROXY_USER); - remoteProcessGroupSchema.setProxyPassword(RemoteProcessGroupSchema.DEFAULT_PROXY_PASSWORD); - remoteProcessGroupSchema.setLocalNetworkInterface(RemoteProcessGroupSchema.DEFAULT_NETWORK_INTERFACE); - } - - private void testV1YmlIfPresent(String name, Map yamlMap) throws IOException, SchemaLoaderException { - InputStream upgradedInputStream = upgradeAndReturn(name + "-v1.yml"); - if (upgradedInputStream != null) { - ConvertableSchema configSchemaConvertableSchema = SchemaLoader.loadConvertableSchemaFromYaml(upgradedInputStream); - ConfigSchema configSchemaUpgradedFromV1 = configSchemaConvertableSchema.convert(); - assertTrue(configSchemaUpgradedFromV1.isValid()); - assertEquals(configSchemaConvertableSchema, configSchemaUpgradedFromV1); - ConfigSchema configSchemaFromCurrent = new ConfigSchema(yamlMap); - List currentProcessors = configSchemaFromCurrent.getProcessGroupSchema().getProcessors(); - List v1Processors = configSchemaUpgradedFromV1.getProcessGroupSchema().getProcessors(); - assertEquals(currentProcessors.size(), v1Processors.size()); - - // V1 doesn't have ids so we need to map the autogenerated ones to the ones from the template - Map v1IdToCurrentIdMap = new HashMap<>(); - for (int i = 0; i < currentProcessors.size(); i++) { - ProcessorSchema currentProcessor = currentProcessors.get(i); - ProcessorSchema v1Processor = v1Processors.get(i); - assertEquals(currentProcessor.getName(), v1Processor.getName()); - v1IdToCurrentIdMap.put(v1Processor.getId(), currentProcessor.getId()); - v1Processor.setId(currentProcessor.getId()); - } - - List currentRPGs = configSchemaFromCurrent.getProcessGroupSchema().getRemoteProcessGroups(); - List v1RPGs = configSchemaUpgradedFromV1.getProcessGroupSchema().getRemoteProcessGroups(); - // V1 doesn't have ids so we need to map the autogenerated ones to the ones from the template - for (int i = 0; i < currentRPGs.size(); i++) { - RemoteProcessGroupSchema currentRPG = currentRPGs.get(i); - RemoteProcessGroupSchema v1RPG = v1RPGs.get(i); - assertEquals(currentRPG.getName(), v1RPG.getName()); - v1IdToCurrentIdMap.put(v1RPG.getId(), currentRPG.getId()); - v1RPG.setId(currentRPG.getId()); - } - - configSchemaUpgradedFromV1.getProcessGroupSchema().getRemoteProcessGroups().stream().flatMap(g -> g.getInputPorts().stream()).map(RemotePortSchema::getId).sequential() - .forEach(id -> v1IdToCurrentIdMap.put(id, id)); - - List currentConnections = configSchemaFromCurrent.getProcessGroupSchema().getConnections(); - List v1Connections = configSchemaUpgradedFromV1.getProcessGroupSchema().getConnections(); - - // Update source and dest ids, can set connection id equal because it isn't referenced elsewhere - assertEquals(currentConnections.size(), v1Connections.size()); - for (int i = 0; i < currentConnections.size(); i++) { - ConnectionSchema currentConnection = currentConnections.get(i); - ConnectionSchema v1Connection = v1Connections.get(i); - assertEquals(currentConnection.getName(), v1Connection.getName()); - v1Connection.setId(currentConnection.getId()); - v1Connection.setSourceId(v1IdToCurrentIdMap.get(v1Connection.getSourceId())); - v1Connection.setDestinationId(v1IdToCurrentIdMap.get(v1Connection.getDestinationId())); - } - - ConfigSchema.getAllProcessGroups(configSchemaFromCurrent.getProcessGroupSchema()).stream().flatMap(p -> p.getRemoteProcessGroups().stream()).forEach(r -> { - clearV3orLaterProperties(r); - r.setTransportProtocol(RemoteProcessGroupSchema.TransportProtocolOptions.RAW.name()); - }); - Map v1YamlMap = configSchemaUpgradedFromV1.toMap(); - assertNoMapDifferences(v1YamlMap, configSchemaFromCurrent.toMap()); - } - } - - private void assertNoMapDifferences(Map templateMap, Map yamlMap) { - List differences = new ArrayList<>(); - getMapDifferences("", differences, yamlMap, templateMap); - if (differences.size() > 0) { - fail(String.join("\n", differences.toArray(new String[differences.size()]))); - } - } - - private void getMapDifferences(String path, List differences, Map expected, Map actual) { - for (Map.Entry stringObjectEntry : expected.entrySet()) { - String key = stringObjectEntry.getKey(); - String newPath = path.isEmpty() ? key : path + "." + key; - if (!actual.containsKey(key)) { - differences.add("Missing key: " + newPath); - } else { - getObjectDifferences(newPath, differences, stringObjectEntry.getValue(), actual.get(key)); - } - } - - Set extraKeys = new HashSet<>(actual.keySet()); - extraKeys.removeAll(expected.keySet()); - for (String extraKey : extraKeys) { - differences.add("Extra key: " + path + " " + extraKey); - } - } - - private void getListDifferences(String path, List differences, List expected, List actual) { - if (expected.size() == actual.size()) { - for (int i = 0; i < expected.size(); i++) { - getObjectDifferences(path + "[" + i + "]", differences, expected.get(i), actual.get(i)); - } - } else { - differences.add("Expect size of " + expected.size() + " for list at " + path + " but got " + actual.size()); - } - } - - private void getObjectDifferences(String path, List differences, Object expectedValue, Object actualValue) { - if (expectedValue instanceof Map) { - if (actualValue instanceof Map) { - getMapDifferences(path, differences, (Map) expectedValue, (Map) actualValue); - } else { - differences.add("Expected map at " + path + " but got " + actualValue); - } - } else if (expectedValue instanceof List) { - if (actualValue instanceof List) { - getListDifferences(path, differences, (List) expectedValue, (List) actualValue); - } else { - differences.add("Expected map at " + path + " but got " + actualValue); - } - } else if (expectedValue == null) { - if (actualValue != null) { - differences.add("Expected null at " + path + " but got " + actualValue); - } - } else if (expectedValue instanceof Number) { - if (actualValue instanceof Number) { - if (!expectedValue.toString().equals(actualValue.toString())) { - differences.add("Expected value of " + expectedValue + " at " + path + " but got " + actualValue); - } - } else { - differences.add("Expected Number at " + path + " but got " + actualValue); - } - } else if (!expectedValue.equals(actualValue)) { - differences.add("Expected " + expectedValue + " at " + path + " but got " + actualValue); - } - } -} diff --git a/minifi/minifi-toolkit/minifi-toolkit-configuration/src/test/java/org/apache/nifi/minifi/toolkit/configuration/dto/BaseSchemaTester.java b/minifi/minifi-toolkit/minifi-toolkit-configuration/src/test/java/org/apache/nifi/minifi/toolkit/configuration/dto/BaseSchemaTester.java deleted file mode 100644 index dfb1eaeb3d..0000000000 --- a/minifi/minifi-toolkit/minifi-toolkit-configuration/src/test/java/org/apache/nifi/minifi/toolkit/configuration/dto/BaseSchemaTester.java +++ /dev/null @@ -1,55 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.nifi.minifi.toolkit.configuration.dto; - -import org.apache.nifi.minifi.commons.schema.common.WritableSchema; -import org.junit.jupiter.api.Test; - -import java.util.Map; -import java.util.function.Function; - -import static org.junit.jupiter.api.Assertions.assertEquals; - -public abstract class BaseSchemaTester { - protected final Function dtoSchemaFunction; - protected final Function mapSchemaFunction; - protected DTO dto; - protected Map map; - - protected BaseSchemaTester(Function dtoSchemaFunction, Function mapSchemaFunction) { - this.dtoSchemaFunction = dtoSchemaFunction; - this.mapSchemaFunction = mapSchemaFunction; - } - - protected void assertDtoAndMapConstructorAreSame(int validationErrors) { - Schema dtoSchema = dtoSchemaFunction.apply(dto); - Schema mapSchema = mapSchemaFunction.apply(map); - assertSchemaEquals(dtoSchema, mapSchema); - assertEquals(dtoSchema.getValidationIssues(), mapSchema.getValidationIssues()); - assertSchemaEquals(dtoSchema, mapSchemaFunction.apply(dtoSchema.toMap())); - assertSchemaEquals(mapSchema, mapSchemaFunction.apply(mapSchema.toMap())); - assertEquals(validationErrors, dtoSchema.getValidationIssues().size(), String.join(", ", dtoSchema.getValidationIssues())); - } - - public abstract void assertSchemaEquals(Schema one, Schema two); - - @Test - public void testFullyPopulatedSame() { - assertDtoAndMapConstructorAreSame(0); - } -} diff --git a/minifi/minifi-toolkit/minifi-toolkit-configuration/src/test/java/org/apache/nifi/minifi/toolkit/configuration/dto/ConnectionSchemaTest.java b/minifi/minifi-toolkit/minifi-toolkit-configuration/src/test/java/org/apache/nifi/minifi/toolkit/configuration/dto/ConnectionSchemaTest.java deleted file mode 100644 index c080d55dfe..0000000000 --- a/minifi/minifi-toolkit/minifi-toolkit-configuration/src/test/java/org/apache/nifi/minifi/toolkit/configuration/dto/ConnectionSchemaTest.java +++ /dev/null @@ -1,179 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.nifi.minifi.toolkit.configuration.dto; - -import org.apache.nifi.connectable.ConnectableType; -import org.apache.nifi.minifi.commons.schema.ConnectionSchema; -import org.apache.nifi.minifi.commons.schema.common.CommonPropertyKeys; -import org.apache.nifi.web.api.dto.ConnectableDTO; -import org.apache.nifi.web.api.dto.ConnectionDTO; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; - -import java.nio.charset.StandardCharsets; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collections; -import java.util.HashMap; -import java.util.LinkedHashSet; -import java.util.List; -import java.util.UUID; - -import static org.junit.jupiter.api.Assertions.assertEquals; - -public class ConnectionSchemaTest extends BaseSchemaTester { - private static final String testId = UUID.nameUUIDFromBytes("testId".getBytes(StandardCharsets.UTF_8)).toString(); - private static final String testName = "testName"; - private static final String testSourceId = "testSourceId"; - private static final String testSelectedRelationship = "testSelectedRelationship"; - private static final String testDestinationId = "testDestinationId"; - private static final long testMaxWorkQueueSize = 101L; - private static final String testMaxWorkQueueDataSize = "120 GB"; - private static final String testFlowfileExpiration = "1 day"; - private static final String testQueuePrioritizerClass = "testQueuePrioritizerClass"; - - public ConnectionSchemaTest() { - super(new ConnectionSchemaFunction(), ConnectionSchema::new); - } - - @BeforeEach - public void setup() { - ConnectableDTO source = new ConnectableDTO(); - source.setId(testSourceId); - - ConnectableDTO destination = new ConnectableDTO(); - destination.setId(testDestinationId); - - dto = new ConnectionDTO(); - dto.setId(testId); - dto.setName(testName); - dto.setSource(source); - dto.setSelectedRelationships(Collections.singleton(testSelectedRelationship)); - dto.setDestination(destination); - dto.setBackPressureObjectThreshold(testMaxWorkQueueSize); - dto.setBackPressureDataSizeThreshold(testMaxWorkQueueDataSize); - dto.setFlowFileExpiration(testFlowfileExpiration); - dto.setPrioritizers(Collections.singletonList(testQueuePrioritizerClass)); - - map = new HashMap<>(); - map.put(CommonPropertyKeys.ID_KEY, testId); - map.put(CommonPropertyKeys.NAME_KEY, testName); - map.put(ConnectionSchema.SOURCE_ID_KEY, testSourceId); - map.put(ConnectionSchema.SOURCE_RELATIONSHIP_NAMES_KEY, new ArrayList<>(Collections.singletonList(testSelectedRelationship))); - map.put(ConnectionSchema.DESTINATION_ID_KEY, testDestinationId); - map.put(ConnectionSchema.MAX_WORK_QUEUE_SIZE_KEY, testMaxWorkQueueSize); - map.put(ConnectionSchema.MAX_WORK_QUEUE_DATA_SIZE_KEY, testMaxWorkQueueDataSize); - map.put(ConnectionSchema.FLOWFILE_EXPIRATION__KEY, testFlowfileExpiration); - map.put(ConnectionSchema.QUEUE_PRIORITIZER_CLASS_KEY, testQueuePrioritizerClass); - } - - @Test - public void testNoName() { - dto.setName(null); - map.remove(CommonPropertyKeys.NAME_KEY); - assertDtoAndMapConstructorAreSame(0); - } - - @Test - public void testNoId() { - dto.setId(null); - map.remove(CommonPropertyKeys.ID_KEY); - assertDtoAndMapConstructorAreSame(1); - } - - @Test - public void testNoSourceId() { - dto.setSource(new ConnectableDTO()); - map.remove(ConnectionSchema.SOURCE_ID_KEY); - assertDtoAndMapConstructorAreSame(1); - } - - @Test - public void testDtoMultipleSourceRelationships() { - List relationships = Arrays.asList("one", "two"); - dto.setSelectedRelationships(new LinkedHashSet<>(relationships)); - map.put(ConnectionSchema.SOURCE_RELATIONSHIP_NAMES_KEY, new ArrayList<>(relationships)); - assertDtoAndMapConstructorAreSame(0); - } - - @Test - public void testNoSelectedRelationshipName() { - dto.setSelectedRelationships(null); - map.remove(ConnectionSchema.SOURCE_RELATIONSHIP_NAMES_KEY); - assertDtoAndMapConstructorAreSame(1); - dto.setSelectedRelationships(Collections.emptySet()); - map.put(ConnectionSchema.SOURCE_RELATIONSHIP_NAMES_KEY, new ArrayList<>()); - assertDtoAndMapConstructorAreSame(1); - } - - @Test - public void testNoDestinationName() { - dto.setDestination(new ConnectableDTO()); - map.remove(ConnectionSchema.DESTINATION_ID_KEY); - assertDtoAndMapConstructorAreSame(1); - } - - @Test - public void testNoMaxWorkQueueSize() { - dto.setBackPressureObjectThreshold(null); - map.remove(ConnectionSchema.MAX_WORK_QUEUE_SIZE_KEY); - assertDtoAndMapConstructorAreSame(0); - } - - @Test - public void testNoMaxWorkQueueDataSize() { - dto.setBackPressureDataSizeThreshold(null); - map.remove(ConnectionSchema.MAX_WORK_QUEUE_DATA_SIZE_KEY); - assertDtoAndMapConstructorAreSame(0); - } - - @Test - public void testNoFlowFileExpiration() { - dto.setFlowFileExpiration(null); - map.remove(ConnectionSchema.FLOWFILE_EXPIRATION__KEY); - assertDtoAndMapConstructorAreSame(0); - } - - @Test - public void testNoQueuePrioritizerClass() { - dto.setPrioritizers(null); - map.remove(ConnectionSchema.QUEUE_PRIORITIZER_CLASS_KEY); - assertDtoAndMapConstructorAreSame(0); - dto.setPrioritizers(Collections.emptyList()); - assertDtoAndMapConstructorAreSame(0); - } - - @Test - public void testFunnelValidationMessage() { - dto.getSource().setType(ConnectableType.FUNNEL.name()); - assertEquals(1, dtoSchemaFunction.apply(dto).getValidationIssues().size()); - } - - @Override - public void assertSchemaEquals(ConnectionSchema one, ConnectionSchema two) { - assertEquals(one.getName(), two.getName()); - assertEquals(one.getId(), two.getId()); - assertEquals(one.getSourceId(), two.getSourceId()); - assertEquals(one.getSourceRelationshipNames(), two.getSourceRelationshipNames()); - assertEquals(one.getDestinationId(), two.getDestinationId()); - assertEquals(one.getMaxWorkQueueSize(), two.getMaxWorkQueueSize()); - assertEquals(one.getMaxWorkQueueDataSize(), two.getMaxWorkQueueDataSize()); - assertEquals(one.getFlowfileExpiration(), two.getFlowfileExpiration()); - assertEquals(one.getQueuePrioritizerClass(), two.getQueuePrioritizerClass()); - } -} diff --git a/minifi/minifi-toolkit/minifi-toolkit-configuration/src/test/java/org/apache/nifi/minifi/toolkit/configuration/dto/FlowControllerSchemaTest.java b/minifi/minifi-toolkit/minifi-toolkit-configuration/src/test/java/org/apache/nifi/minifi/toolkit/configuration/dto/FlowControllerSchemaTest.java deleted file mode 100644 index 6347d5a565..0000000000 --- a/minifi/minifi-toolkit/minifi-toolkit-configuration/src/test/java/org/apache/nifi/minifi/toolkit/configuration/dto/FlowControllerSchemaTest.java +++ /dev/null @@ -1,70 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.nifi.minifi.toolkit.configuration.dto; - -import org.apache.nifi.minifi.commons.schema.FlowControllerSchema; -import org.apache.nifi.minifi.commons.schema.common.CommonPropertyKeys; -import org.apache.nifi.web.api.dto.TemplateDTO; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; - -import java.util.HashMap; - -import static org.junit.jupiter.api.Assertions.assertEquals; - -public class FlowControllerSchemaTest extends BaseSchemaTester { - private static final String TEST_NAME = "testName"; - private static final String TEST_COMMENT = "testComment"; - - public FlowControllerSchemaTest() { - super(new FlowControllerSchemaFunction(), FlowControllerSchema::new); - } - - @BeforeEach - public void setup() { - dto = new TemplateDTO(); - - dto.setName(TEST_NAME); - dto.setDescription(TEST_COMMENT); - - map = new HashMap<>(); - - map.put(CommonPropertyKeys.NAME_KEY, TEST_NAME); - map.put(CommonPropertyKeys.COMMENT_KEY, TEST_COMMENT); - } - - @Test - public void testNoNameSame() { - dto.setName(null); - map.remove(CommonPropertyKeys.NAME_KEY); - assertDtoAndMapConstructorAreSame(1); - } - - @Test - public void testNoCommentSame() { - dto.setDescription(null); - map.remove(CommonPropertyKeys.COMMENT_KEY); - assertDtoAndMapConstructorAreSame(0); - } - - @Override - public void assertSchemaEquals(FlowControllerSchema one, FlowControllerSchema two) { - assertEquals(one.getName(), two.getName()); - assertEquals(one.getComment(), two.getComment()); - } -} diff --git a/minifi/minifi-toolkit/minifi-toolkit-configuration/src/test/java/org/apache/nifi/minifi/toolkit/configuration/dto/PortSchemaFunctionTest.java b/minifi/minifi-toolkit/minifi-toolkit-configuration/src/test/java/org/apache/nifi/minifi/toolkit/configuration/dto/PortSchemaFunctionTest.java deleted file mode 100644 index 9adae7e5f6..0000000000 --- a/minifi/minifi-toolkit/minifi-toolkit-configuration/src/test/java/org/apache/nifi/minifi/toolkit/configuration/dto/PortSchemaFunctionTest.java +++ /dev/null @@ -1,75 +0,0 @@ -/* - * - * * Licensed to the Apache Software Foundation (ASF) under one or more - * * contributor license agreements. See the NOTICE file distributed with - * * this work for additional information regarding copyright ownership. - * * The ASF licenses this file to You under the Apache License, Version 2.0 - * * (the "License"); you may not use this file except in compliance with - * * the License. You may obtain a copy of the License at - * * - * * http://www.apache.org/licenses/LICENSE-2.0 - * * - * * Unless required by applicable law or agreed to in writing, software - * * distributed under the License is distributed on an "AS IS" BASIS, - * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * * See the License for the specific language governing permissions and - * * limitations under the License. - * - */ - -package org.apache.nifi.minifi.toolkit.configuration.dto; - -import org.apache.nifi.minifi.commons.schema.PortSchema; -import org.apache.nifi.web.api.dto.PortDTO; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; - -import java.nio.charset.StandardCharsets; -import java.util.UUID; - -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertFalse; -import static org.junit.jupiter.api.Assertions.assertTrue; - -public class PortSchemaFunctionTest { - private String testId; - private String testName; - private PortDTO portDTO; - private PortSchemaFunction portSchemaFunction; - - @BeforeEach - public void setup() { - testId = UUID.nameUUIDFromBytes("testId".getBytes(StandardCharsets.UTF_8)).toString(); - testName = "testName"; - portDTO = new PortDTO(); - portDTO.setId(testId); - portDTO.setName(testName); - portSchemaFunction = new PortSchemaFunction("testWrapperName"); - } - - @Test - public void testFullMap() { - PortSchema portSchema = portSchemaFunction.apply(portDTO); - assertEquals(testId, portSchema.getId()); - assertEquals(testName, portSchema.getName()); - assertTrue(portSchema.isValid()); - } - - @Test - public void testNoId() { - portDTO.setId(null); - PortSchema portSchema = portSchemaFunction.apply(portDTO); - assertEquals("", portSchema.getId()); - assertEquals(testName, portSchema.getName()); - assertFalse(portSchema.isValid()); - } - - @Test - public void testNoName() { - portDTO.setName(null); - PortSchema portSchema = portSchemaFunction.apply(portDTO); - assertEquals(testId, portSchema.getId()); - assertEquals("", portSchema.getName()); - assertTrue(portSchema.isValid()); - } -} diff --git a/minifi/minifi-toolkit/minifi-toolkit-configuration/src/test/java/org/apache/nifi/minifi/toolkit/configuration/dto/ProcessorSchemaTest.java b/minifi/minifi-toolkit/minifi-toolkit-configuration/src/test/java/org/apache/nifi/minifi/toolkit/configuration/dto/ProcessorSchemaTest.java deleted file mode 100644 index 8fb44115b8..0000000000 --- a/minifi/minifi-toolkit/minifi-toolkit-configuration/src/test/java/org/apache/nifi/minifi/toolkit/configuration/dto/ProcessorSchemaTest.java +++ /dev/null @@ -1,234 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.nifi.minifi.toolkit.configuration.dto; - -import org.apache.nifi.minifi.commons.schema.ProcessorSchema; -import org.apache.nifi.minifi.commons.schema.common.CommonPropertyKeys; -import org.apache.nifi.scheduling.SchedulingStrategy; -import org.apache.nifi.web.api.dto.ProcessorConfigDTO; -import org.apache.nifi.web.api.dto.ProcessorDTO; -import org.apache.nifi.web.api.dto.RelationshipDTO; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; - -import java.nio.charset.StandardCharsets; -import java.util.Arrays; -import java.util.HashMap; -import java.util.Map; -import java.util.UUID; - -import static org.apache.nifi.minifi.commons.schema.common.CommonPropertyKeys.ANNOTATION_DATA_KEY; -import static org.apache.nifi.minifi.commons.schema.common.CommonPropertyKeys.CLASS_KEY; -import static org.apache.nifi.minifi.commons.schema.common.CommonPropertyKeys.PROPERTIES_KEY; -import static org.junit.jupiter.api.Assertions.assertEquals; - -public class ProcessorSchemaTest extends BaseSchemaTester { - private final String testName = "testName"; - private final String testId = UUID.nameUUIDFromBytes("testId".getBytes(StandardCharsets.UTF_8)).toString(); - private final String testProcessorClass = "testProcessorClass"; - private final String testSchedulingStrategy = SchedulingStrategy.TIMER_DRIVEN.name(); - private final String testSchedulingPeriod = "10 s"; - private final int testMaxConcurrentTasks = 101; - private final String testYieldDuration = "5 s"; - private final long testRunDurationNanos = 1111000L; - private final String testRelationship = "testRelationship"; - private final String testKey = "testKey"; - private final String testValue = "testValue"; - private final String testPenalizationPeriod = "55 s"; - private final String testAnnotationData = "<criteria>\n" + - " <flowFilePolicy>USE_ORIGINAL</flowFilePolicy>\n" + - " <rules>\n" + - " <actions>\n" + - " <attribute>it's the one</attribute>\n" + - " <id>364cdf1a-3ac8-46a7-9d5d-e83618d9dfde</id>\n" + - " <value>true</value>\n" + - " </actions>\n" + - " <conditions>\n" + - " <expression>${uuid:equals(\"theUUID!\")}</expression>\n" + - " <id>c0ceadcd-6de5-4994-bf76-f76a5fd1640c</id>\n" + - " </conditions>\n" + - " <id>e6fb8d10-49ac-494b-8f02-ced1109c9445</id>\n" + - " <name>testRule</name>\n" + - " </rules>\n" + - " <rules>\n" + - " <actions>\n" + - " <attribute>the tenth</attribute>\n" + - " <id>0bbe7bcd-e6b9-4801-b1e4-337da5f0532f</id>\n" + - " <value>${hostname()}</value>\n" + - " </actions>\n" + - " <conditions>\n" + - " <expression>${random:mod(10):equals(0)}</expression>\n" + - " <id>48735bfe-02ef-4634-8d99-f384ebcd8fa8</id>\n" + - " </conditions>\n" + - " <id>74b86ba2-4a8b-4952-8aef-ea672847c589</id>\n" + - " <name>testRule3</name>\n" + - " </rules>\n" + - "</criteria>"; - private ProcessorConfigDTO config; - - public ProcessorSchemaTest() { - super(new ProcessorSchemaFunction(), ProcessorSchema::new); - } - - @BeforeEach - public void setup() { - config = new ProcessorConfigDTO(); - - RelationshipDTO relationshipDTO = new RelationshipDTO(); - relationshipDTO.setName(testRelationship); - relationshipDTO.setAutoTerminate(true); - - dto = new ProcessorDTO(); - dto.setConfig(config); - dto.setName(testName); - dto.setId(testId); - dto.setType(testProcessorClass); - config.setSchedulingStrategy(testSchedulingStrategy); - config.setSchedulingPeriod(testSchedulingPeriod); - config.setConcurrentlySchedulableTaskCount(testMaxConcurrentTasks); - config.setPenaltyDuration(testPenalizationPeriod); - config.setYieldDuration(testYieldDuration); - config.setRunDurationMillis(testRunDurationNanos / 1000); - config.setAnnotationData(testAnnotationData); - dto.setRelationships(Arrays.asList(relationshipDTO)); - Map properties = new HashMap<>(); - properties.put(testKey, testValue); - config.setProperties(properties); - - map = new HashMap<>(); - map.put(CommonPropertyKeys.NAME_KEY, testName); - map.put(CommonPropertyKeys.ID_KEY, testId); - map.put(CLASS_KEY, testProcessorClass); - map.put(CommonPropertyKeys.SCHEDULING_STRATEGY_KEY, testSchedulingStrategy); - map.put(CommonPropertyKeys.SCHEDULING_PERIOD_KEY, testSchedulingPeriod); - map.put(CommonPropertyKeys.MAX_CONCURRENT_TASKS_KEY, testMaxConcurrentTasks); - map.put(ProcessorSchema.PENALIZATION_PERIOD_KEY, testPenalizationPeriod); - map.put(CommonPropertyKeys.YIELD_PERIOD_KEY, testYieldDuration); - map.put(ProcessorSchema.RUN_DURATION_NANOS_KEY, testRunDurationNanos); - map.put(ProcessorSchema.AUTO_TERMINATED_RELATIONSHIPS_LIST_KEY, Arrays.asList(testRelationship)); - map.put(PROPERTIES_KEY, new HashMap<>(properties)); - map.put(ANNOTATION_DATA_KEY, testAnnotationData); - } - - @Test - public void testNoName() { - dto.setName(null); - map.remove(CommonPropertyKeys.NAME_KEY); - assertDtoAndMapConstructorAreSame(0); - } - - @Test - public void testNoId() { - dto.setId(null); - map.remove(CommonPropertyKeys.ID_KEY); - assertDtoAndMapConstructorAreSame(1); - } - - @Test - public void testNoProcessorClass() { - dto.setType(null); - map.remove(CLASS_KEY); - assertDtoAndMapConstructorAreSame(1); - } - - @Test - public void testNoSchedulingStrategy() { - config.setSchedulingStrategy(null); - map.remove(CommonPropertyKeys.SCHEDULING_STRATEGY_KEY); - assertDtoAndMapConstructorAreSame(1); - } - - @Test - public void testInvalidSchedulingStrategy() { - String fake = "fake"; - config.setSchedulingStrategy(fake); - map.put(CommonPropertyKeys.SCHEDULING_STRATEGY_KEY, fake); - assertDtoAndMapConstructorAreSame(1); - } - - @Test - public void testNoSchedulingPeriod() { - config.setSchedulingPeriod(null); - map.remove(CommonPropertyKeys.SCHEDULING_PERIOD_KEY); - assertDtoAndMapConstructorAreSame(1); - } - - @Test - public void testNoMaxConcurrentTasks() { - config.setConcurrentlySchedulableTaskCount(null); - map.remove(CommonPropertyKeys.MAX_CONCURRENT_TASKS_KEY); - assertDtoAndMapConstructorAreSame(0); - } - - @Test - public void testNoPenalizationPeriod() { - config.setPenaltyDuration(null); - map.remove(ProcessorSchema.PENALIZATION_PERIOD_KEY); - assertDtoAndMapConstructorAreSame(0); - } - - @Test - public void testNoYieldPeriod() { - config.setYieldDuration(null); - map.remove(CommonPropertyKeys.YIELD_PERIOD_KEY); - assertDtoAndMapConstructorAreSame(0); - } - - @Test - public void testNoRunDurationNanos() { - config.setRunDurationMillis(null); - map.remove(ProcessorSchema.RUN_DURATION_NANOS_KEY); - assertDtoAndMapConstructorAreSame(0); - } - - @Test - public void testNoAutoTerminateRelationships() { - dto.setRelationships(null); - map.remove(ProcessorSchema.AUTO_TERMINATED_RELATIONSHIPS_LIST_KEY); - assertDtoAndMapConstructorAreSame(0); - } - - @Test - public void testNoProperties() { - config.setProperties(null); - map.remove(PROPERTIES_KEY); - assertDtoAndMapConstructorAreSame(0); - } - - @Test - public void testNoAnnotationData() { - config.setAnnotationData(null); - map.remove(ANNOTATION_DATA_KEY); - assertDtoAndMapConstructorAreSame(0); - } - - @Override - public void assertSchemaEquals(ProcessorSchema one, ProcessorSchema two) { - assertEquals(one.getName(), two.getName()); - assertEquals(one.getProcessorClass(), two.getProcessorClass()); - assertEquals(one.getSchedulingStrategy(), two.getSchedulingStrategy()); - assertEquals(one.getSchedulingPeriod(), two.getSchedulingPeriod()); - assertEquals(one.getMaxConcurrentTasks(), two.getMaxConcurrentTasks()); - assertEquals(one.getPenalizationPeriod(), two.getPenalizationPeriod()); - assertEquals(one.getYieldPeriod(), two.getYieldPeriod()); - assertEquals(one.getRunDurationNanos(), two.getRunDurationNanos()); - assertEquals(one.getAutoTerminatedRelationshipsList(), two.getAutoTerminatedRelationshipsList()); - assertEquals(one.getProperties(), two.getProperties()); - assertEquals(one.getAnnotationData(), two.getAnnotationData()); - } -} diff --git a/minifi/minifi-toolkit/minifi-toolkit-configuration/src/test/java/org/apache/nifi/minifi/toolkit/configuration/dto/RemoteInputPortSchemaTest.java b/minifi/minifi-toolkit/minifi-toolkit-configuration/src/test/java/org/apache/nifi/minifi/toolkit/configuration/dto/RemoteInputPortSchemaTest.java deleted file mode 100644 index 15113569e0..0000000000 --- a/minifi/minifi-toolkit/minifi-toolkit-configuration/src/test/java/org/apache/nifi/minifi/toolkit/configuration/dto/RemoteInputPortSchemaTest.java +++ /dev/null @@ -1,104 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.nifi.minifi.toolkit.configuration.dto; - -import org.apache.nifi.minifi.commons.schema.RemotePortSchema; -import org.apache.nifi.minifi.commons.schema.common.CommonPropertyKeys; -import org.apache.nifi.web.api.dto.RemoteProcessGroupPortDTO; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; - -import java.nio.charset.StandardCharsets; -import java.util.HashMap; -import java.util.UUID; - -import static org.junit.jupiter.api.Assertions.assertEquals; - -public class RemoteInputPortSchemaTest extends BaseSchemaTester { - - private String testId = UUID.nameUUIDFromBytes("testId".getBytes(StandardCharsets.UTF_8)).toString(); - private String testName = "testName"; - private String testComment = "testComment"; - private int testMaxConcurrentTasks = 111; - private boolean testUseCompression = false; - - public RemoteInputPortSchemaTest() { - super(new RemotePortSchemaFunction(), RemotePortSchema::new); - } - - @BeforeEach - public void setup() { - dto = new RemoteProcessGroupPortDTO(); - dto.setId(testId); - dto.setName(testName); - dto.setComments(testComment); - dto.setConcurrentlySchedulableTaskCount(testMaxConcurrentTasks); - dto.setUseCompression(testUseCompression); - - map = new HashMap<>(); - map.put(CommonPropertyKeys.ID_KEY, testId); - map.put(CommonPropertyKeys.NAME_KEY, testName); - map.put(CommonPropertyKeys.COMMENT_KEY, testComment); - map.put(CommonPropertyKeys.MAX_CONCURRENT_TASKS_KEY, testMaxConcurrentTasks); - map.put(CommonPropertyKeys.USE_COMPRESSION_KEY, testUseCompression); - } - - @Test - public void testNoId() { - dto.setId(null); - map.remove(CommonPropertyKeys.ID_KEY); - assertDtoAndMapConstructorAreSame(1); - } - - @Test - public void testNoName() { - dto.setName(null); - map.remove(CommonPropertyKeys.NAME_KEY); - assertDtoAndMapConstructorAreSame(0); - } - - @Test - public void testNoComment() { - dto.setComments(null); - map.remove(CommonPropertyKeys.COMMENT_KEY); - assertDtoAndMapConstructorAreSame(0); - } - - @Test - public void testNoMaxConcurrentTasks() { - dto.setConcurrentlySchedulableTaskCount(null); - map.remove(CommonPropertyKeys.MAX_CONCURRENT_TASKS_KEY); - assertDtoAndMapConstructorAreSame(0); - } - - @Test - public void testNoUseCompression() { - dto.setUseCompression(null); - map.remove(CommonPropertyKeys.USE_COMPRESSION_KEY); - assertDtoAndMapConstructorAreSame(0); - } - - @Override - public void assertSchemaEquals(RemotePortSchema one, RemotePortSchema two) { - assertEquals(one.getId(), two.getId()); - assertEquals(one.getName(), two.getName()); - assertEquals(one.getComment(), two.getComment()); - assertEquals(one.getMax_concurrent_tasks(), two.getMax_concurrent_tasks()); - assertEquals(one.getUseCompression(), two.getUseCompression()); - } -} diff --git a/minifi/minifi-toolkit/minifi-toolkit-configuration/src/test/java/org/apache/nifi/minifi/toolkit/configuration/dto/RemoteProcessGroupSchemaTest.java b/minifi/minifi-toolkit/minifi-toolkit-configuration/src/test/java/org/apache/nifi/minifi/toolkit/configuration/dto/RemoteProcessGroupSchemaTest.java deleted file mode 100644 index 2a912fb47c..0000000000 --- a/minifi/minifi-toolkit/minifi-toolkit-configuration/src/test/java/org/apache/nifi/minifi/toolkit/configuration/dto/RemoteProcessGroupSchemaTest.java +++ /dev/null @@ -1,176 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.nifi.minifi.toolkit.configuration.dto; - -import org.apache.nifi.minifi.commons.schema.RemotePortSchema; -import org.apache.nifi.minifi.commons.schema.RemoteProcessGroupSchema; -import org.apache.nifi.minifi.commons.schema.common.CommonPropertyKeys; -import org.apache.nifi.web.api.dto.RemoteProcessGroupContentsDTO; -import org.apache.nifi.web.api.dto.RemoteProcessGroupDTO; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; - -import java.util.ArrayList; -import java.util.Collections; -import java.util.HashMap; -import java.util.List; -import java.util.UUID; - -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertNotNull; -import static org.junit.jupiter.api.Assertions.assertNull; - -public class RemoteProcessGroupSchemaTest extends BaseSchemaTester { - private final RemoteInputPortSchemaTest remoteInputPortSchemaTest; - private String testId = UUID.randomUUID().toString(); - private String testName = "testName"; - private String testUrl = "testUrl"; - private String testComment = "testComment"; - private String testTimeout = "11 s"; - private String testYieldPeriod = "22 s"; - private String transportProtocol = "HTTP"; - private String localNetworkInterface = "eth0"; - - public RemoteProcessGroupSchemaTest() { - super(new RemoteProcessGroupSchemaFunction(new RemotePortSchemaFunction()), RemoteProcessGroupSchema::new); - remoteInputPortSchemaTest = new RemoteInputPortSchemaTest(); - } - - @BeforeEach - public void setup() { - remoteInputPortSchemaTest.setup(); - - dto = new RemoteProcessGroupDTO(); - dto.setId(testId); - dto.setName(testName); - dto.setTargetUri(testUrl); - - RemoteProcessGroupContentsDTO contents = new RemoteProcessGroupContentsDTO(); - contents.setInputPorts(Collections.singleton(remoteInputPortSchemaTest.dto)); - dto.setContents(contents); - - dto.setComments(testComment); - dto.setCommunicationsTimeout(testTimeout); - dto.setYieldDuration(testYieldPeriod); - dto.setTransportProtocol(transportProtocol); - - map = new HashMap<>(); - - map.put(CommonPropertyKeys.ID_KEY, testId); - map.put(CommonPropertyKeys.NAME_KEY, testName); - map.put(RemoteProcessGroupSchema.URL_KEY, testUrl); - map.put(CommonPropertyKeys.INPUT_PORTS_KEY, new ArrayList<>(Collections.singletonList(remoteInputPortSchemaTest.map))); - map.put(CommonPropertyKeys.COMMENT_KEY, testComment); - map.put(RemoteProcessGroupSchema.TIMEOUT_KEY, testTimeout); - map.put(CommonPropertyKeys.YIELD_PERIOD_KEY, testYieldPeriod); - map.put(RemoteProcessGroupSchema.TRANSPORT_PROTOCOL_KEY, transportProtocol); - } - - @Test - public void testNoId() { - dto.setId(null); - map.remove(CommonPropertyKeys.ID_KEY); - assertDtoAndMapConstructorAreSame(1); - } - - @Test - public void testNoName() { - dto.setName(null); - map.remove(CommonPropertyKeys.NAME_KEY); - assertDtoAndMapConstructorAreSame(0); - } - - @Test - public void testNoUrl() { - dto.setTargetUri(null); - map.remove(RemoteProcessGroupSchema.URL_KEY); - assertDtoAndMapConstructorAreSame(1); - } - - @Test - public void testNoInputPorts() { - dto.getContents().setInputPorts(null); - map.remove(CommonPropertyKeys.INPUT_PORTS_KEY); - assertDtoAndMapConstructorAreSame(1); - } - - @Test - public void testNoComment() { - dto.setComments(null); - map.remove(CommonPropertyKeys.COMMENT_KEY); - assertDtoAndMapConstructorAreSame(0); - } - - @Test - public void testNoTimeout() { - dto.setCommunicationsTimeout(null); - map.remove(RemoteProcessGroupSchema.TIMEOUT_KEY); - assertDtoAndMapConstructorAreSame(0); - } - - @Test - public void testNoYield() { - dto.setYieldDuration(null); - map.remove(CommonPropertyKeys.YIELD_PERIOD_KEY); - assertDtoAndMapConstructorAreSame(0); - } - - @Test - public void testNoTransportProtocol() { - dto.setTransportProtocol(null); - map.remove(RemoteProcessGroupSchema.TRANSPORT_PROTOCOL_KEY); - assertDtoAndMapConstructorAreSame(0); - } - - @Test - public void testNoLocalNetworkInterface() { - dto.setLocalNetworkInterface(null); - map.remove(RemoteProcessGroupSchema.LOCAL_NETWORK_INTERFACE_KEY); - assertDtoAndMapConstructorAreSame(0); - } - - @Test - public void testLocalNetworkInterface() { - dto.setLocalNetworkInterface(localNetworkInterface); - map.put(RemoteProcessGroupSchema.LOCAL_NETWORK_INTERFACE_KEY, localNetworkInterface); - assertDtoAndMapConstructorAreSame(0); - } - - @Override - public void assertSchemaEquals(RemoteProcessGroupSchema one, RemoteProcessGroupSchema two) { - assertEquals(one.getName(), two.getName()); - assertEquals(one.getUrls(), two.getUrls()); - - List oneInputPorts = one.getInputPorts(); - List twoInputPorts = two.getInputPorts(); - if (oneInputPorts == null) { - assertNull(twoInputPorts); - } else { - assertNotNull(twoInputPorts); - assertEquals(oneInputPorts.size(), twoInputPorts.size()); - for (int i = 0; i < oneInputPorts.size(); i++) { - remoteInputPortSchemaTest.assertSchemaEquals(oneInputPorts.get(i), twoInputPorts.get(i)); - } - } - - assertEquals(one.getComment(), two.getComment()); - assertEquals(one.getTimeout(), two.getTimeout()); - assertEquals(one.getYieldPeriod(), two.getYieldPeriod()); - assertEquals(one.getLocalNetworkInterface(), two.getLocalNetworkInterface()); - } -} diff --git a/minifi/minifi-toolkit/minifi-toolkit-configuration/src/test/resources/1.5_RPG_Handling.xml b/minifi/minifi-toolkit/minifi-toolkit-configuration/src/test/resources/1.5_RPG_Handling.xml deleted file mode 100644 index 49cfd45944..0000000000 --- a/minifi/minifi-toolkit/minifi-toolkit-configuration/src/test/resources/1.5_RPG_Handling.xml +++ /dev/null @@ -1,173 +0,0 @@ - - - - - \ No newline at end of file diff --git a/minifi/minifi-toolkit/minifi-toolkit-configuration/src/test/resources/1.5_RPG_Handling.yml b/minifi/minifi-toolkit/minifi-toolkit-configuration/src/test/resources/1.5_RPG_Handling.yml deleted file mode 100644 index 82b2f55392..0000000000 --- a/minifi/minifi-toolkit/minifi-toolkit-configuration/src/test/resources/1.5_RPG_Handling.yml +++ /dev/null @@ -1,122 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the \"License\"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an \"AS IS\" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -MiNiFi Config Version: 3 -Flow Controller: - name: 1.5 RPG Handling - comment: '' -Core Properties: - flow controller graceful shutdown period: 10 sec - flow service write delay interval: 500 ms - administrative yield duration: 30 sec - bored yield duration: 10 millis - max concurrent threads: 1 - variable registry properties: '' -FlowFile Repository: - implementation: org.apache.nifi.controller.repository.WriteAheadFlowFileRepository - partitions: 256 - checkpoint interval: 2 mins - always sync: false - Swap: - threshold: 20000 - in period: 5 sec - in threads: 1 - out period: 5 sec - out threads: 4 -Content Repository: - implementation: org.apache.nifi.controller.repository.FileSystemRepository - content claim max appendable size: 10 MB - content claim max flow files: 100 - always sync: false - content repository archive max retention period: 12 hours - content repository archive max usage percentage: 50% - content repository archive enabled: false -Provenance Repository: - provenance rollover time: 1 min - implementation: org.apache.nifi.provenance.WriteAheadProvenanceRepository - provenance max storage time: 24 hours - provenance max storage size: 1 GB - provenance index shard size: 500 MB - provenance buffer size: 10000 -Component Status Repository: - buffer size: 1440 - snapshot frequency: 1 min -Security Properties: - keystore: '' - keystore type: '' - keystore password: '' - key password: '' - truststore: '' - truststore type: '' - truststore password: '' - ssl protocol: '' - Sensitive Props: - key: - algorithm: NIFI_PBKDF2_AES_GCM_256 -Processors: -- id: d80d4090-c56d-33ab-0000-000000000000 - name: GenerateFlowFile - class: org.apache.nifi.processors.standard.GenerateFlowFile - max concurrent tasks: 1 - scheduling strategy: TIMER_DRIVEN - scheduling period: 1 sec - penalization period: 30 sec - yield period: 1 sec - run duration nanos: 0 - auto-terminated relationships list: [] - Properties: - Batch Size: '1' - Data Format: Text - File Size: 1 kB - Unique FlowFiles: 'false' - character-set: UTF-8 - generate-ff-custom-text: -Controller Services: [] -Process Groups: [] -Input Ports: [] -Output Ports: [] -Funnels: [] -Connections: -- id: 89b6b729-aa58-32f5-0000-000000000000 - name: GenerateFlowFile/success/d7c01635-0160-1000-68aa-fd8a4f9d168e - source id: d80d4090-c56d-33ab-0000-000000000000 - source relationship names: - - success - destination id: d7c01635-0160-1000-68aa-fd8a4f9d168e - max work queue size: 10000 - max work queue data size: 1 GB - flowfile expiration: 0 sec - queue prioritizer class: '' -Remote Process Groups: -- id: 9abd6b8c-816a-3f66-0000-000000000000 - name: '' - url: http://localhost:8080/nifi - comment: '' - timeout: 30 sec - yield period: 10 sec - transport protocol: RAW - proxy host: '' - proxy port: '' - proxy user: '' - proxy password: '' - local network interface: '' - Input Ports: - - id: d7c01635-0160-1000-68aa-fd8a4f9d168e - name: From instance - comment: '' - max concurrent tasks: 1 - use compression: false - Output Ports: [] -NiFi Properties Overrides: {} diff --git a/minifi/minifi-toolkit/minifi-toolkit-configuration/src/test/resources/CsvToJson-v1.yml b/minifi/minifi-toolkit/minifi-toolkit-configuration/src/test/resources/CsvToJson-v1.yml deleted file mode 100644 index dd3527d00b..0000000000 --- a/minifi/minifi-toolkit/minifi-toolkit-configuration/src/test/resources/CsvToJson-v1.yml +++ /dev/null @@ -1,177 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the \"License\"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an \"AS IS\" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -Flow Controller: - name: CsvToJsonWorking - comment: '' -Core Properties: - flow controller graceful shutdown period: 10 sec - flow service write delay interval: 500 ms - administrative yield duration: 30 sec - bored yield duration: 10 millis - max concurrent threads: 1 -FlowFile Repository: - partitions: 256 - checkpoint interval: 2 mins - always sync: false - Swap: - threshold: 20000 - in period: 5 sec - in threads: 1 - out period: 5 sec - out threads: 4 -Content Repository: - content claim max appendable size: 10 MB - content claim max flow files: 100 - always sync: false -Provenance Repository: - provenance rollover time: 1 min -Component Status Repository: - buffer size: 1440 - snapshot frequency: 1 min -Security Properties: - keystore: '' - keystore type: '' - keystore password: '' - key password: '' - truststore: '' - truststore type: '' - truststore password: '' - ssl protocol: '' - Sensitive Props: - key: - algorithm: NIFI_PBKDF2_AES_GCM_256 -Processors: -- name: ExtractText - class: org.apache.nifi.processors.standard.ExtractText - max concurrent tasks: 1 - scheduling strategy: TIMER_DRIVEN - scheduling period: 0 sec - penalization period: 30 sec - yield period: 1 sec - run duration nanos: 0 - auto-terminated relationships list: - - unmatched - Properties: - Character Set: UTF-8 - Enable Canonical Equivalence: 'false' - Enable Case-insensitive Matching: 'false' - Enable DOTALL Mode: 'false' - Enable Literal Parsing of the Pattern: 'false' - Enable Multiline Mode: 'false' - Enable Unicode Predefined Character Classes: 'false' - Enable Unicode-aware Case Folding: 'false' - Enable Unix Lines Mode: 'false' - Include Capture Group 0: 'false' - Maximum Buffer Size: 1 MB - Maximum Capture Group Length: '1024' - Permit Whitespace and Comments in Pattern: 'false' - csv: (.+),(.+),(.+),(.+) -- name: GenerateFlowFile - class: org.apache.nifi.processors.standard.GenerateFlowFile - max concurrent tasks: 1 - scheduling strategy: TIMER_DRIVEN - scheduling period: 10 sec - penalization period: 30 sec - yield period: 1 sec - run duration nanos: 0 - auto-terminated relationships list: [] - Properties: - Batch Size: '1' - Data Format: Binary - File Size: 1 b - Unique FlowFiles: 'false' -- name: ReplaceText - class: org.apache.nifi.processors.standard.ReplaceText - max concurrent tasks: 1 - scheduling strategy: TIMER_DRIVEN - scheduling period: 0 sec - penalization period: 30 sec - yield period: 1 sec - run duration nanos: 0 - auto-terminated relationships list: - - failure - Properties: - Character Set: UTF-8 - Evaluation Mode: Entire text - Maximum Buffer Size: 1 MB - Regular Expression: (?s:^.*$) - Replacement Strategy: - Replacement Value: a,b,c,d -- name: ReplaceText2 - class: org.apache.nifi.processors.standard.ReplaceText - max concurrent tasks: 1 - scheduling strategy: TIMER_DRIVEN - scheduling period: 0 sec - penalization period: 30 sec - yield period: 1 sec - run duration nanos: 0 - auto-terminated relationships list: - - failure - Properties: - Character Set: UTF-8 - Evaluation Mode: Entire text - Maximum Buffer Size: 1 MB - Regular Expression: (?s:^.*$) - Replacement Strategy: - Replacement Value: "{ \"field1\" : \"${csv.1}\", \"field2\" : \"${csv.2}\",\n\ - \ \"field3\" : \"${csv.3}\", \"field4\" : \"${csv.4}\"\ - \ }\n " -- name: UpdateAttribute - class: org.apache.nifi.processors.attributes.UpdateAttribute - max concurrent tasks: 1 - scheduling strategy: TIMER_DRIVEN - scheduling period: 0 sec - penalization period: 30 sec - yield period: 1 sec - run duration nanos: 0 - auto-terminated relationships list: - - success - Properties: - Delete Attributes Expression: -Connections: -- name: ExtractText/matched/ReplaceText2 - source name: ExtractText - source relationship name: matched - destination name: ReplaceText2 - max work queue size: 0 - max work queue data size: 0 MB - flowfile expiration: 0 sec - queue prioritizer class: org.apache.nifi.prioritizer.FirstInFirstOutPrioritizer -- name: GenerateFlowFile/success/ReplaceText - source name: GenerateFlowFile - source relationship name: success - destination name: ReplaceText - max work queue size: 0 - max work queue data size: 0 MB - flowfile expiration: 0 sec - queue prioritizer class: org.apache.nifi.prioritizer.FirstInFirstOutPrioritizer -- name: ReplaceText/success/ExtractText - source name: ReplaceText - source relationship name: success - destination name: ExtractText - max work queue size: 0 - max work queue data size: 0 MB - flowfile expiration: 0 sec - queue prioritizer class: org.apache.nifi.prioritizer.FirstInFirstOutPrioritizer -- name: ReplaceText2/success/UpdateAttribute - source name: ReplaceText2 - source relationship name: success - destination name: UpdateAttribute - max work queue size: 0 - max work queue data size: 0 MB - flowfile expiration: 0 sec - queue prioritizer class: org.apache.nifi.prioritizer.FirstInFirstOutPrioritizer -Remote Processing Groups: [] diff --git a/minifi/minifi-toolkit/minifi-toolkit-configuration/src/test/resources/CsvToJson.xml b/minifi/minifi-toolkit/minifi-toolkit-configuration/src/test/resources/CsvToJson.xml deleted file mode 100644 index 4d11709bba..0000000000 --- a/minifi/minifi-toolkit/minifi-toolkit-configuration/src/test/resources/CsvToJson.xml +++ /dev/null @@ -1,20 +0,0 @@ - - - \ No newline at end of file diff --git a/minifi/minifi-toolkit/minifi-toolkit-configuration/src/test/resources/MultipleRelationships.yml b/minifi/minifi-toolkit/minifi-toolkit-configuration/src/test/resources/MultipleRelationships.yml deleted file mode 100644 index 187293637c..0000000000 --- a/minifi/minifi-toolkit/minifi-toolkit-configuration/src/test/resources/MultipleRelationships.yml +++ /dev/null @@ -1,165 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the \"License\"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an \"AS IS\" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -MiNiFi Config Version: 3 -Flow Controller: - name: MultipleRelationships - comment: '' -Core Properties: - flow controller graceful shutdown period: 10 sec - flow service write delay interval: 500 ms - administrative yield duration: 30 sec - bored yield duration: 10 millis - max concurrent threads: 1 - variable registry properties: '' -FlowFile Repository: - implementation: org.apache.nifi.controller.repository.WriteAheadFlowFileRepository - partitions: 256 - checkpoint interval: 2 mins - always sync: false - Swap: - threshold: 20000 - in period: 5 sec - in threads: 1 - out period: 5 sec - out threads: 4 -Content Repository: - implementation: org.apache.nifi.controller.repository.FileSystemRepository - content claim max appendable size: 10 MB - content claim max flow files: 100 - always sync: false - content repository archive max retention period: 12 hours - content repository archive max usage percentage: 50% - content repository archive enabled: false -Provenance Repository: - provenance rollover time: 1 min - implementation: org.apache.nifi.provenance.WriteAheadProvenanceRepository - provenance max storage time: 24 hours - provenance max storage size: 1 GB - provenance index shard size: 500 MB - provenance buffer size: 10000 -Component Status Repository: - buffer size: 1440 - snapshot frequency: 1 min -Security Properties: - keystore: '' - keystore type: '' - keystore password: '' - key password: '' - truststore: '' - truststore type: '' - truststore password: '' - ssl protocol: '' - Sensitive Props: - key: - algorithm: NIFI_PBKDF2_AES_GCM_256 -Processors: -- name: GenerateFlowFile - id: 7c755ed6-0157-1000-0000-000000000000 - class: org.apache.nifi.processors.standard.GenerateFlowFile - max concurrent tasks: 1 - scheduling strategy: TIMER_DRIVEN - scheduling period: 0 sec - penalization period: 30 sec - yield period: 1 sec - run duration nanos: 0 - auto-terminated relationships list: [] - Properties: - Batch Size: '1' - Data Format: Binary - File Size: 0 B - Unique FlowFiles: 'false' -- name: LogAttribute - id: 7c75ab71-0157-1000-0000-000000000000 - class: org.apache.nifi.processors.standard.LogAttribute - max concurrent tasks: 1 - scheduling strategy: TIMER_DRIVEN - scheduling period: 0 sec - penalization period: 30 sec - yield period: 1 sec - run duration nanos: 0 - auto-terminated relationships list: - - success - Properties: - Attributes to Ignore: - Attributes to Log: - Log Level: info - Log Payload: 'false' - Log prefix: -- name: RouteOnAttribute - id: 7c768622-0157-1000-0000-000000000000 - class: org.apache.nifi.processors.standard.RouteOnAttribute - max concurrent tasks: 1 - scheduling strategy: TIMER_DRIVEN - scheduling period: 0 sec - penalization period: 30 sec - yield period: 1 sec - run duration nanos: 0 - auto-terminated relationships list: [] - Properties: - Routing Strategy: Route to Property name - abc: ${filename:equals('abc')} -- name: UpdateAttribute - id: 7c79ba25-0157-1000-0000-000000000000 - class: org.apache.nifi.processors.attributes.UpdateAttribute - max concurrent tasks: 1 - scheduling strategy: TIMER_DRIVEN - scheduling period: 0 sec - penalization period: 30 sec - yield period: 1 sec - run duration nanos: 0 - auto-terminated relationships list: [] - Properties: - Delete Attributes Expression: - filename: abc -Controller Services: [] -Process Groups: [] -Input Ports: [] -Output Ports: [] -Funnels: [] -Connections: -- name: GenerateFlowFile/success/UpdateAttribute - id: 7c79cce3-0157-1000-0000-000000000000 - source id: 7c755ed6-0157-1000-0000-000000000000 - source relationship names: - - success - destination id: 7c79ba25-0157-1000-0000-000000000000 - max work queue size: 10000 - max work queue data size: 1 GB - flowfile expiration: 0 sec - queue prioritizer class: '' -- name: RouteOnAttribute/abc/LogAttribute - id: 7c798ca3-0157-1000-0000-000000000000 - source id: 7c768622-0157-1000-0000-000000000000 - source relationship names: - - abc - - unmatched - destination id: 7c75ab71-0157-1000-0000-000000000000 - max work queue size: 10000 - max work queue data size: 1 GB - flowfile expiration: 0 sec - queue prioritizer class: '' -- name: UpdateAttribute/success/RouteOnAttribute - id: 7c79d576-0157-1000-0000-000000000000 - source id: 7c79ba25-0157-1000-0000-000000000000 - source relationship names: - - success - destination id: 7c768622-0157-1000-0000-000000000000 - max work queue size: 10000 - max work queue data size: 1 GB - flowfile expiration: 0 sec - queue prioritizer class: '' -Remote Process Groups: [] -NiFi Properties Overrides: {} diff --git a/minifi/minifi-toolkit/minifi-toolkit-configuration/src/test/resources/MultipleUriRPG.xml b/minifi/minifi-toolkit/minifi-toolkit-configuration/src/test/resources/MultipleUriRPG.xml deleted file mode 100644 index 887812f984..0000000000 --- a/minifi/minifi-toolkit/minifi-toolkit-configuration/src/test/resources/MultipleUriRPG.xml +++ /dev/null @@ -1,215 +0,0 @@ - - - \ No newline at end of file diff --git a/minifi/minifi-toolkit/minifi-toolkit-configuration/src/test/resources/MultipleUriRPG.yml b/minifi/minifi-toolkit/minifi-toolkit-configuration/src/test/resources/MultipleUriRPG.yml deleted file mode 100644 index 833556d8fb..0000000000 --- a/minifi/minifi-toolkit/minifi-toolkit-configuration/src/test/resources/MultipleUriRPG.yml +++ /dev/null @@ -1,131 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the \"License\"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an \"AS IS\" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -MiNiFi Config Version: 3 -Flow Controller: - name: Simple TailFile To RPG - comment: '' -Core Properties: - flow controller graceful shutdown period: 10 sec - flow service write delay interval: 500 ms - administrative yield duration: 30 sec - bored yield duration: 10 millis - max concurrent threads: 1 - variable registry properties: '' -FlowFile Repository: - implementation: org.apache.nifi.controller.repository.WriteAheadFlowFileRepository - partitions: 256 - checkpoint interval: 2 mins - always sync: false - Swap: - threshold: 20000 - in period: 5 sec - in threads: 1 - out period: 5 sec - out threads: 4 -Content Repository: - implementation: org.apache.nifi.controller.repository.FileSystemRepository - content claim max appendable size: 10 MB - content claim max flow files: 100 - always sync: false - content repository archive max retention period: 12 hours - content repository archive max usage percentage: 50% - content repository archive enabled: false -Provenance Repository: - provenance rollover time: 1 min - implementation: org.apache.nifi.provenance.WriteAheadProvenanceRepository - provenance max storage time: 24 hours - provenance max storage size: 1 GB - provenance index shard size: 500 MB - provenance buffer size: 10000 -Component Status Repository: - buffer size: 1440 - snapshot frequency: 1 min -Security Properties: - keystore: '' - keystore type: '' - keystore password: '' - key password: '' - truststore: '' - truststore type: '' - truststore password: '' - ssl protocol: '' - Sensitive Props: - key: - algorithm: NIFI_PBKDF2_AES_GCM_256 -Processors: -- id: b0c04f28-0158-1000-0000-000000000000 - name: TailFile - class: org.apache.nifi.processors.standard.TailFile - max concurrent tasks: 1 - scheduling strategy: TIMER_DRIVEN - scheduling period: 1 sec - penalization period: 30 sec - yield period: 1 sec - run duration nanos: 0 - auto-terminated relationships list: [] - Properties: - File Location: Local - File to Tail: ./logs/minifi-app.log - Initial Start Position: Beginning of File - Rolling Filename Pattern: - tail-base-directory: - tail-mode: Single file - tailfile-lookup-frequency: 10 minutes - tailfile-maximum-age: 24 hours - tailfile-recursive-lookup: 'false' - tailfile-rolling-strategy: Fixed name -Controller Services: [] -Process Groups: [] -Input Ports: [] -Output Ports: [] -Funnels: [] -Connections: -- id: b0c0c3cc-0158-1000-0000-000000000000 - name: TailFile/success/ac0e798c-0158-1000-0588-cda9b944e011 - source id: b0c04f28-0158-1000-0000-000000000000 - source relationship names: - - success - destination id: ac0e798c-0158-1000-0588-cda9b944e011 - max work queue size: 10000 - max work queue data size: 1 GB - flowfile expiration: 0 sec - queue prioritizer class: '' -Remote Process Groups: -- id: b0c09ff0-0158-1000-0000-000000000000 - name: '' - url: http://localhost:8080/nifi,http://localhost:8081/nifi,http://localhost:8082/nifi - comment: '' - timeout: 30 sec - yield period: 10 sec - transport protocol: RAW - proxy host: '' - proxy port: '' - proxy user: '' - proxy password: '' - local network interface: '' - Input Ports: - - id: aca664f8-0158-1000-a139-92485891d349 - name: test2 - comment: '' - max concurrent tasks: 1 - use compression: false - - id: ac0e798c-0158-1000-0588-cda9b944e011 - name: test - comment: '' - max concurrent tasks: 1 - use compression: false - Output Ports: [] -NiFi Properties Overrides: {} diff --git a/minifi/minifi-toolkit/minifi-toolkit-configuration/src/test/resources/NestedControllerServices.xml b/minifi/minifi-toolkit/minifi-toolkit-configuration/src/test/resources/NestedControllerServices.xml deleted file mode 100644 index 13fc40e718..0000000000 --- a/minifi/minifi-toolkit/minifi-toolkit-configuration/src/test/resources/NestedControllerServices.xml +++ /dev/null @@ -1,1089 +0,0 @@ - - - \ No newline at end of file diff --git a/minifi/minifi-toolkit/minifi-toolkit-configuration/src/test/resources/NestedControllerServices.yml b/minifi/minifi-toolkit/minifi-toolkit-configuration/src/test/resources/NestedControllerServices.yml deleted file mode 100644 index 91e84671d1..0000000000 --- a/minifi/minifi-toolkit/minifi-toolkit-configuration/src/test/resources/NestedControllerServices.yml +++ /dev/null @@ -1,301 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the \"License\"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an \"AS IS\" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -MiNiFi Config Version: 3 -Flow Controller: - name: NestedControllerServices - comment: '' -Core Properties: - flow controller graceful shutdown period: 10 sec - flow service write delay interval: 500 ms - administrative yield duration: 30 sec - bored yield duration: 10 millis - max concurrent threads: 1 - variable registry properties: '' -FlowFile Repository: - implementation: org.apache.nifi.controller.repository.WriteAheadFlowFileRepository - partitions: 256 - checkpoint interval: 2 mins - always sync: false - Swap: - threshold: 20000 - in period: 5 sec - in threads: 1 - out period: 5 sec - out threads: 4 -Content Repository: - implementation: org.apache.nifi.controller.repository.FileSystemRepository - content claim max appendable size: 10 MB - content claim max flow files: 100 - always sync: false - content repository archive max retention period: 12 hours - content repository archive max usage percentage: 50% - content repository archive enabled: false -Provenance Repository: - provenance rollover time: 1 min - implementation: org.apache.nifi.provenance.WriteAheadProvenanceRepository - provenance max storage time: 24 hours - provenance max storage size: 1 GB - provenance index shard size: 500 MB - provenance buffer size: 10000 -Component Status Repository: - buffer size: 1440 - snapshot frequency: 1 min -Security Properties: - keystore: '' - keystore type: '' - keystore password: '' - key password: '' - truststore: '' - truststore type: '' - truststore password: '' - ssl protocol: '' - Sensitive Props: - key: - algorithm: NIFI_PBKDF2_AES_GCM_256 -Processors: -- id: d5232ad7-0158-1000-0000-000000000000 - name: GetHTTP - class: org.apache.nifi.processors.standard.GetHTTP - max concurrent tasks: 1 - scheduling strategy: TIMER_DRIVEN - scheduling period: 0 sec - penalization period: 30 sec - yield period: 1 sec - run duration nanos: 0 - auto-terminated relationships list: [] - Properties: - Accept Content-Type: - Connection Timeout: 30 sec - Data Timeout: 30 sec - Filename: nifiCorrectGet - Follow Redirects: 'false' - Password: - Proxy Host: - Proxy Port: - SSL Context Service: d4c47b18-0158-1000-0000-000000000000 - URL: https://localhost:8080/nifi - User Agent: - Username: - redirect-cookie-policy: default -- id: d5233667-0158-1000-0000-000000000000 - name: LogAttribute - class: org.apache.nifi.processors.standard.LogAttribute - max concurrent tasks: 1 - scheduling strategy: TIMER_DRIVEN - scheduling period: 0 sec - penalization period: 30 sec - yield period: 1 sec - run duration nanos: 0 - auto-terminated relationships list: - - success - Properties: - Attributes to Ignore: - Attributes to Log: - Log Level: info - Log Payload: 'false' - Log prefix: -Controller Services: -- id: d4c47b18-0158-1000-0000-000000000000 - name: test-root - type: org.apache.nifi.ssl.StandardSSLContextService - Properties: - Keystore Filename: - Keystore Password: - Keystore Type: - SSL Protocol: - Truststore Filename: - Truststore Password: - Truststore Type: - key-password: -Process Groups: -- id: d4c4b4e4-0158-1000-0000-000000000000 - name: middle - Processors: - - id: d520168b-0158-1000-0000-000000000000 - name: HandleHttpRequest - class: org.apache.nifi.processors.standard.HandleHttpRequest - max concurrent tasks: 1 - scheduling strategy: TIMER_DRIVEN - scheduling period: 0 sec - penalization period: 30 sec - yield period: 1 sec - run duration nanos: 0 - auto-terminated relationships list: [] - Properties: - Additional HTTP Methods: - Allow DELETE: 'true' - Allow GET: 'true' - Allow HEAD: 'false' - Allow OPTIONS: 'false' - Allow POST: 'true' - Allow PUT: 'true' - Allowed Paths: - Client Authentication: No Authentication - Default URL Character Set: UTF-8 - HTTP Context Map: d4c4c947-0158-1000-0000-000000000000 - Hostname: - Listening Port: '80' - SSL Context Service: - container-queue-size: '50' - - id: d522d82b-0158-1000-0000-000000000000 - name: HandleHttpResponse - class: org.apache.nifi.processors.standard.HandleHttpResponse - max concurrent tasks: 1 - scheduling strategy: TIMER_DRIVEN - scheduling period: 0 sec - penalization period: 30 sec - yield period: 1 sec - run duration nanos: 0 - auto-terminated relationships list: [] - Properties: - HTTP Context Map: d4c4c947-0158-1000-0000-000000000000 - HTTP Status Code: '200' - Controller Services: - - id: d4c4c947-0158-1000-0000-000000000000 - name: test-middle - type: org.apache.nifi.http.StandardHttpContextMap - Properties: - Maximum Outstanding Requests: - Request Expiration: - Process Groups: - - id: d4c52c16-0158-1000-0000-000000000000 - name: inner - Processors: - - id: d5205c43-0158-1000-0000-000000000000 - name: DetectDuplicate - class: org.apache.nifi.processors.standard.DetectDuplicate - max concurrent tasks: 1 - scheduling strategy: TIMER_DRIVEN - scheduling period: 0 sec - penalization period: 30 sec - yield period: 1 sec - run duration nanos: 0 - auto-terminated relationships list: [] - Properties: - Age Off Duration: - Cache Entry Identifier: ${hash.value} - Cache The Entry Identifier: 'true' - Distributed Cache Service: d4c5554d-0158-1000-0000-000000000000 - FlowFile Description: test - - id: d5211ba3-0158-1000-0000-000000000000 - name: GetHTTP - class: org.apache.nifi.processors.standard.GetHTTP - max concurrent tasks: 1 - scheduling strategy: TIMER_DRIVEN - scheduling period: 0 sec - penalization period: 30 sec - yield period: 1 sec - run duration nanos: 0 - auto-terminated relationships list: [] - Properties: - Accept Content-Type: - Connection Timeout: 30 sec - Data Timeout: 30 sec - Filename: getLocalhost - Follow Redirects: 'false' - Password: - Proxy Host: - Proxy Port: - SSL Context Service: - URL: https://localhost:8080/ - User Agent: - Username: - redirect-cookie-policy: default - - id: d521ab50-0158-1000-0000-000000000000 - name: LogAttribute - class: org.apache.nifi.processors.standard.LogAttribute - max concurrent tasks: 1 - scheduling strategy: TIMER_DRIVEN - scheduling period: 0 sec - penalization period: 30 sec - yield period: 1 sec - run duration nanos: 0 - auto-terminated relationships list: - - success - Properties: - Attributes to Ignore: - Attributes to Log: - Log Level: info - Log Payload: 'false' - Log prefix: - Controller Services: - - id: d4c5554d-0158-1000-0000-000000000000 - name: test-inner - type: org.apache.nifi.distributed.cache.client.DistributedMapCacheClientService - Properties: - Communications Timeout: - SSL Context Service: d4c47b18-0158-1000-0000-000000000000 - Server Hostname: localhost - Server Port: - Process Groups: [] - Input Ports: [] - Output Ports: [] - Funnels: [] - Connections: - - id: d521c3e8-0158-1000-0000-000000000000 - name: DetectDuplicate/failure/LogAttribute - source id: d5205c43-0158-1000-0000-000000000000 - source relationship names: - - duplicate - - failure - - non-duplicate - destination id: d521ab50-0158-1000-0000-000000000000 - max work queue size: 10000 - max work queue data size: 1 GB - flowfile expiration: 0 sec - queue prioritizer class: '' - - id: d52147e4-0158-1000-0000-000000000000 - name: GetHTTP/success/DetectDuplicate - source id: d5211ba3-0158-1000-0000-000000000000 - source relationship names: - - success - destination id: d5205c43-0158-1000-0000-000000000000 - max work queue size: 10000 - max work queue data size: 1 GB - flowfile expiration: 0 sec - queue prioritizer class: '' - Remote Process Groups: [] - Input Ports: [] - Output Ports: [] - Funnels: [] - Connections: - - id: d522e203-0158-1000-0000-000000000000 - name: HandleHttpRequest/success/HandleHttpResponse - source id: d520168b-0158-1000-0000-000000000000 - source relationship names: - - success - destination id: d522d82b-0158-1000-0000-000000000000 - max work queue size: 10000 - max work queue data size: 1 GB - flowfile expiration: 0 sec - queue prioritizer class: '' - Remote Process Groups: [] -Input Ports: [] -Output Ports: [] -Funnels: [] -Connections: -- id: d5234d12-0158-1000-0000-000000000000 - name: GetHTTP/success/LogAttribute - source id: d5232ad7-0158-1000-0000-000000000000 - source relationship names: - - success - destination id: d5233667-0158-1000-0000-000000000000 - max work queue size: 10000 - max work queue data size: 1 GB - flowfile expiration: 0 sec - queue prioritizer class: '' -Remote Process Groups: [] -NiFi Properties Overrides: {} diff --git a/minifi/minifi-toolkit/minifi-toolkit-configuration/src/test/resources/NoTemplateEncodingVersion.xml b/minifi/minifi-toolkit/minifi-toolkit-configuration/src/test/resources/NoTemplateEncodingVersion.xml deleted file mode 100644 index e4ea628343..0000000000 --- a/minifi/minifi-toolkit/minifi-toolkit-configuration/src/test/resources/NoTemplateEncodingVersion.xml +++ /dev/null @@ -1,214 +0,0 @@ - - - \ No newline at end of file diff --git a/minifi/minifi-toolkit/minifi-toolkit-configuration/src/test/resources/NoTemplateEncodingVersion.yml b/minifi/minifi-toolkit/minifi-toolkit-configuration/src/test/resources/NoTemplateEncodingVersion.yml deleted file mode 100644 index 199af94bd8..0000000000 --- a/minifi/minifi-toolkit/minifi-toolkit-configuration/src/test/resources/NoTemplateEncodingVersion.yml +++ /dev/null @@ -1,131 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the \"License\"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an \"AS IS\" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -MiNiFi Config Version: 3 -Flow Controller: - name: Simple TailFile To RPG - comment: '' -Core Properties: - flow controller graceful shutdown period: 10 sec - flow service write delay interval: 500 ms - administrative yield duration: 30 sec - bored yield duration: 10 millis - max concurrent threads: 1 - variable registry properties: '' -FlowFile Repository: - implementation: org.apache.nifi.controller.repository.WriteAheadFlowFileRepository - partitions: 256 - checkpoint interval: 2 mins - always sync: false - Swap: - threshold: 20000 - in period: 5 sec - in threads: 1 - out period: 5 sec - out threads: 4 -Content Repository: - implementation: org.apache.nifi.controller.repository.FileSystemRepository - content claim max appendable size: 10 MB - content claim max flow files: 100 - always sync: false - content repository archive max retention period: 12 hours - content repository archive max usage percentage: 50% - content repository archive enabled: false -Provenance Repository: - provenance rollover time: 1 min - implementation: org.apache.nifi.provenance.WriteAheadProvenanceRepository - provenance max storage time: 24 hours - provenance max storage size: 1 GB - provenance index shard size: 500 MB - provenance buffer size: 10000 -Component Status Repository: - buffer size: 1440 - snapshot frequency: 1 min -Security Properties: - keystore: '' - keystore type: '' - keystore password: '' - key password: '' - truststore: '' - truststore type: '' - truststore password: '' - ssl protocol: '' - Sensitive Props: - key: - algorithm: NIFI_PBKDF2_AES_GCM_256 -Processors: -- id: b0c04f28-0158-1000-0000-000000000000 - name: TailFile - class: org.apache.nifi.processors.standard.TailFile - max concurrent tasks: 1 - scheduling strategy: TIMER_DRIVEN - scheduling period: 1 sec - penalization period: 30 sec - yield period: 1 sec - run duration nanos: 0 - auto-terminated relationships list: [] - Properties: - File Location: Local - File to Tail: ./logs/minifi-app.log - Initial Start Position: Beginning of File - Rolling Filename Pattern: - tail-base-directory: - tail-mode: Single file - tailfile-lookup-frequency: 10 minutes - tailfile-maximum-age: 24 hours - tailfile-recursive-lookup: 'false' - tailfile-rolling-strategy: Fixed name -Controller Services: [] -Process Groups: [] -Input Ports: [] -Output Ports: [] -Funnels: [] -Connections: -- id: b0c0c3cc-0158-1000-0000-000000000000 - name: TailFile/success/ac0e798c-0158-1000-0588-cda9b944e011 - source id: b0c04f28-0158-1000-0000-000000000000 - source relationship names: - - success - destination id: ac0e798c-0158-1000-0588-cda9b944e011 - max work queue size: 10000 - max work queue data size: 1 GB - flowfile expiration: 0 sec - queue prioritizer class: '' -Remote Process Groups: -- id: b0c09ff0-0158-1000-0000-000000000000 - name: '' - url: http://localhost:8080/nifi - comment: '' - timeout: 30 sec - yield period: 10 sec - transport protocol: RAW - proxy host: '' - proxy port: '' - proxy user: '' - proxy password: '' - local network interface: '' - Input Ports: - - id: aca664f8-0158-1000-a139-92485891d349 - name: test2 - comment: '' - max concurrent tasks: 1 - use compression: false - - id: ac0e798c-0158-1000-0588-cda9b944e011 - name: test - comment: '' - max concurrent tasks: 1 - use compression: false - Output Ports: [] -NiFi Properties Overrides: {} diff --git a/minifi/minifi-toolkit/minifi-toolkit-configuration/src/test/resources/ProcessGroupsAndRemoteProcessGroups-v2.yml b/minifi/minifi-toolkit/minifi-toolkit-configuration/src/test/resources/ProcessGroupsAndRemoteProcessGroups-v2.yml deleted file mode 100644 index d288bc5dbe..0000000000 --- a/minifi/minifi-toolkit/minifi-toolkit-configuration/src/test/resources/ProcessGroupsAndRemoteProcessGroups-v2.yml +++ /dev/null @@ -1,287 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the \"License\"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an \"AS IS\" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -MiNiFi Config Version: 2 -Flow Controller: - name: ProcessGroupsAndRemoteProcessGroups - comment: '' -Core Properties: - flow controller graceful shutdown period: 10 sec - flow service write delay interval: 500 ms - administrative yield duration: 30 sec - bored yield duration: 10 millis - max concurrent threads: 1 -FlowFile Repository: - implementation: org.apache.nifi.controller.repository.WriteAheadFlowFileRepository - partitions: 256 - checkpoint interval: 2 mins - always sync: false - Swap: - threshold: 20000 - in period: 5 sec - in threads: 1 - out period: 5 sec - out threads: 4 -Content Repository: - content claim max appendable size: 10 MB - content claim max flow files: 100 - always sync: false -Provenance Repository: - provenance rollover time: 1 min -Component Status Repository: - buffer size: 1440 - snapshot frequency: 1 min -Security Properties: - keystore: '' - keystore type: '' - keystore password: '' - key password: '' - truststore: '' - truststore type: '' - truststore password: '' - ssl protocol: '' - Sensitive Props: - key: - algorithm: NIFI_PBKDF2_AES_GCM_256 -Processors: -- id: 207748d1-0158-1000-0000-000000000000 - name: GenerateFlowFile - class: org.apache.nifi.processors.standard.GenerateFlowFile - max concurrent tasks: 1 - scheduling strategy: TIMER_DRIVEN - scheduling period: 0 sec - penalization period: 30 sec - yield period: 1 sec - run duration nanos: 0 - auto-terminated relationships list: [] - Properties: - Batch Size: '1' - Data Format: Binary - File Size: 1 b - Unique FlowFiles: 'false' -- id: 2079e8bd-0158-1000-0000-000000000000 - name: LogAttribute - class: org.apache.nifi.processors.standard.LogAttribute - max concurrent tasks: 1 - scheduling strategy: TIMER_DRIVEN - scheduling period: 0 sec - penalization period: 30 sec - yield period: 1 sec - run duration nanos: 0 - auto-terminated relationships list: - - success - Properties: - Attributes to Ignore: - Attributes to Log: - Log Level: info - Log Payload: 'false' - Log prefix: -- id: 2077ab1e-0158-1000-0000-000000000000 - name: UpdateAttribute - class: org.apache.nifi.processors.attributes.UpdateAttribute - max concurrent tasks: 1 - scheduling strategy: TIMER_DRIVEN - scheduling period: 0 sec - penalization period: 30 sec - yield period: 1 sec - run duration nanos: 0 - auto-terminated relationships list: [] - Properties: - Delete Attributes Expression: - top: top -Process Groups: -- id: 207888b1-0158-1000-0000-000000000000 - name: middle - Processors: - - id: 2078f34e-0158-1000-0000-000000000000 - name: UpdateAttribute - class: org.apache.nifi.processors.attributes.UpdateAttribute - max concurrent tasks: 1 - scheduling strategy: TIMER_DRIVEN - scheduling period: 0 sec - penalization period: 30 sec - yield period: 1 sec - run duration nanos: 0 - auto-terminated relationships list: [] - Properties: - Delete Attributes Expression: - middle: middle - Process Groups: - - id: 20794cd4-0158-1000-0000-000000000000 - name: bottom - Processors: - - id: 207a89ba-0158-1000-0000-000000000000 - name: UpdateAttribute - class: org.apache.nifi.processors.attributes.UpdateAttribute - max concurrent tasks: 1 - scheduling strategy: TIMER_DRIVEN - scheduling period: 0 sec - penalization period: 30 sec - yield period: 1 sec - run duration nanos: 0 - auto-terminated relationships list: [] - Properties: - Delete Attributes Expression: - bottom: bottom - Process Groups: [] - Input Ports: - - id: 207a5f50-0158-1000-0000-000000000000 - name: input - Output Ports: - - id: 207a6d92-0158-1000-0000-000000000000 - name: output - Funnels: [] - Connections: - - id: 21a6abb9-0158-1000-0000-000000000000 - name: UpdateAttribute/success/21a39aba-0158-1000-a1a0-1b55bcddcd72 - source id: 207a89ba-0158-1000-0000-000000000000 - source relationship names: - - success - destination id: 21a39aba-0158-1000-a1a0-1b55bcddcd72 - max work queue size: 10000 - max work queue data size: 1 GB - flowfile expiration: 0 sec - queue prioritizer class: '' - - id: 207ad5e9-0158-1000-0000-000000000000 - name: UpdateAttribute/success/output - source id: 207a89ba-0158-1000-0000-000000000000 - source relationship names: - - success - destination id: 207a6d92-0158-1000-0000-000000000000 - max work queue size: 10000 - max work queue data size: 1 GB - flowfile expiration: 0 sec - queue prioritizer class: '' - - id: 207aca0d-0158-1000-0000-000000000000 - name: input//UpdateAttribute - source id: 207a5f50-0158-1000-0000-000000000000 - source relationship names: [] - destination id: 207a89ba-0158-1000-0000-000000000000 - max work queue size: 10000 - max work queue data size: 1 GB - flowfile expiration: 0 sec - queue prioritizer class: '' - Remote Process Groups: - - id: 21a651bb-0158-1000-0000-000000000000 - name: '' - url: http://localhost:9091/nifi - comment: '' - timeout: 30 sec - yield period: 10 sec - transport protocol: RAW - proxy host: '' - proxy port: '' - proxy user: '' - proxy password: '' - Input Ports: - - id: 21a39aba-0158-1000-a1a0-1b55bcddcd72 - name: input2 - comment: '' - max concurrent tasks: 1 - use compression: false - Input Ports: - - id: 2078c936-0158-1000-0000-000000000000 - name: input - Output Ports: - - id: 2079b327-0158-1000-0000-000000000000 - name: output - Funnels: [] - Connections: - - id: 21a5b1f1-0158-1000-0000-000000000000 - name: UpdateAttribute/success/21a2fb5e-0158-1000-3b5e-5a7d3aaee01b - source id: 2078f34e-0158-1000-0000-000000000000 - source relationship names: - - success - destination id: 21a2fb5e-0158-1000-3b5e-5a7d3aaee01b - max work queue size: 10000 - max work queue data size: 1 GB - flowfile expiration: 0 sec - queue prioritizer class: '' - - id: 207b0eb1-0158-1000-0000-000000000000 - name: UpdateAttribute/success/input - source id: 2078f34e-0158-1000-0000-000000000000 - source relationship names: - - success - destination id: 207a5f50-0158-1000-0000-000000000000 - max work queue size: 10000 - max work queue data size: 1 GB - flowfile expiration: 0 sec - queue prioritizer class: '' - - id: 20792ec2-0158-1000-0000-000000000000 - name: input//UpdateAttribute - source id: 2078c936-0158-1000-0000-000000000000 - source relationship names: [] - destination id: 2078f34e-0158-1000-0000-000000000000 - max work queue size: 10000 - max work queue data size: 1 GB - flowfile expiration: 0 sec - queue prioritizer class: '' - - id: 207b1880-0158-1000-0000-000000000000 - name: output//output - source id: 207a6d92-0158-1000-0000-000000000000 - source relationship names: [] - destination id: 2079b327-0158-1000-0000-000000000000 - max work queue size: 10000 - max work queue data size: 1 GB - flowfile expiration: 0 sec - queue prioritizer class: '' - Remote Process Groups: - - id: 21a4e2f0-0158-1000-0000-000000000000 - name: '' - url: http://localhost:9090/nifi - comment: '' - timeout: 30 sec - yield period: 10 sec - transport protocol: HTTP - Input Ports: - - id: 21a2fb5e-0158-1000-3b5e-5a7d3aaee01b - name: input - comment: '' - max concurrent tasks: 1 - use compression: false -Input Ports: [] -Output Ports: [] -Funnels: [] -Connections: -- id: 2077bf8f-0158-1000-0000-000000000000 - name: GenerateFlowFile/success/UpdateAttribute - source id: 207748d1-0158-1000-0000-000000000000 - source relationship names: - - success - destination id: 2077ab1e-0158-1000-0000-000000000000 - max work queue size: 10000 - max work queue data size: 1 GB - flowfile expiration: 0 sec - queue prioritizer class: '' -- id: 2079cf6f-0158-1000-0000-000000000000 - name: UpdateAttribute/success/input - source id: 2077ab1e-0158-1000-0000-000000000000 - source relationship names: - - success - destination id: 2078c936-0158-1000-0000-000000000000 - max work queue size: 10000 - max work queue data size: 1 GB - flowfile expiration: 0 sec - queue prioritizer class: '' -- id: 2079faa0-0158-1000-0000-000000000000 - name: output//LogAttribute - source id: 2079b327-0158-1000-0000-000000000000 - source relationship names: [] - destination id: 2079e8bd-0158-1000-0000-000000000000 - max work queue size: 10000 - max work queue data size: 1 GB - flowfile expiration: 0 sec - queue prioritizer class: '' -Remote Process Groups: [] diff --git a/minifi/minifi-toolkit/minifi-toolkit-configuration/src/test/resources/ProcessGroupsAndRemoteProcessGroups.xml b/minifi/minifi-toolkit/minifi-toolkit-configuration/src/test/resources/ProcessGroupsAndRemoteProcessGroups.xml deleted file mode 100644 index 3c797779bd..0000000000 --- a/minifi/minifi-toolkit/minifi-toolkit-configuration/src/test/resources/ProcessGroupsAndRemoteProcessGroups.xml +++ /dev/null @@ -1,650 +0,0 @@ - - - \ No newline at end of file diff --git a/minifi/minifi-toolkit/minifi-toolkit-configuration/src/test/resources/ProcessGroupsAndRemoteProcessGroups.yml b/minifi/minifi-toolkit/minifi-toolkit-configuration/src/test/resources/ProcessGroupsAndRemoteProcessGroups.yml deleted file mode 100644 index 25ae476fea..0000000000 --- a/minifi/minifi-toolkit/minifi-toolkit-configuration/src/test/resources/ProcessGroupsAndRemoteProcessGroups.yml +++ /dev/null @@ -1,309 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the \"License\"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an \"AS IS\" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -MiNiFi Config Version: 3 -Flow Controller: - name: ProcessGroupsAndRemoteProcessGroups - comment: '' -Core Properties: - flow controller graceful shutdown period: 10 sec - flow service write delay interval: 500 ms - administrative yield duration: 30 sec - bored yield duration: 10 millis - max concurrent threads: 1 - variable registry properties: '' -FlowFile Repository: - implementation: org.apache.nifi.controller.repository.WriteAheadFlowFileRepository - partitions: 256 - checkpoint interval: 2 mins - always sync: false - Swap: - threshold: 20000 - in period: 5 sec - in threads: 1 - out period: 5 sec - out threads: 4 -Content Repository: - implementation: org.apache.nifi.controller.repository.FileSystemRepository - content claim max appendable size: 10 MB - content claim max flow files: 100 - always sync: false - content repository archive max retention period: 12 hours - content repository archive max usage percentage: 50% - content repository archive enabled: false -Provenance Repository: - provenance rollover time: 1 min - implementation: org.apache.nifi.provenance.WriteAheadProvenanceRepository - provenance max storage time: 24 hours - provenance max storage size: 1 GB - provenance index shard size: 500 MB - provenance buffer size: 10000 -Component Status Repository: - buffer size: 1440 - snapshot frequency: 1 min -Security Properties: - keystore: '' - keystore type: '' - keystore password: '' - key password: '' - truststore: '' - truststore type: '' - truststore password: '' - ssl protocol: '' - Sensitive Props: - key: - algorithm: NIFI_PBKDF2_AES_GCM_256 -Processors: -- id: 207748d1-0158-1000-0000-000000000000 - name: GenerateFlowFile - class: org.apache.nifi.processors.standard.GenerateFlowFile - max concurrent tasks: 1 - scheduling strategy: TIMER_DRIVEN - scheduling period: 0 sec - penalization period: 30 sec - yield period: 1 sec - run duration nanos: 0 - auto-terminated relationships list: [] - Properties: - Batch Size: '1' - Data Format: Binary - File Size: 1 b - Unique FlowFiles: 'false' -- id: 2079e8bd-0158-1000-0000-000000000000 - name: LogAttribute - class: org.apache.nifi.processors.standard.LogAttribute - max concurrent tasks: 1 - scheduling strategy: TIMER_DRIVEN - scheduling period: 0 sec - penalization period: 30 sec - yield period: 1 sec - run duration nanos: 0 - auto-terminated relationships list: - - success - Properties: - Attributes to Ignore: - Attributes to Log: - Log Level: info - Log Payload: 'false' - Log prefix: -- id: 2077ab1e-0158-1000-0000-000000000000 - name: UpdateAttribute - class: org.apache.nifi.processors.attributes.UpdateAttribute - max concurrent tasks: 1 - scheduling strategy: TIMER_DRIVEN - scheduling period: 0 sec - penalization period: 30 sec - yield period: 1 sec - run duration nanos: 0 - auto-terminated relationships list: [] - Properties: - Delete Attributes Expression: - top: top -Controller Services: [] -Process Groups: -- id: 207888b1-0158-1000-0000-000000000000 - name: middle - Processors: - - id: 2078f34e-0158-1000-0000-000000000000 - name: UpdateAttribute - class: org.apache.nifi.processors.attributes.UpdateAttribute - max concurrent tasks: 1 - scheduling strategy: TIMER_DRIVEN - scheduling period: 0 sec - penalization period: 30 sec - yield period: 1 sec - run duration nanos: 0 - auto-terminated relationships list: [] - Properties: - Delete Attributes Expression: - middle: middle - Controller Services: [] - Process Groups: - - id: 20794cd4-0158-1000-0000-000000000000 - name: bottom - Processors: - - id: 207a89ba-0158-1000-0000-000000000000 - name: UpdateAttribute - class: org.apache.nifi.processors.attributes.UpdateAttribute - max concurrent tasks: 1 - scheduling strategy: TIMER_DRIVEN - scheduling period: 0 sec - penalization period: 30 sec - yield period: 1 sec - run duration nanos: 0 - auto-terminated relationships list: [] - Properties: - Delete Attributes Expression: - bottom: bottom - Controller Services: [] - Process Groups: [] - Input Ports: - - id: 207a5f50-0158-1000-0000-000000000000 - name: input - Output Ports: - - id: 207a6d92-0158-1000-0000-000000000000 - name: output - Funnels: [] - Connections: - - id: 21a6abb9-0158-1000-0000-000000000000 - name: UpdateAttribute/success/21a39aba-0158-1000-a1a0-1b55bcddcd72 - source id: 207a89ba-0158-1000-0000-000000000000 - source relationship names: - - success - destination id: 21a39aba-0158-1000-a1a0-1b55bcddcd72 - max work queue size: 10000 - max work queue data size: 1 GB - flowfile expiration: 0 sec - queue prioritizer class: '' - - id: 207ad5e9-0158-1000-0000-000000000000 - name: UpdateAttribute/success/output - source id: 207a89ba-0158-1000-0000-000000000000 - source relationship names: - - success - destination id: 207a6d92-0158-1000-0000-000000000000 - max work queue size: 10000 - max work queue data size: 1 GB - flowfile expiration: 0 sec - queue prioritizer class: '' - - id: 207aca0d-0158-1000-0000-000000000000 - name: input//UpdateAttribute - source id: 207a5f50-0158-1000-0000-000000000000 - source relationship names: [] - destination id: 207a89ba-0158-1000-0000-000000000000 - max work queue size: 10000 - max work queue data size: 1 GB - flowfile expiration: 0 sec - queue prioritizer class: '' - Remote Process Groups: - - id: 21a651bb-0158-1000-0000-000000000000 - name: '' - url: http://localhost:9091/nifi - comment: '' - timeout: 30 sec - yield period: 10 sec - transport protocol: RAW - proxy host: '' - proxy port: '' - proxy user: '' - proxy password: '' - local network interface: 'en0' - Input Ports: - - id: 21a39aba-0158-1000-a1a0-1b55bcddcd72 - name: input2 - comment: '' - max concurrent tasks: 1 - use compression: false - Output Ports: [] - Input Ports: - - id: 2078c936-0158-1000-0000-000000000000 - name: input - Output Ports: - - id: 2079b327-0158-1000-0000-000000000000 - name: output - Funnels: [] - Connections: - - id: 21a5b1f1-0158-1000-0000-000000000000 - name: UpdateAttribute/success/21a2fb5e-0158-1000-3b5e-5a7d3aaee01b - source id: 2078f34e-0158-1000-0000-000000000000 - source relationship names: - - success - destination id: 21a2fb5e-0158-1000-3b5e-5a7d3aaee01b - max work queue size: 10000 - max work queue data size: 1 GB - flowfile expiration: 0 sec - queue prioritizer class: '' - - id: 207b0eb1-0158-1000-0000-000000000000 - name: UpdateAttribute/success/input - source id: 2078f34e-0158-1000-0000-000000000000 - source relationship names: - - success - destination id: 207a5f50-0158-1000-0000-000000000000 - max work queue size: 10000 - max work queue data size: 1 GB - flowfile expiration: 0 sec - queue prioritizer class: '' - - id: 20792ec2-0158-1000-0000-000000000000 - name: input//UpdateAttribute - source id: 2078c936-0158-1000-0000-000000000000 - source relationship names: [] - destination id: 2078f34e-0158-1000-0000-000000000000 - max work queue size: 10000 - max work queue data size: 1 GB - flowfile expiration: 0 sec - queue prioritizer class: '' - - id: 207b1880-0158-1000-0000-000000000000 - name: output//output - source id: 207a6d92-0158-1000-0000-000000000000 - source relationship names: [] - destination id: 2079b327-0158-1000-0000-000000000000 - max work queue size: 10000 - max work queue data size: 1 GB - flowfile expiration: 0 sec - queue prioritizer class: '' - Remote Process Groups: - - id: 21a4e2f0-0158-1000-0000-000000000000 - name: '' - url: http://localhost:9090/nifi - comment: '' - timeout: 30 sec - yield period: 10 sec - transport protocol: HTTP - proxy host: '' - proxy port: '' - proxy user: '' - proxy password: '' - local network interface: 'en1' - Input Ports: - - id: 21a2fb5e-0158-1000-3b5e-5a7d3aaee01b - name: input - comment: '' - max concurrent tasks: 1 - use compression: false - Output Ports: [] -Input Ports: [] -Output Ports: [] -Funnels: [] -Connections: -- id: 2077bf8f-0158-1000-0000-000000000000 - name: GenerateFlowFile/success/UpdateAttribute - source id: 207748d1-0158-1000-0000-000000000000 - source relationship names: - - success - destination id: 2077ab1e-0158-1000-0000-000000000000 - max work queue size: 10000 - max work queue data size: 1 GB - flowfile expiration: 0 sec - queue prioritizer class: '' -- id: 2079cf6f-0158-1000-0000-000000000000 - name: UpdateAttribute/success/input - source id: 2077ab1e-0158-1000-0000-000000000000 - source relationship names: - - success - destination id: 2078c936-0158-1000-0000-000000000000 - max work queue size: 10000 - max work queue data size: 1 GB - flowfile expiration: 0 sec - queue prioritizer class: '' -- id: 2079faa0-0158-1000-0000-000000000000 - name: output//LogAttribute - source id: 2079b327-0158-1000-0000-000000000000 - source relationship names: [] - destination id: 2079e8bd-0158-1000-0000-000000000000 - max work queue size: 10000 - max work queue data size: 1 GB - flowfile expiration: 0 sec - queue prioritizer class: '' -Remote Process Groups: [] -NiFi Properties Overrides: {} diff --git a/minifi/minifi-toolkit/minifi-toolkit-configuration/src/test/resources/ReplaceTextExpressionLanguageCSVReformatting-v1.yml b/minifi/minifi-toolkit/minifi-toolkit-configuration/src/test/resources/ReplaceTextExpressionLanguageCSVReformatting-v1.yml deleted file mode 100644 index 30dd2427da..0000000000 --- a/minifi/minifi-toolkit/minifi-toolkit-configuration/src/test/resources/ReplaceTextExpressionLanguageCSVReformatting-v1.yml +++ /dev/null @@ -1,146 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the \"License\"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an \"AS IS\" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -Flow Controller: - name: ReplaceTextExpressionLanguageCSVReformatting - comment: '' -Core Properties: - flow controller graceful shutdown period: 10 sec - flow service write delay interval: 500 ms - administrative yield duration: 30 sec - bored yield duration: 10 millis - max concurrent threads: 1 -FlowFile Repository: - implementation: org.apache.nifi.controller.repository.WriteAheadFlowFileRepository - partitions: 256 - checkpoint interval: 2 mins - always sync: false - Swap: - threshold: 20000 - in period: 5 sec - in threads: 1 - out period: 5 sec - out threads: 4 -Content Repository: - content claim max appendable size: 10 MB - content claim max flow files: 100 - always sync: false -Provenance Repository: - provenance rollover time: 1 min -Component Status Repository: - buffer size: 1440 - snapshot frequency: 1 min -Security Properties: - keystore: '' - keystore type: '' - keystore password: '' - key password: '' - truststore: '' - truststore type: '' - truststore password: '' - ssl protocol: '' - Sensitive Props: - key: - algorithm: NIFI_PBKDF2_AES_GCM_256 -Processors: -- name: Generate Empty File - class: org.apache.nifi.processors.standard.GenerateFlowFile - max concurrent tasks: 1 - scheduling strategy: TIMER_DRIVEN - scheduling period: 3 sec - penalization period: 30 sec - yield period: 1 sec - run duration nanos: 0 - auto-terminated relationships list: [] - Properties: - Batch Size: - Data Format: - File Size: 0b - Unique FlowFiles: -- name: No-Op Termination - class: org.apache.nifi.processors.attributes.UpdateAttribute - max concurrent tasks: 1 - scheduling strategy: TIMER_DRIVEN - scheduling period: 0 sec - penalization period: 30 sec - yield period: 1 sec - run duration nanos: 0 - auto-terminated relationships list: - - success - Properties: - Delete Attributes Expression: -- name: Reformat Date Column - class: org.apache.nifi.processors.standard.ReplaceText - max concurrent tasks: 1 - scheduling strategy: TIMER_DRIVEN - scheduling period: 0 sec - penalization period: 30 sec - yield period: 1 sec - run duration nanos: 0 - auto-terminated relationships list: - - failure - Properties: - Character Set: - Evaluation Mode: Line-by-Line - Maximum Buffer Size: - Regular Expression: (.*?),(.*?),(\d+.*) - Replacement Strategy: - Replacement Value: $1,$2,${ '$3':toDate('ddMMMyyyy'):format('yyyy/MM/dd') } -- name: Set CSV Content - class: org.apache.nifi.processors.standard.ReplaceText - max concurrent tasks: 1 - scheduling strategy: TIMER_DRIVEN - scheduling period: 0 sec - penalization period: 30 sec - yield period: 1 sec - run duration nanos: 0 - auto-terminated relationships list: - - failure - Properties: - Character Set: - Evaluation Mode: - Maximum Buffer Size: - Regular Expression: - Replacement Strategy: - Replacement Value: |- - 2006,10-01-2004,10may2004 - 2007,15-05-2006,10jun2005 - 2009,8-8-2008,10aug2008 -Connections: -- name: Generate Empty File/success/Set CSV Content - source name: Generate Empty File - source relationship name: success - destination name: Set CSV Content - max work queue size: 0 - max work queue data size: 0 MB - flowfile expiration: 0 sec - queue prioritizer class: org.apache.nifi.prioritizer.FirstInFirstOutPrioritizer -- name: Reformat Date Column/success/No-Op Termination - source name: Reformat Date Column - source relationship name: success - destination name: No-Op Termination - max work queue size: 0 - max work queue data size: 0 MB - flowfile expiration: 0 sec - queue prioritizer class: org.apache.nifi.prioritizer.FirstInFirstOutPrioritizer -- name: Set CSV Content/success/Reformat Date Column - source name: Set CSV Content - source relationship name: success - destination name: Reformat Date Column - max work queue size: 0 - max work queue data size: 0 MB - flowfile expiration: 0 sec - queue prioritizer class: org.apache.nifi.prioritizer.FirstInFirstOutPrioritizer -Remote Processing Groups: [] diff --git a/minifi/minifi-toolkit/minifi-toolkit-configuration/src/test/resources/ReplaceTextExpressionLanguageCSVReformatting.xml b/minifi/minifi-toolkit/minifi-toolkit-configuration/src/test/resources/ReplaceTextExpressionLanguageCSVReformatting.xml deleted file mode 100644 index 5793d3e603..0000000000 --- a/minifi/minifi-toolkit/minifi-toolkit-configuration/src/test/resources/ReplaceTextExpressionLanguageCSVReformatting.xml +++ /dev/null @@ -1,20 +0,0 @@ - - - \ No newline at end of file diff --git a/minifi/minifi-toolkit/minifi-toolkit-configuration/src/test/resources/SimpleRPGToLogAttributes.yml b/minifi/minifi-toolkit/minifi-toolkit-configuration/src/test/resources/SimpleRPGToLogAttributes.yml deleted file mode 100644 index 414bbed015..0000000000 --- a/minifi/minifi-toolkit/minifi-toolkit-configuration/src/test/resources/SimpleRPGToLogAttributes.yml +++ /dev/null @@ -1,121 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the \"License\"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an \"AS IS\" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -MiNiFi Config Version: 3 -Flow Controller: - name: SimpleRPGToLogAttributes - comment: '' -Core Properties: - flow controller graceful shutdown period: 10 sec - flow service write delay interval: 500 ms - administrative yield duration: 30 sec - bored yield duration: 10 millis - max concurrent threads: 1 - variable registry properties: '' -FlowFile Repository: - implementation: org.apache.nifi.controller.repository.WriteAheadFlowFileRepository - partitions: 256 - checkpoint interval: 2 mins - always sync: false - Swap: - threshold: 20000 - in period: 5 sec - in threads: 1 - out period: 5 sec - out threads: 4 -Content Repository: - implementation: org.apache.nifi.controller.repository.FileSystemRepository - content claim max appendable size: 10 MB - content claim max flow files: 100 - always sync: false - content repository archive max retention period: 12 hours - content repository archive max usage percentage: 50% - content repository archive enabled: false -Provenance Repository: - provenance rollover time: 1 min - implementation: org.apache.nifi.provenance.WriteAheadProvenanceRepository - provenance max storage time: 24 hours - provenance max storage size: 1 GB - provenance index shard size: 500 MB - provenance buffer size: 10000 -Component Status Repository: - buffer size: 1440 - snapshot frequency: 1 min -Security Properties: - keystore: '' - keystore type: '' - keystore password: '' - key password: '' - truststore: '' - truststore type: '' - truststore password: '' - ssl protocol: '' - Sensitive Props: - key: - algorithm: NIFI_PBKDF2_AES_GCM_256 -Processors: -- id: 6b97126a-015a-1000-0000-000000000000 - name: LogAttribute - class: org.apache.nifi.processors.standard.LogAttribute - max concurrent tasks: 1 - scheduling strategy: TIMER_DRIVEN - scheduling period: 0 sec - penalization period: 30 sec - yield period: 1 sec - run duration nanos: 0 - auto-terminated relationships list: - - success - Properties: - Attributes to Ignore: - Attributes to Log: - Log Level: info - Log Payload: 'false' - Log prefix: -Controller Services: [] -Process Groups: [] -Input Ports: [] -Output Ports: [] -Funnels: [] -Connections: -- id: 6b97306b-015a-1000-0000-000000000000 - name: 6b965db7-015a-1000-4c43-9d20931a1b9c//LogAttribute - source id: 6b965db7-015a-1000-4c43-9d20931a1b9c - source relationship names: [] - destination id: 6b97126a-015a-1000-0000-000000000000 - max work queue size: 10000 - max work queue data size: 1 GB - flowfile expiration: 0 sec - queue prioritizer class: '' -Remote Process Groups: -- id: 6b95d112-015a-1000-0000-000000000000 - name: '' - url: http://localhost:8080/nifi - comment: '' - timeout: 30 sec - yield period: 10 sec - transport protocol: HTTP - proxy host: '' - proxy port: '' - proxy user: '' - proxy password: '' - local network interface: '' - Input Ports: [] - Output Ports: - - id: 6b965db7-015a-1000-4c43-9d20931a1b9c - name: Output - comment: '' - max concurrent tasks: 1 - use compression: false -NiFi Properties Overrides: {} diff --git a/minifi/minifi-toolkit/minifi-toolkit-configuration/src/test/resources/SimpleTailFileToRPG-v1.yml b/minifi/minifi-toolkit/minifi-toolkit-configuration/src/test/resources/SimpleTailFileToRPG-v1.yml deleted file mode 100644 index 44a3c40133..0000000000 --- a/minifi/minifi-toolkit/minifi-toolkit-configuration/src/test/resources/SimpleTailFileToRPG-v1.yml +++ /dev/null @@ -1,102 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the \"License\"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an \"AS IS\" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -Flow Controller: - name: Simple TailFile To RPG - comment: '' -Core Properties: - flow controller graceful shutdown period: 10 sec - flow service write delay interval: 500 ms - administrative yield duration: 30 sec - bored yield duration: 10 millis - max concurrent threads: 1 -FlowFile Repository: - implementation: org.apache.nifi.controller.repository.WriteAheadFlowFileRepository - partitions: 256 - checkpoint interval: 2 mins - always sync: false - Swap: - threshold: 20000 - in period: 5 sec - in threads: 1 - out period: 5 sec - out threads: 4 -Content Repository: - content claim max appendable size: 10 MB - content claim max flow files: 100 - always sync: false -Provenance Repository: - provenance rollover time: 1 min -Component Status Repository: - buffer size: 1440 - snapshot frequency: 1 min -Security Properties: - keystore: '' - keystore type: '' - keystore password: '' - key password: '' - truststore: '' - truststore type: '' - truststore password: '' - ssl protocol: '' - Sensitive Props: - key: - algorithm: NIFI_PBKDF2_AES_GCM_256 -Processors: -- name: TailFile - class: org.apache.nifi.processors.standard.TailFile - max concurrent tasks: 1 - scheduling strategy: TIMER_DRIVEN - scheduling period: 1 sec - penalization period: 30 sec - yield period: 1 sec - run duration nanos: 0 - auto-terminated relationships list: [] - Properties: - File Location: Local - File to Tail: ./logs/minifi-app.log - Initial Start Position: Beginning of File - Rolling Filename Pattern: - tail-base-directory: - tail-mode: Single file - tailfile-lookup-frequency: 10 minutes - tailfile-maximum-age: 24 hours - tailfile-recursive-lookup: 'false' - tailfile-rolling-strategy: Fixed name -Connections: -- name: TailFile/success/ac0e798c-0158-1000-0588-cda9b944e011 - source name: TailFile - source relationship name: success - destination name: ac0e798c-0158-1000-0588-cda9b944e011 - max work queue size: 10000 - max work queue data size: 1 GB - flowfile expiration: 0 sec - queue prioritizer class: '' -Remote Processing Groups: -- url: http://localhost:8080/nifi - comment: '' - timeout: 30 sec - yield period: 10 sec - Input Ports: - - id: aca664f8-0158-1000-a139-92485891d349 - name: test2 - comment: '' - max concurrent tasks: 1 - use compression: false - - id: ac0e798c-0158-1000-0588-cda9b944e011 - name: test - comment: '' - max concurrent tasks: 1 - use compression: false diff --git a/minifi/minifi-toolkit/minifi-toolkit-configuration/src/test/resources/SimpleTailFileToRPG-v2.yml b/minifi/minifi-toolkit/minifi-toolkit-configuration/src/test/resources/SimpleTailFileToRPG-v2.yml deleted file mode 100644 index e3489a0395..0000000000 --- a/minifi/minifi-toolkit/minifi-toolkit-configuration/src/test/resources/SimpleTailFileToRPG-v2.yml +++ /dev/null @@ -1,113 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the \"License\"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an \"AS IS\" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -MiNiFi Config Version: 2 -Flow Controller: - name: Simple TailFile To RPG - comment: '' -Core Properties: - flow controller graceful shutdown period: 10 sec - flow service write delay interval: 500 ms - administrative yield duration: 30 sec - bored yield duration: 10 millis - max concurrent threads: 1 -FlowFile Repository: - implementation: org.apache.nifi.controller.repository.WriteAheadFlowFileRepository - partitions: 256 - checkpoint interval: 2 mins - always sync: false - Swap: - threshold: 20000 - in period: 5 sec - in threads: 1 - out period: 5 sec - out threads: 4 -Content Repository: - content claim max appendable size: 10 MB - content claim max flow files: 100 - always sync: false -Provenance Repository: - provenance rollover time: 1 min -Component Status Repository: - buffer size: 1440 - snapshot frequency: 1 min -Security Properties: - keystore: '' - keystore type: '' - keystore password: '' - key password: '' - truststore: '' - truststore type: '' - truststore password: '' - ssl protocol: '' - Sensitive Props: - key: - algorithm: NIFI_PBKDF2_AES_GCM_256 -Processors: -- id: b0c04f28-0158-1000-0000-000000000000 - name: TailFile - class: org.apache.nifi.processors.standard.TailFile - max concurrent tasks: 1 - scheduling strategy: TIMER_DRIVEN - scheduling period: 1 sec - penalization period: 30 sec - yield period: 1 sec - run duration nanos: 0 - auto-terminated relationships list: [] - Properties: - File Location: Local - File to Tail: ./logs/minifi-app.log - Initial Start Position: Beginning of File - Rolling Filename Pattern: - tail-base-directory: - tail-mode: Single file - tailfile-lookup-frequency: 10 minutes - tailfile-maximum-age: 24 hours - tailfile-recursive-lookup: 'false' - tailfile-rolling-strategy: Fixed name -Process Groups: [] -Input Ports: [] -Output Ports: [] -Funnels: [] -Connections: -- id: b0c0c3cc-0158-1000-0000-000000000000 - name: TailFile/success/ac0e798c-0158-1000-0588-cda9b944e011 - source id: b0c04f28-0158-1000-0000-000000000000 - source relationship names: - - success - destination id: ac0e798c-0158-1000-0588-cda9b944e011 - max work queue size: 10000 - max work queue data size: 1 GB - flowfile expiration: 0 sec - queue prioritizer class: '' -Remote Process Groups: -- id: b0c09ff0-0158-1000-0000-000000000000 - name: '' - url: http://localhost:8080/nifi - comment: '' - timeout: 30 sec - yield period: 10 sec - transport protocol: RAW - Input Ports: - - id: aca664f8-0158-1000-a139-92485891d349 - name: test2 - comment: '' - max concurrent tasks: 1 - use compression: false - - id: ac0e798c-0158-1000-0588-cda9b944e011 - name: test - comment: '' - max concurrent tasks: 1 - use compression: false diff --git a/minifi/minifi-toolkit/minifi-toolkit-configuration/src/test/resources/SimpleTailFileToRPG.xml b/minifi/minifi-toolkit/minifi-toolkit-configuration/src/test/resources/SimpleTailFileToRPG.xml deleted file mode 100644 index 4e804beb97..0000000000 --- a/minifi/minifi-toolkit/minifi-toolkit-configuration/src/test/resources/SimpleTailFileToRPG.xml +++ /dev/null @@ -1,214 +0,0 @@ - - - \ No newline at end of file diff --git a/minifi/minifi-toolkit/minifi-toolkit-configuration/src/test/resources/SimpleTailFileToRPG.yml b/minifi/minifi-toolkit/minifi-toolkit-configuration/src/test/resources/SimpleTailFileToRPG.yml deleted file mode 100644 index 199af94bd8..0000000000 --- a/minifi/minifi-toolkit/minifi-toolkit-configuration/src/test/resources/SimpleTailFileToRPG.yml +++ /dev/null @@ -1,131 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the \"License\"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an \"AS IS\" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -MiNiFi Config Version: 3 -Flow Controller: - name: Simple TailFile To RPG - comment: '' -Core Properties: - flow controller graceful shutdown period: 10 sec - flow service write delay interval: 500 ms - administrative yield duration: 30 sec - bored yield duration: 10 millis - max concurrent threads: 1 - variable registry properties: '' -FlowFile Repository: - implementation: org.apache.nifi.controller.repository.WriteAheadFlowFileRepository - partitions: 256 - checkpoint interval: 2 mins - always sync: false - Swap: - threshold: 20000 - in period: 5 sec - in threads: 1 - out period: 5 sec - out threads: 4 -Content Repository: - implementation: org.apache.nifi.controller.repository.FileSystemRepository - content claim max appendable size: 10 MB - content claim max flow files: 100 - always sync: false - content repository archive max retention period: 12 hours - content repository archive max usage percentage: 50% - content repository archive enabled: false -Provenance Repository: - provenance rollover time: 1 min - implementation: org.apache.nifi.provenance.WriteAheadProvenanceRepository - provenance max storage time: 24 hours - provenance max storage size: 1 GB - provenance index shard size: 500 MB - provenance buffer size: 10000 -Component Status Repository: - buffer size: 1440 - snapshot frequency: 1 min -Security Properties: - keystore: '' - keystore type: '' - keystore password: '' - key password: '' - truststore: '' - truststore type: '' - truststore password: '' - ssl protocol: '' - Sensitive Props: - key: - algorithm: NIFI_PBKDF2_AES_GCM_256 -Processors: -- id: b0c04f28-0158-1000-0000-000000000000 - name: TailFile - class: org.apache.nifi.processors.standard.TailFile - max concurrent tasks: 1 - scheduling strategy: TIMER_DRIVEN - scheduling period: 1 sec - penalization period: 30 sec - yield period: 1 sec - run duration nanos: 0 - auto-terminated relationships list: [] - Properties: - File Location: Local - File to Tail: ./logs/minifi-app.log - Initial Start Position: Beginning of File - Rolling Filename Pattern: - tail-base-directory: - tail-mode: Single file - tailfile-lookup-frequency: 10 minutes - tailfile-maximum-age: 24 hours - tailfile-recursive-lookup: 'false' - tailfile-rolling-strategy: Fixed name -Controller Services: [] -Process Groups: [] -Input Ports: [] -Output Ports: [] -Funnels: [] -Connections: -- id: b0c0c3cc-0158-1000-0000-000000000000 - name: TailFile/success/ac0e798c-0158-1000-0588-cda9b944e011 - source id: b0c04f28-0158-1000-0000-000000000000 - source relationship names: - - success - destination id: ac0e798c-0158-1000-0588-cda9b944e011 - max work queue size: 10000 - max work queue data size: 1 GB - flowfile expiration: 0 sec - queue prioritizer class: '' -Remote Process Groups: -- id: b0c09ff0-0158-1000-0000-000000000000 - name: '' - url: http://localhost:8080/nifi - comment: '' - timeout: 30 sec - yield period: 10 sec - transport protocol: RAW - proxy host: '' - proxy port: '' - proxy user: '' - proxy password: '' - local network interface: '' - Input Ports: - - id: aca664f8-0158-1000-a139-92485891d349 - name: test2 - comment: '' - max concurrent tasks: 1 - use compression: false - - id: ac0e798c-0158-1000-0588-cda9b944e011 - name: test - comment: '' - max concurrent tasks: 1 - use compression: false - Output Ports: [] -NiFi Properties Overrides: {} diff --git a/minifi/minifi-toolkit/minifi-toolkit-configuration/src/test/resources/StressTestFramework-v1.yml b/minifi/minifi-toolkit/minifi-toolkit-configuration/src/test/resources/StressTestFramework-v1.yml deleted file mode 100644 index 155dc4a2ba..0000000000 --- a/minifi/minifi-toolkit/minifi-toolkit-configuration/src/test/resources/StressTestFramework-v1.yml +++ /dev/null @@ -1,114 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the \"License\"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an \"AS IS\" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -Flow Controller: - name: StressTestFramework - comment: '' -Core Properties: - flow controller graceful shutdown period: 10 sec - flow service write delay interval: 500 ms - administrative yield duration: 30 sec - bored yield duration: 10 millis - max concurrent threads: 1 -FlowFile Repository: - implementation: org.apache.nifi.controller.repository.WriteAheadFlowFileRepository - partitions: 256 - checkpoint interval: 2 mins - always sync: false - Swap: - threshold: 20000 - in period: 5 sec - in threads: 1 - out period: 5 sec - out threads: 4 -Content Repository: - content claim max appendable size: 10 MB - content claim max flow files: 100 - always sync: false -Provenance Repository: - provenance rollover time: 1 min -Component Status Repository: - buffer size: 1440 - snapshot frequency: 1 min -Security Properties: - keystore: '' - keystore type: '' - keystore password: '' - key password: '' - truststore: '' - truststore type: '' - truststore password: '' - ssl protocol: '' - Sensitive Props: - key: - algorithm: NIFI_PBKDF2_AES_GCM_256 -Processors: -- name: GenerateFlowFile - class: org.apache.nifi.processors.standard.GenerateFlowFile - max concurrent tasks: 1 - scheduling strategy: TIMER_DRIVEN - scheduling period: 0 sec - penalization period: 30 sec - yield period: 1 sec - run duration nanos: 0 - auto-terminated relationships list: [] - Properties: - Batch Size: '100' - Data Format: - File Size: 0 KB - Unique FlowFiles: -- name: RouteOnAttribute - class: org.apache.nifi.processors.standard.RouteOnAttribute - max concurrent tasks: 2 - scheduling strategy: TIMER_DRIVEN - scheduling period: 0 sec - penalization period: 30 sec - yield period: 1 sec - run duration nanos: 50000 - auto-terminated relationships list: - - unmatched - Properties: - Routing Strategy: -- name: UpdateAttribute - class: org.apache.nifi.processors.attributes.UpdateAttribute - max concurrent tasks: 2 - scheduling strategy: TIMER_DRIVEN - scheduling period: 0 sec - penalization period: 30 sec - yield period: 1 sec - run duration nanos: 100000 - auto-terminated relationships list: [] - Properties: - Delete Attributes Expression: - property 1: value 1 - property 2: value 2 ${nextInt()} -Connections: -- name: GenerateFlowFile/success/UpdateAttribute - source name: GenerateFlowFile - source relationship name: success - destination name: UpdateAttribute - max work queue size: 200 - max work queue data size: 0 MB - flowfile expiration: 0 sec - queue prioritizer class: org.apache.nifi.prioritizer.FirstInFirstOutPrioritizer -- name: UpdateAttribute/success/RouteOnAttribute - source name: UpdateAttribute - source relationship name: success - destination name: RouteOnAttribute - max work queue size: 2000 - max work queue data size: 0 MB - flowfile expiration: 0 sec - queue prioritizer class: org.apache.nifi.prioritizer.FirstInFirstOutPrioritizer -Remote Processing Groups: [] diff --git a/minifi/minifi-toolkit/minifi-toolkit-configuration/src/test/resources/StressTestFramework.xml b/minifi/minifi-toolkit/minifi-toolkit-configuration/src/test/resources/StressTestFramework.xml deleted file mode 100644 index 0ee52d7a7d..0000000000 --- a/minifi/minifi-toolkit/minifi-toolkit-configuration/src/test/resources/StressTestFramework.xml +++ /dev/null @@ -1,18 +0,0 @@ - - - \ No newline at end of file diff --git a/minifi/minifi-toolkit/minifi-toolkit-configuration/src/test/resources/StressTestFrameworkFunnel.yml b/minifi/minifi-toolkit/minifi-toolkit-configuration/src/test/resources/StressTestFrameworkFunnel.yml deleted file mode 100644 index 8b4f5796ee..0000000000 --- a/minifi/minifi-toolkit/minifi-toolkit-configuration/src/test/resources/StressTestFrameworkFunnel.yml +++ /dev/null @@ -1,223 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the \"License\"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an \"AS IS\" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -MiNiFi Config Version: 3 -Flow Controller: - name: StressTestFramework - comment: '' -Core Properties: - flow controller graceful shutdown period: 10 sec - flow service write delay interval: 500 ms - administrative yield duration: 30 sec - bored yield duration: 10 millis - max concurrent threads: 1 - variable registry properties: '' -FlowFile Repository: - implementation: org.apache.nifi.controller.repository.WriteAheadFlowFileRepository - partitions: 256 - checkpoint interval: 2 mins - always sync: false - Swap: - threshold: 20000 - in period: 5 sec - in threads: 1 - out period: 5 sec - out threads: 4 -Content Repository: - implementation: org.apache.nifi.controller.repository.FileSystemRepository - content claim max appendable size: 10 MB - content claim max flow files: 100 - always sync: false - content repository archive max retention period: 12 hours - content repository archive max usage percentage: 50% - content repository archive enabled: false -Provenance Repository: - provenance rollover time: 1 min - implementation: org.apache.nifi.provenance.WriteAheadProvenanceRepository - provenance max storage time: 24 hours - provenance max storage size: 1 GB - provenance index shard size: 500 MB - provenance buffer size: 10000 -Component Status Repository: - buffer size: 1440 - snapshot frequency: 1 min -Security Properties: - keystore: '' - keystore type: '' - keystore password: '' - key password: '' - truststore: '' - truststore type: '' - truststore password: '' - ssl protocol: '' - Sensitive Props: - key: - algorithm: NIFI_PBKDF2_AES_GCM_256 -Processors: -- name: GenerateFlowFile - id: 16a47794-5391-4ad2-0000-000000000000 - class: org.apache.nifi.processors.standard.GenerateFlowFile - max concurrent tasks: 1 - scheduling strategy: TIMER_DRIVEN - scheduling period: 0 sec - penalization period: 30 sec - yield period: 1 sec - run duration nanos: 0 - auto-terminated relationships list: [] - Properties: - Batch Size: '100' - Data Format: Binary - File Size: 0 KB - Unique FlowFiles: 'false' -- name: GenerateFlowFile - id: 4ad21391-16a4-1794-0000-000000000000 - class: org.apache.nifi.processors.standard.GenerateFlowFile - max concurrent tasks: 1 - scheduling strategy: TIMER_DRIVEN - scheduling period: 0 sec - penalization period: 30 sec - yield period: 1 sec - run duration nanos: 0 - auto-terminated relationships list: [] - Properties: - Batch Size: '100' - Data Format: Binary - File Size: 0 KB - Unique FlowFiles: 'false' -- name: GenerateFlowFile - id: 53914ad2-7794-16a4-0000-000000000000 - class: org.apache.nifi.processors.standard.GenerateFlowFile - max concurrent tasks: 1 - scheduling strategy: TIMER_DRIVEN - scheduling period: 0 sec - penalization period: 30 sec - yield period: 1 sec - run duration nanos: 0 - auto-terminated relationships list: [] - Properties: - Batch Size: '100' - Data Format: Binary - File Size: 0 KB - Unique FlowFiles: 'false' -- name: GenerateFlowFile - id: 779416a4-4ad2-1391-0000-000000000000 - class: org.apache.nifi.processors.standard.GenerateFlowFile - max concurrent tasks: 1 - scheduling strategy: TIMER_DRIVEN - scheduling period: 0 sec - penalization period: 30 sec - yield period: 1 sec - run duration nanos: 0 - auto-terminated relationships list: [] - Properties: - Batch Size: '100' - Data Format: Binary - File Size: 0 KB - Unique FlowFiles: 'false' -- name: RouteOnAttribute - id: 397a4910-cc01-4c6b-0000-000000000000 - class: org.apache.nifi.processors.standard.RouteOnAttribute - max concurrent tasks: 2 - scheduling strategy: TIMER_DRIVEN - scheduling period: 0 sec - penalization period: 30 sec - yield period: 1 sec - run duration nanos: 50000 - auto-terminated relationships list: - - unmatched - Properties: - Routing Strategy: Route to Property name -- name: UpdateAttribute - id: 92557c76-f251-45a4-0000-000000000000 - class: org.apache.nifi.processors.attributes.UpdateAttribute - max concurrent tasks: 2 - scheduling strategy: TIMER_DRIVEN - scheduling period: 0 sec - penalization period: 30 sec - yield period: 1 sec - run duration nanos: 100000 - auto-terminated relationships list: [] - Properties: - Delete Attributes Expression: - property 1: value 1 - property 2: value 2 ${nextInt()} -Controller Services: [] -Process Groups: [] -Input Ports: [] -Output Ports: [] -Funnels: -- id: 4ad21392-16a4-1794-0000-000000000000 -Connections: -- name: 4ad21392-16a4-1794-0000-000000000000//UpdateAttribute - id: 4ad21397-16a4-1794-0000-000000000000 - source id: 4ad21392-16a4-1794-0000-000000000000 - source relationship names: [] - destination id: 92557c76-f251-45a4-0000-000000000000 - max work queue size: 10000 - max work queue data size: 1 GB - flowfile expiration: 0 sec - queue prioritizer class: '' -- name: GenerateFlowFile/success/4ad21392-16a4-1794-0000-000000000000 - id: 4ad21393-16a4-1794-0000-000000000000 - source id: 16a47794-5391-4ad2-0000-000000000000 - source relationship names: - - success - destination id: 4ad21392-16a4-1794-0000-000000000000 - max work queue size: 10000 - max work queue data size: 1 GB - flowfile expiration: 0 sec - queue prioritizer class: '' -- name: GenerateFlowFile/success/4ad21392-16a4-1794-0000-000000000000 - id: 4ad21394-16a4-1794-0000-000000000000 - source id: 53914ad2-7794-16a4-0000-000000000000 - source relationship names: - - success - destination id: 4ad21392-16a4-1794-0000-000000000000 - max work queue size: 10000 - max work queue data size: 1 GB - flowfile expiration: 0 sec - queue prioritizer class: '' -- name: GenerateFlowFile/success/4ad21392-16a4-1794-0000-000000000000 - id: 4ad21395-16a4-1794-0000-000000000000 - source id: 779416a4-4ad2-1391-0000-000000000000 - source relationship names: - - success - destination id: 4ad21392-16a4-1794-0000-000000000000 - max work queue size: 10000 - max work queue data size: 1 GB - flowfile expiration: 0 sec - queue prioritizer class: '' -- name: GenerateFlowFile/success/4ad21392-16a4-1794-0000-000000000000 - id: 4ad21396-16a4-1794-0000-000000000000 - source id: 4ad21391-16a4-1794-0000-000000000000 - source relationship names: - - success - destination id: 4ad21392-16a4-1794-0000-000000000000 - max work queue size: 10000 - max work queue data size: 1 GB - flowfile expiration: 0 sec - queue prioritizer class: '' -- name: UpdateAttribute/success/RouteOnAttribute - id: 4c53556e-eb46-458c-0000-000000000000 - source id: 92557c76-f251-45a4-0000-000000000000 - source relationship names: - - success - destination id: 397a4910-cc01-4c6b-0000-000000000000 - max work queue size: 2000 - max work queue data size: 0 MB - flowfile expiration: 0 sec - queue prioritizer class: org.apache.nifi.prioritizer.FirstInFirstOutPrioritizer -Remote Process Groups: [] -NiFi Properties Overrides: {} diff --git a/minifi/minifi-toolkit/minifi-toolkit-configuration/src/test/resources/TemplateWithFunnel.xml b/minifi/minifi-toolkit/minifi-toolkit-configuration/src/test/resources/TemplateWithFunnel.xml deleted file mode 100644 index 2e1b04f879..0000000000 --- a/minifi/minifi-toolkit/minifi-toolkit-configuration/src/test/resources/TemplateWithFunnel.xml +++ /dev/null @@ -1,18 +0,0 @@ - - - \ No newline at end of file diff --git a/minifi/minifi-toolkit/minifi-toolkit-configuration/src/test/resources/TemplateWithInputPort.xml b/minifi/minifi-toolkit/minifi-toolkit-configuration/src/test/resources/TemplateWithInputPort.xml deleted file mode 100644 index 37c2664dd9..0000000000 --- a/minifi/minifi-toolkit/minifi-toolkit-configuration/src/test/resources/TemplateWithInputPort.xml +++ /dev/null @@ -1,18 +0,0 @@ - - - \ No newline at end of file diff --git a/minifi/minifi-toolkit/minifi-toolkit-configuration/src/test/resources/TemplateWithOutputPort.xml b/minifi/minifi-toolkit/minifi-toolkit-configuration/src/test/resources/TemplateWithOutputPort.xml deleted file mode 100644 index 2ceeddd601..0000000000 --- a/minifi/minifi-toolkit/minifi-toolkit-configuration/src/test/resources/TemplateWithOutputPort.xml +++ /dev/null @@ -1,18 +0,0 @@ - - - \ No newline at end of file diff --git a/minifi/minifi-toolkit/minifi-toolkit-configuration/src/test/resources/TemplateWithProcessGroup.xml b/minifi/minifi-toolkit/minifi-toolkit-configuration/src/test/resources/TemplateWithProcessGroup.xml deleted file mode 100644 index 997e3ffb4d..0000000000 --- a/minifi/minifi-toolkit/minifi-toolkit-configuration/src/test/resources/TemplateWithProcessGroup.xml +++ /dev/null @@ -1,19 +0,0 @@ - -