diff --git a/nifi-assembly/pom.xml b/nifi-assembly/pom.xml index abcbd91366..76e001c4cb 100644 --- a/nifi-assembly/pom.xml +++ b/nifi-assembly/pom.xml @@ -336,12 +336,6 @@ language governing permissions and limitations under the License. --> 2.0.0-SNAPSHOT nar - - org.apache.nifi - nifi-html-nar - 2.0.0-SNAPSHOT - nar - org.apache.nifi nifi-lookup-services-nar @@ -420,12 +414,6 @@ language governing permissions and limitations under the License. --> 2.0.0-SNAPSHOT nar - - org.apache.nifi - nifi-hl7-nar - 2.0.0-SNAPSHOT - nar - org.apache.nifi nifi-language-translation-nar @@ -494,7 +482,7 @@ language governing permissions and limitations under the License. --> org.apache.nifi - nifi-riemann-nar + nifi-hl7-nar 2.0.0-SNAPSHOT nar @@ -552,12 +540,6 @@ language governing permissions and limitations under the License. --> 2.0.0-SNAPSHOT nar - - org.apache.nifi - nifi-cybersecurity-nar - 2.0.0-SNAPSHOT - nar - org.apache.nifi nifi-email-nar @@ -696,12 +678,6 @@ language governing permissions and limitations under the License. --> 2.0.0-SNAPSHOT nar - - org.apache.nifi - nifi-tcp-nar - 2.0.0-SNAPSHOT - nar - org.apache.nifi nifi-gcp-nar @@ -792,18 +768,6 @@ language governing permissions and limitations under the License. --> 2.0.0-SNAPSHOT nar - - org.apache.nifi - nifi-metrics-reporter-service-api-nar - 2.0.0-SNAPSHOT - nar - - - org.apache.nifi - nifi-metrics-reporting-nar - 2.0.0-SNAPSHOT - nar - org.apache.nifi nifi-kerberos-credentials-service-nar @@ -1084,24 +1048,6 @@ language governing permissions and limitations under the License. --> - - include-rules - - - false - - allProfiles - - - - - org.apache.nifi - nifi-rules-action-handler-nar - 2.0.0-SNAPSHOT - nar - - - include-sql-reporting @@ -1118,12 +1064,6 @@ language governing permissions and limitations under the License. --> 2.0.0-SNAPSHOT nar - - org.apache.nifi - nifi-rules-action-handler-nar - 2.0.0-SNAPSHOT - nar - @@ -1440,23 +1380,6 @@ language governing permissions and limitations under the License. --> - - include-riemann - - false - - allProfiles - - - - - org.apache.nifi - nifi-riemann-nar - 2.0.0-SNAPSHOT - nar - - - include-snowflake diff --git a/nifi-docs/src/main/asciidoc/developer-guide.adoc b/nifi-docs/src/main/asciidoc/developer-guide.adoc index 509d81a9ae..185883d655 100644 --- a/nifi-docs/src/main/asciidoc/developer-guide.adoc +++ b/nifi-docs/src/main/asciidoc/developer-guide.adoc @@ -2730,10 +2730,7 @@ deprecationLogger.warn( | ASN.1 Support | include-asn1 | Adds support for ASN.1 | Contribution Check | contrib-check | Runs various quality checks that are required to be accepted before a contribution can be accepted into the core NiFi code base. | Graph Database Bundle | include-graph | Adds support for various common graph database scenarios. Support is currently for https://neo4j.com/developer/cypher[Cypher] and https://tinkerpop.apache.org/gremlin.html[Gremlin]-compatible databases such as Neo4J and JanusGraph. Includes controller services that provide driver functionality and a suite of processors for ingestion and querying. -| GRPC Bundle | include-grpc | **This profile is active in official builds and should be active** Provides support for the GRPC protocol. | Media Bundle | include-media | The media bundle provides functionality based on https://tika.apache.org[Apache Tika] for extracting content and metadata from various types of binary formats supported by Apache Tika (ex. PDF, Microsoft Office). -| Riemann Bundle | include-riemann | Adds support for Riemann database components -| Rules Engine Bundle | include-rules | Adds support for creating scripted rules engines that can be integrated into existing flows. These rules engines can provide flexibility to people who need more complex flow logic or are more comfortable with flow decision-making using custom code. | Snowflake Bundle | include-snowflake | Adds support for integration with the https://www.snowflake.com[Snowflake platform]. | SQL Reporting Bundle | include-sql-reporting | Adds reporting tasks that are designed to use SQL to update a RDBMS with metrics and other related data from Apache NiFi. |================================================================================================================================================== diff --git a/nifi-nar-bundles/nifi-cybersecurity-bundle/nifi-cybersecurity-nar/pom.xml b/nifi-nar-bundles/nifi-cybersecurity-bundle/nifi-cybersecurity-nar/pom.xml deleted file mode 100644 index a2e244f23d..0000000000 --- a/nifi-nar-bundles/nifi-cybersecurity-bundle/nifi-cybersecurity-nar/pom.xml +++ /dev/null @@ -1,39 +0,0 @@ - - - - 4.0.0 - - - org.apache.nifi - nifi-cybersecurity-bundle - 2.0.0-SNAPSHOT - - - nifi-cybersecurity-nar - nar - - true - true - - - - - org.apache.nifi - nifi-cybersecurity-processors - 2.0.0-SNAPSHOT - - - diff --git a/nifi-nar-bundles/nifi-cybersecurity-bundle/nifi-cybersecurity-nar/src/main/resources/META-INF/LICENSE b/nifi-nar-bundles/nifi-cybersecurity-bundle/nifi-cybersecurity-nar/src/main/resources/META-INF/LICENSE deleted file mode 100644 index 1806c0a1d2..0000000000 --- a/nifi-nar-bundles/nifi-cybersecurity-bundle/nifi-cybersecurity-nar/src/main/resources/META-INF/LICENSE +++ /dev/null @@ -1,257 +0,0 @@ - - Apache License - Version 2.0, January 2004 - http://www.apache.org/licenses/ - - TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION - - 1. Definitions. - - "License" shall mean the terms and conditions for use, reproduction, - and distribution as defined by Sections 1 through 9 of this document. - - "Licensor" shall mean the copyright owner or entity authorized by - the copyright owner that is granting the License. - - "Legal Entity" shall mean the union of the acting entity and all - other entities that control, are controlled by, or are under common - control with that entity. For the purposes of this definition, - "control" means (i) the power, direct or indirect, to cause the - direction or management of such entity, whether by contract or - otherwise, or (ii) ownership of fifty percent (50%) or more of the - outstanding shares, or (iii) beneficial ownership of such entity. - - "You" (or "Your") shall mean an individual or Legal Entity - exercising permissions granted by this License. - - "Source" form shall mean the preferred form for making modifications, - including but not limited to software source code, documentation - source, and configuration files. - - "Object" form shall mean any form resulting from mechanical - transformation or translation of a Source form, including but - not limited to compiled object code, generated documentation, - and conversions to other media types. - - "Work" shall mean the work of authorship, whether in Source or - Object form, made available under the License, as indicated by a - copyright notice that is included in or attached to the work - (an example is provided in the Appendix below). - - "Derivative Works" shall mean any work, whether in Source or Object - form, that is based on (or derived from) the Work and for which the - editorial revisions, annotations, elaborations, or other modifications - represent, as a whole, an original work of authorship. For the purposes - of this License, Derivative Works shall not include works that remain - separable from, or merely link (or bind by name) to the interfaces of, - the Work and Derivative Works thereof. - - "Contribution" shall mean any work of authorship, including - the original version of the Work and any modifications or additions - to that Work or Derivative Works thereof, that is intentionally - submitted to Licensor for inclusion in the Work by the copyright owner - or by an individual or Legal Entity authorized to submit on behalf of - the copyright owner. For the purposes of this definition, "submitted" - means any form of electronic, verbal, or written communication sent - to the Licensor or its representatives, including but not limited to - communication on electronic mailing lists, source code control systems, - and issue tracking systems that are managed by, or on behalf of, the - Licensor for the purpose of discussing and improving the Work, but - excluding communication that is conspicuously marked or otherwise - designated in writing by the copyright owner as "Not a Contribution." - - "Contributor" shall mean Licensor and any individual or Legal Entity - on behalf of whom a Contribution has been received by Licensor and - subsequently incorporated within the Work. - - 2. Grant of Copyright License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - copyright license to reproduce, prepare Derivative Works of, - publicly display, publicly perform, sublicense, and distribute the - Work and such Derivative Works in Source or Object form. - - 3. Grant of Patent License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - (except as stated in this section) patent license to make, have made, - use, offer to sell, sell, import, and otherwise transfer the Work, - where such license applies only to those patent claims licensable - by such Contributor that are necessarily infringed by their - Contribution(s) alone or by combination of their Contribution(s) - with the Work to which such Contribution(s) was submitted. If You - institute patent litigation against any entity (including a - cross-claim or counterclaim in a lawsuit) alleging that the Work - or a Contribution incorporated within the Work constitutes direct - or contributory patent infringement, then any patent licenses - granted to You under this License for that Work shall terminate - as of the date such litigation is filed. - - 4. Redistribution. You may reproduce and distribute copies of the - Work or Derivative Works thereof in any medium, with or without - modifications, and in Source or Object form, provided that You - meet the following conditions: - - (a) You must give any other recipients of the Work or - Derivative Works a copy of this License; and - - (b) You must cause any modified files to carry prominent notices - stating that You changed the files; and - - (c) You must retain, in the Source form of any Derivative Works - that You distribute, all copyright, patent, trademark, and - attribution notices from the Source form of the Work, - excluding those notices that do not pertain to any part of - the Derivative Works; and - - (d) If the Work includes a "NOTICE" text file as part of its - distribution, then any Derivative Works that You distribute must - include a readable copy of the attribution notices contained - within such NOTICE file, excluding those notices that do not - pertain to any part of the Derivative Works, in at least one - of the following places: within a NOTICE text file distributed - as part of the Derivative Works; within the Source form or - documentation, if provided along with the Derivative Works; or, - within a display generated by the Derivative Works, if and - wherever such third-party notices normally appear. The contents - of the NOTICE file are for informational purposes only and - do not modify the License. You may add Your own attribution - notices within Derivative Works that You distribute, alongside - or as an addendum to the NOTICE text from the Work, provided - that such additional attribution notices cannot be construed - as modifying the License. - - You may add Your own copyright statement to Your modifications and - may provide additional or different license terms and conditions - for use, reproduction, or distribution of Your modifications, or - for any such Derivative Works as a whole, provided Your use, - reproduction, and distribution of the Work otherwise complies with - the conditions stated in this License. - - 5. Submission of Contributions. Unless You explicitly state otherwise, - any Contribution intentionally submitted for inclusion in the Work - by You to the Licensor shall be under the terms and conditions of - this License, without any additional terms or conditions. - Notwithstanding the above, nothing herein shall supersede or modify - the terms of any separate license agreement you may have executed - with Licensor regarding such Contributions. - - 6. Trademarks. This License does not grant permission to use the trade - names, trademarks, service marks, or product names of the Licensor, - except as required for reasonable and customary use in describing the - origin of the Work and reproducing the content of the NOTICE file. - - 7. Disclaimer of Warranty. Unless required by applicable law or - agreed to in writing, Licensor provides the Work (and each - Contributor provides its Contributions) on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - implied, including, without limitation, any warranties or conditions - of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A - PARTICULAR PURPOSE. You are solely responsible for determining the - appropriateness of using or redistributing the Work and assume any - risks associated with Your exercise of permissions under this License. - - 8. Limitation of Liability. In no event and under no legal theory, - whether in tort (including negligence), contract, or otherwise, - unless required by applicable law (such as deliberate and grossly - negligent acts) or agreed to in writing, shall any Contributor be - liable to You for damages, including any direct, indirect, special, - incidental, or consequential damages of any character arising as a - result of this License or out of the use or inability to use the - Work (including but not limited to damages for loss of goodwill, - work stoppage, computer failure or malfunction, or any and all - other commercial damages or losses), even if such Contributor - has been advised of the possibility of such damages. - - 9. Accepting Warranty or Additional Liability. While redistributing - the Work or Derivative Works thereof, You may choose to offer, - and charge a fee for, acceptance of support, warranty, indemnity, - or other liability obligations and/or rights consistent with this - License. However, in accepting such obligations, You may act only - on Your own behalf and on Your sole responsibility, not on behalf - of any other Contributor, and only if You agree to indemnify, - defend, and hold each Contributor harmless for any liability - incurred by, or claims asserted against, such Contributor by reason - of your accepting any such warranty or additional liability. - - END OF TERMS AND CONDITIONS - - APPENDIX: How to apply the Apache License to your work. - - To apply the Apache License to your work, attach the following - boilerplate notice, with the fields enclosed by brackets "[]" - replaced with your own identifying information. (Don't include - the brackets!) The text should be enclosed in the appropriate - comment syntax for the file format. We also recommend that a - file or class name and description of purpose be included on the - same "printed page" as the copyright notice for easier - identification within third-party archives. - - Copyright [yyyy] [name of copyright owner] - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - - APACHE NIFI SUBCOMPONENTS: - - The Apache NiFi project contains subcomponents with separate copyright - notices and license terms. Your use of the source code for the these - subcomponents is subject to the terms and conditions of the following - licenses. - - The binary distribution of this project bundles "java-spamsum" - which is available under an MIT style license. - - Copyright 2015 Thibault Debatty. - - Permission is hereby granted, free of charge, to any person obtaining - a copy of this software and associated documentation files (the - "Software"), to deal in the Software without restriction, including - without limitation the rights to use, copy, modify, merge, publish, - distribute, sublicense, and/or sell copies of the Software, and to - permit persons to whom the Software is furnished to do so, subject to - the following conditions: - - The above copyright notice and this permission notice shall be - included in all copies or substantial portions of the Software. - - THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, - EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF - MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND - NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE - LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION - OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION - WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. - - The binary distribution of this project bundles "java-string-similarity" - which is available under an MIT style license. - - Copyright 2015 Thibault Debatty. - - Permission is hereby granted, free of charge, to any person obtaining - a copy of this software and associated documentation files (the - "Software"), to deal in the Software without restriction, including - without limitation the rights to use, copy, modify, merge, publish, - distribute, sublicense, and/or sell copies of the Software, and to - permit persons to whom the Software is furnished to do so, subject to - the following conditions: - - The above copyright notice and this permission notice shall be - included in all copies or substantial portions of the Software. - - THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, - EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF - MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND - NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE - LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION - OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION - WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. \ No newline at end of file diff --git a/nifi-nar-bundles/nifi-cybersecurity-bundle/nifi-cybersecurity-nar/src/main/resources/META-INF/NOTICE b/nifi-nar-bundles/nifi-cybersecurity-bundle/nifi-cybersecurity-nar/src/main/resources/META-INF/NOTICE deleted file mode 100644 index 15b9022255..0000000000 --- a/nifi-nar-bundles/nifi-cybersecurity-bundle/nifi-cybersecurity-nar/src/main/resources/META-INF/NOTICE +++ /dev/null @@ -1,34 +0,0 @@ -nifi-cybersecurity-nar -Copyright 2015-2020 The Apache Software Foundation - - -=========================================== -Apache Software License v2 -=========================================== - -The following binary components are provided under the Apache Software License v2 - - (ASLv2) tlsh - The following NOTICE information applies: - Java port of Trend Locality Sensitive Hash (TLSH) - Copyright 2000-2016 Idealista, S.A - - This product includes software developed at - Idealista, S.A (http://www.idealista.com/) - - Based on the algorithms described in Trend Micro's TLSH official - repository, available in: - - https://github.com/trendmicro/tlsh - - Refer to the following publication for more information: - - Jonathan Oliver, Chun Cheng and Yanggui Chen, “TLSH - A Locality - Sensitive Hash” - 4th Cybercrime and Trustworthy Computing Workshop, - Sydney, November 2013 - - https://drive.google.com/file/d/0B6FS3SVQ1i0GTXk5eDl3Y29QWlk/edit?usp=sharing - - This software is inspired in the previous Java port developed by TripleCheck: - - https://github.com/triplecheck/TLSH \ No newline at end of file diff --git a/nifi-nar-bundles/nifi-cybersecurity-bundle/nifi-cybersecurity-processors/pom.xml b/nifi-nar-bundles/nifi-cybersecurity-bundle/nifi-cybersecurity-processors/pom.xml deleted file mode 100644 index e6385d5430..0000000000 --- a/nifi-nar-bundles/nifi-cybersecurity-bundle/nifi-cybersecurity-processors/pom.xml +++ /dev/null @@ -1,76 +0,0 @@ - - - - 4.0.0 - - - org.apache.nifi - nifi-cybersecurity-bundle - 2.0.0-SNAPSHOT - - - nifi-cybersecurity-processors - jar - - - - org.apache.nifi - nifi-api - - - org.apache.nifi - nifi-utils - 2.0.0-SNAPSHOT - - - info.debatty - java-spamsum - 0.2 - - - com.idealista - tlsh - 1.0.0 - - - org.apache.nifi - nifi-mock - 2.0.0-SNAPSHOT - test - - - org.apache.nifi - nifi-properties - compile - - - - - - org.apache.rat - apache-rat-plugin - - - src/test/resources/blank_ssdeep.list - src/test/resources/empty.list - src/test/resources/ssdeep.list - src/test/resources/tlsh.list - - - - - - diff --git a/nifi-nar-bundles/nifi-cybersecurity-bundle/nifi-cybersecurity-processors/src/main/java/org/apache/nifi/processors/cybersecurity/AbstractFuzzyHashProcessor.java b/nifi-nar-bundles/nifi-cybersecurity-bundle/nifi-cybersecurity-processors/src/main/java/org/apache/nifi/processors/cybersecurity/AbstractFuzzyHashProcessor.java deleted file mode 100644 index c5b560d7d2..0000000000 --- a/nifi-nar-bundles/nifi-cybersecurity-bundle/nifi-cybersecurity-processors/src/main/java/org/apache/nifi/processors/cybersecurity/AbstractFuzzyHashProcessor.java +++ /dev/null @@ -1,88 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.nifi.processors.cybersecurity; - - -import com.idealista.tlsh.TLSH; -import info.debatty.java.spamsum.SpamSum; -import org.apache.nifi.components.AllowableValue; -import org.apache.nifi.components.PropertyDescriptor; -import org.apache.nifi.flowfile.FlowFile; -import org.apache.nifi.processor.AbstractProcessor; -import org.apache.nifi.processor.Relationship; -import org.apache.nifi.processor.util.StandardValidators; - -import java.util.List; -import java.util.Set; - -abstract class AbstractFuzzyHashProcessor extends AbstractProcessor { - final protected static String ssdeep = "ssdeep"; - final protected static String tlsh = "tlsh"; - - public static final AllowableValue allowableValueSSDEEP = new AllowableValue( - ssdeep, - ssdeep, - "Uses ssdeep / SpamSum 'context triggered piecewise hash'."); - public static final AllowableValue allowableValueTLSH = new AllowableValue( - tlsh, - tlsh, - "Uses TLSH (Trend 'Locality Sensitive Hash'). Note: FlowFile Content must be at least 512 characters long"); - - public static final PropertyDescriptor ATTRIBUTE_NAME = new PropertyDescriptor.Builder() - .name("ATTRIBUTE_NAME") - .displayName("Hash Attribute Name") - .description("The name of the FlowFile Attribute that should hold the Fuzzy Hash Value") - .required(true) - .addValidator(StandardValidators.NON_EMPTY_VALIDATOR) - .defaultValue("fuzzyhash.value") - .build(); - - public static final PropertyDescriptor HASH_ALGORITHM = new PropertyDescriptor.Builder() - .name("HASH_ALGORITHM") - .displayName("Hashing Algorithm") - .description("The hashing algorithm utilised") - .allowableValues(allowableValueSSDEEP, allowableValueTLSH) - .required(true) - .addValidator(StandardValidators.NON_EMPTY_VALIDATOR) - .build(); - - - protected List descriptors; - - protected Set relationships; - - protected boolean checkMinimumAlgorithmRequirements(String algorithm, FlowFile flowFile) { - // Check if content matches minimum length requirement - if (algorithm.equals(tlsh) && flowFile.getSize() < 512 ) { - return false; - } else { - return true; - } - } - - - protected String generateHash(String algorithm, String content) { - switch (algorithm) { - case tlsh: - return new TLSH(content).hash(); - case ssdeep: - return new SpamSum().HashString(content); - default: - return null; - } - } -} diff --git a/nifi-nar-bundles/nifi-cybersecurity-bundle/nifi-cybersecurity-processors/src/main/java/org/apache/nifi/processors/cybersecurity/CompareFuzzyHash.java b/nifi-nar-bundles/nifi-cybersecurity-bundle/nifi-cybersecurity-processors/src/main/java/org/apache/nifi/processors/cybersecurity/CompareFuzzyHash.java deleted file mode 100644 index af520740e1..0000000000 --- a/nifi-nar-bundles/nifi-cybersecurity-bundle/nifi-cybersecurity-processors/src/main/java/org/apache/nifi/processors/cybersecurity/CompareFuzzyHash.java +++ /dev/null @@ -1,266 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.nifi.processors.cybersecurity; - -import org.apache.nifi.annotation.behavior.InputRequirement; -import org.apache.nifi.annotation.behavior.SideEffectFree; -import org.apache.nifi.annotation.behavior.SupportsBatching; -import org.apache.nifi.annotation.behavior.WritesAttribute; -import org.apache.nifi.annotation.behavior.WritesAttributes; -import org.apache.nifi.annotation.documentation.CapabilityDescription; -import org.apache.nifi.annotation.documentation.SeeAlso; -import org.apache.nifi.annotation.documentation.Tags; -import org.apache.nifi.components.AllowableValue; -import org.apache.nifi.components.PropertyDescriptor; -import org.apache.nifi.components.resource.ResourceCardinality; -import org.apache.nifi.components.resource.ResourceType; -import org.apache.nifi.flowfile.FlowFile; -import org.apache.nifi.logging.ComponentLog; -import org.apache.nifi.processor.ProcessContext; -import org.apache.nifi.processor.ProcessSession; -import org.apache.nifi.processor.ProcessorInitializationContext; -import org.apache.nifi.processor.Relationship; -import org.apache.nifi.processor.exception.ProcessException; -import org.apache.nifi.processor.util.StandardValidators; -import org.apache.nifi.processors.cybersecurity.matchers.FuzzyHashMatcher; -import org.apache.nifi.processors.cybersecurity.matchers.SSDeepHashMatcher; -import org.apache.nifi.processors.cybersecurity.matchers.TLSHHashMatcher; -import org.apache.nifi.util.StringUtils; - -import java.io.BufferedReader; -import java.io.IOException; -import java.util.ArrayList; -import java.util.Collections; -import java.util.HashMap; -import java.util.HashSet; -import java.util.LinkedHashMap; -import java.util.List; -import java.util.Map; -import java.util.Objects; -import java.util.Set; - - -@SideEffectFree -@SupportsBatching -@InputRequirement(InputRequirement.Requirement.INPUT_REQUIRED) -@SeeAlso({FuzzyHashContent.class}) -@Tags({"hashing", "fuzzy-hashing", "cyber-security"}) -@CapabilityDescription("Compares an attribute containing a Fuzzy Hash against a file containing a list of fuzzy hashes, " + - "appending an attribute to the FlowFile in case of a successful match.") - -@WritesAttributes({ - @WritesAttribute(attribute = "XXXX.N.match", description = "The match that resembles the attribute specified " + - "by the property. Note that: 'XXX' gets replaced with the "), - @WritesAttribute(attribute = "XXXX.N.similarity", description = "The similarity score between this flowfile" + - "and its match of the same number N. Note that: 'XXX' gets replaced with the ")}) - -public class CompareFuzzyHash extends AbstractFuzzyHashProcessor { - public static final AllowableValue singleMatch = new AllowableValue( - "single", - "single", - "Send FlowFile to matched after the first match above threshold"); - public static final AllowableValue multiMatch = new AllowableValue( - "multi-match", - "multi-match", - "Iterate full list of hashes before deciding to send FlowFile to matched or unmatched"); - - public static final PropertyDescriptor HASH_LIST_FILE = new PropertyDescriptor.Builder() - .name("HASH_LIST_FILE") - .displayName("Hash List Source File") - .description("Path to the file containing hashes to be validated against") - .required(true) - .identifiesExternalResource(ResourceCardinality.SINGLE, ResourceType.FILE) - .build(); - - // Note we add a PropertyDescriptor HASH_ALGORITHM and ATTRIBUTE_NAME from parent class - - public static final PropertyDescriptor MATCH_THRESHOLD = new PropertyDescriptor.Builder() - // Note that while both TLSH and SSDeep seems to return int, we treat them as double in code. - // The rationale behind being the expectation that other algorithms thatmay return double values - // may be added to the processor later on. - .name("MATCH_THRESHOLD") - .displayName("Match Threshold") - .description("The similarity score must exceed or be equal to in order for" + - "match to be considered true. Refer to Additional Information for differences between TLSH " + - "and SSDEEP scores and how they relate to this property.") - .required(true) - .addValidator(StandardValidators.NUMBER_VALIDATOR) - .build(); - - public static final PropertyDescriptor MATCHING_MODE = new PropertyDescriptor.Builder() - .name("MATCHING_MODE") - .displayName("Matching Mode") - .description("Defines if the Processor should try to match as many entries as possible (" + multiMatch.getDisplayName() + - ") or if it should stop after the first match (" + singleMatch.getDisplayName() + ")") - .required(true) - .allowableValues(singleMatch,multiMatch) - .defaultValue(singleMatch.getValue()) - .build(); - - public static final Relationship REL_FOUND = new Relationship.Builder() - .name("found") - .description("Any FlowFile that is successfully matched to an existing hash will be sent to this Relationship.") - .build(); - - public static final Relationship REL_NOT_FOUND = new Relationship.Builder() - .name("not-found") - .description("Any FlowFile that cannot be matched to an existing hash will be sent to this Relationship.") - .build(); - - public static final Relationship REL_FAILURE = new Relationship.Builder() - .name("failure") - .description("Any FlowFile that cannot be matched, e.g. (lacks the attribute) will be sent to this Relationship.") - .build(); - - @Override - protected void init(final ProcessorInitializationContext context) { - final List descriptors = new ArrayList(); - descriptors.add(HASH_LIST_FILE); - // As mentioned above, add the PropertyDescriptor HASH_ALGORITHM and ATTRIBUTE_NAME from parent class - descriptors.add(HASH_ALGORITHM); - descriptors.add(ATTRIBUTE_NAME); - descriptors.add(MATCH_THRESHOLD); - descriptors.add(MATCHING_MODE); - this.descriptors = Collections.unmodifiableList(descriptors); - - final Set relationships = new HashSet(); - relationships.add(REL_FOUND); - relationships.add(REL_NOT_FOUND); - relationships.add(REL_FAILURE); - this.relationships = Collections.unmodifiableSet(relationships); - } - - @Override - public Set getRelationships() { - return this.relationships; - } - - @Override - public final List getSupportedPropertyDescriptors() { - return descriptors; - } - - - @Override - public void onTrigger(ProcessContext context, ProcessSession session) throws ProcessException { - - FlowFile flowFile = session.get(); - if (flowFile == null) { - return; - } - - final ComponentLog logger = getLogger(); - String algorithm = context.getProperty(HASH_ALGORITHM).getValue(); - - final String attributeName = context.getProperty(ATTRIBUTE_NAME).getValue(); - String inputHash = flowFile.getAttribute(attributeName); - - if (inputHash == null) { - getLogger().info("FlowFile {} lacks the required '{}' attribute, routing to failure.", - new Object[]{flowFile, attributeName}); - session.transfer(flowFile, REL_FAILURE); - return; - } - - final FuzzyHashMatcher fuzzyHashMatcher; - - switch (algorithm) { - case tlsh: - fuzzyHashMatcher = new TLSHHashMatcher(getLogger()); - break; - case ssdeep: - fuzzyHashMatcher = new SSDeepHashMatcher(getLogger()); - break; - default: - getLogger().error("Seems like the processor is configured to use unsupported algorithm '{}' ? Yielding.", - new Object[]{algorithm}); - context.yield(); - return; - } - - if (!fuzzyHashMatcher.isValidHash(inputHash)) { - // and if that is the case we log - logger.error("Invalid hash provided for {}. Sending to failure", flowFile); - // and send to failure - session.transfer(flowFile, REL_FAILURE); - return; - } - - double similarity = 0; - double matchThreshold = context.getProperty(MATCH_THRESHOLD).asDouble(); - - try { - Map matched = new LinkedHashMap<>(); - - try (BufferedReader reader = fuzzyHashMatcher.getReader(context.getProperty(HASH_LIST_FILE).getValue())) { - String line = null; - - while ((line = reader.readLine()) != null) { - similarity = fuzzyHashMatcher.getSimilarity(inputHash, line); - - if (fuzzyHashMatcher.matchExceedsThreshold(similarity, matchThreshold)) { - String match = fuzzyHashMatcher.getMatch(line); - // A malformed file may cause a match with no filename - // Because this would simply look odd, we ignore such entry and log - if (!StringUtils.isEmpty(match)) { - matched.put(match, similarity); - } else { - logger.error("Found a match against a malformed entry '{}'. Please inspect the contents of" + - "the {} file and ensure they are properly formatted", - new Object[]{line, HASH_LIST_FILE.getDisplayName()}); - } - } - - // Check if single match is desired and if a match has been made - if (Objects.equals(context.getProperty(MATCHING_MODE).getValue(), singleMatch.getValue()) && (matched.size() > 0)) { - // and save time by breaking the outer loop - break; - } - } - } - - // Then by iterating over the hashmap of matches - if (matched.size() > 0) { - // no matter if the break was called or not, Continue processing - // First by creating a new map to hold attributes - final Map attributes = new HashMap<>(); - - int x = 0; - for (Map.Entry entry : matched.entrySet()) { - // defining attributes accordingly - attributes.put(attributeName + "." + x + ".match", entry.getKey()); - attributes.put(attributeName + "." + x + ".similarity", String.valueOf(entry.getValue())); - x++; - } - - // Finally, append the attributes to the flowfile and sent to match - flowFile = session.putAllAttributes(flowFile, attributes); - session.transfer(flowFile, REL_FOUND); - } else { - // Otherwise send it to non-match - session.transfer(flowFile, REL_NOT_FOUND); - } - } catch (final IOException e) { - logger.error("Error while reading the hash input source for {}", flowFile, e); - session.transfer(flowFile, REL_FAILURE); - } - } - - - -} diff --git a/nifi-nar-bundles/nifi-cybersecurity-bundle/nifi-cybersecurity-processors/src/main/java/org/apache/nifi/processors/cybersecurity/FuzzyHashContent.java b/nifi-nar-bundles/nifi-cybersecurity-bundle/nifi-cybersecurity-processors/src/main/java/org/apache/nifi/processors/cybersecurity/FuzzyHashContent.java deleted file mode 100644 index f6acd1f11c..0000000000 --- a/nifi-nar-bundles/nifi-cybersecurity-bundle/nifi-cybersecurity-processors/src/main/java/org/apache/nifi/processors/cybersecurity/FuzzyHashContent.java +++ /dev/null @@ -1,175 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.nifi.processors.cybersecurity; - -import com.idealista.tlsh.exceptions.InsufficientComplexityException; - -import org.apache.nifi.annotation.behavior.InputRequirement; -import org.apache.nifi.annotation.behavior.SideEffectFree; -import org.apache.nifi.annotation.behavior.SupportsBatching; -import org.apache.nifi.annotation.behavior.WritesAttribute; -import org.apache.nifi.annotation.behavior.WritesAttributes; -import org.apache.nifi.annotation.documentation.CapabilityDescription; -import org.apache.nifi.annotation.documentation.SeeAlso; -import org.apache.nifi.annotation.documentation.Tags; -import org.apache.nifi.annotation.lifecycle.OnScheduled; - -import org.apache.nifi.components.PropertyDescriptor; -import org.apache.nifi.flowfile.FlowFile; -import org.apache.nifi.logging.ComponentLog; -import org.apache.nifi.processor.exception.ProcessException; -import org.apache.nifi.processor.ProcessContext; -import org.apache.nifi.processor.ProcessSession; -import org.apache.nifi.processor.ProcessorInitializationContext; -import org.apache.nifi.processor.Relationship; -import org.apache.nifi.processor.io.InputStreamCallback; -import org.apache.nifi.processor.util.StandardValidators; -import org.apache.nifi.util.StringUtils; - -import org.apache.nifi.stream.io.StreamUtils; - -import java.io.ByteArrayOutputStream; -import java.io.IOException; -import java.io.InputStream; -import java.util.ArrayList; -import java.util.Collections; -import java.util.HashSet; -import java.util.List; -import java.util.Set; -import java.util.concurrent.atomic.AtomicReference; - - -@SideEffectFree -@SupportsBatching -@InputRequirement(InputRequirement.Requirement.INPUT_REQUIRED) -@Tags({"hashing", "fuzzy-hashing", "cyber-security"}) -@CapabilityDescription("Calculates a fuzzy/locality-sensitive hash value for the Content of a FlowFile and puts that " + - "hash value on the FlowFile as an attribute whose name is determined by the property." + - "Note: this processor only offers non-cryptographic hash algorithms. And it should be not be " + - "seen as a replacement to the HashContent processor." + - "Note: The underlying library loads the entirety of the streamed content into and performs result " + - "evaluations in memory. Accordingly, it is important to consider the anticipated profile of content being " + - "evaluated by this processor and the hardware supporting it especially when working against large files.") - -@SeeAlso(value = {CompareFuzzyHash.class}) -@WritesAttributes({@WritesAttribute(attribute = "", description = "This Processor adds an attribute whose value is the result of Hashing the " - + "existing FlowFile content. The name of this attribute is specified by the property")}) - -public class FuzzyHashContent extends AbstractFuzzyHashProcessor { - - - - public static final PropertyDescriptor HASH_ALGORITHM = new PropertyDescriptor.Builder() - .name("HASH_ALGORITHM") - .displayName("Hashing Algorithm") - .description("The hashing algorithm utilised") - .allowableValues(allowableValueSSDEEP, allowableValueTLSH) - .required(true) - .addValidator(StandardValidators.NON_EMPTY_VALIDATOR) - .build(); - - public static final Relationship REL_SUCCESS = new Relationship.Builder() - .name("success") - .description("Any FlowFile that is successfully hashed will be sent to this Relationship.") - .build(); - - public static final Relationship REL_FAILURE = new Relationship.Builder() - .name("failure") - .description("Any FlowFile that is successfully hashed will be sent to this Relationship.") - .build(); - - private List descriptors; - - private Set relationships; - - @Override - protected void init(final ProcessorInitializationContext context) { - final List descriptors = new ArrayList(); - descriptors.add(ATTRIBUTE_NAME); - descriptors.add(HASH_ALGORITHM); - this.descriptors = Collections.unmodifiableList(descriptors); - - final Set relationships = new HashSet(); - relationships.add(REL_SUCCESS); - relationships.add(REL_FAILURE); - this.relationships = Collections.unmodifiableSet(relationships); - } - - @Override - public Set getRelationships() { - return this.relationships; - } - - @Override - public final List getSupportedPropertyDescriptors() { - return descriptors; - } - - @OnScheduled - public void onScheduled(final ProcessContext context) { - } - - @Override - public void onTrigger(final ProcessContext context, final ProcessSession session) throws ProcessException { - FlowFile flowFile = session.get(); - if (flowFile == null) { - return; - } - - final ComponentLog logger = getLogger(); - String algorithm = context.getProperty(HASH_ALGORITHM).getValue(); - - // Check if content matches minimum length requirement - - if (checkMinimumAlgorithmRequirements(algorithm, flowFile) == false) { - logger.error("The content of '{}' is smaller than the minimum required by {}, routing to failure", - new Object[]{flowFile, algorithm}); - session.transfer(flowFile, REL_FAILURE); - return; - } - - final AtomicReference hashValueHolder = new AtomicReference<>(null); - - try { - session.read(flowFile, new InputStreamCallback() { - @Override - public void process(final InputStream in) throws IOException { - try (ByteArrayOutputStream holder = new ByteArrayOutputStream()) { - StreamUtils.copy(in,holder); - - String hashValue = generateHash(algorithm, holder.toString()); - if (StringUtils.isBlank(hashValue) == false) { - hashValueHolder.set(hashValue); - } - - - } - } - }); - - final String attributeName = context.getProperty(ATTRIBUTE_NAME).getValue(); - flowFile = session.putAttribute(flowFile, attributeName, hashValueHolder.get()); - logger.info("Successfully added attribute '{}' to {} with a value of {}; routing to success", new Object[]{attributeName, flowFile, hashValueHolder.get()}); - session.getProvenanceReporter().modifyAttributes(flowFile); - session.transfer(flowFile, REL_SUCCESS); - } catch (final InsufficientComplexityException | ProcessException e) { - logger.error("Failed to process {} due to {}; routing to failure", new Object[]{flowFile, e}); - session.transfer(flowFile, REL_FAILURE); - } - } - -} diff --git a/nifi-nar-bundles/nifi-cybersecurity-bundle/nifi-cybersecurity-processors/src/main/java/org/apache/nifi/processors/cybersecurity/matchers/FuzzyHashMatcher.java b/nifi-nar-bundles/nifi-cybersecurity-bundle/nifi-cybersecurity-processors/src/main/java/org/apache/nifi/processors/cybersecurity/matchers/FuzzyHashMatcher.java deleted file mode 100644 index 91dbd1237c..0000000000 --- a/nifi-nar-bundles/nifi-cybersecurity-bundle/nifi-cybersecurity-processors/src/main/java/org/apache/nifi/processors/cybersecurity/matchers/FuzzyHashMatcher.java +++ /dev/null @@ -1,35 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.nifi.processors.cybersecurity.matchers; - -import java.io.BufferedReader; -import java.io.IOException; - -public interface FuzzyHashMatcher { - - BufferedReader getReader(String source) throws IOException; - - boolean matchExceedsThreshold(double similarity, double matchThreshold) ; - - double getSimilarity(String inputHash, String existingHash); - - boolean isValidHash(String inputHash); - - String getHash(String line); - - String getMatch(String line); -} diff --git a/nifi-nar-bundles/nifi-cybersecurity-bundle/nifi-cybersecurity-processors/src/main/java/org/apache/nifi/processors/cybersecurity/matchers/SSDeepHashMatcher.java b/nifi-nar-bundles/nifi-cybersecurity-bundle/nifi-cybersecurity-processors/src/main/java/org/apache/nifi/processors/cybersecurity/matchers/SSDeepHashMatcher.java deleted file mode 100644 index 9371bfd527..0000000000 --- a/nifi-nar-bundles/nifi-cybersecurity-bundle/nifi-cybersecurity-processors/src/main/java/org/apache/nifi/processors/cybersecurity/matchers/SSDeepHashMatcher.java +++ /dev/null @@ -1,126 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.nifi.processors.cybersecurity.matchers; - -import info.debatty.java.spamsum.SpamSum; -import org.apache.nifi.logging.ComponentLog; - -import java.io.BufferedReader; -import java.io.File; -import java.io.FileInputStream; -import java.io.IOException; -import java.io.InputStreamReader; -import java.util.Scanner; - -public class SSDeepHashMatcher implements FuzzyHashMatcher { - - ComponentLog logger; - - public SSDeepHashMatcher() { - - } - - public SSDeepHashMatcher(ComponentLog logger) { - this.logger = logger; - } - - @Override - public BufferedReader getReader(String source) throws IOException { - - File file = new File(source); - - FileInputStream fileInputStream = new FileInputStream(file); - BufferedReader reader = new BufferedReader(new InputStreamReader(fileInputStream)); - - // If SSdeep skip the first line (as the usual format used by other tools add a header line - // to a file list - reader.readLine(); - - return reader; - } - - @Override - public boolean matchExceedsThreshold(double similarity, double matchThreshold) { - if (similarity >= matchThreshold) { - return true; - } else { - return false; - } - } - - @Override - public double getSimilarity(String inputHash, String existingHash) { - String[] hashToCompare = existingHash.split(",", 2); - if (hashToCompare.length > 0) { - return new SpamSum().match(inputHash, hashToCompare[0]); - } else { - return Double.NaN; - } - } - - @Override - public boolean isValidHash(String inputHash) { - // format looks like - // blocksize:hash:hash - - String [] fields = inputHash.split(":", 3); - - if (fields.length == 3) { - Scanner sc = new Scanner(fields[0]); - - boolean isNumber = sc.hasNextInt(); - if (isNumber == false && logger != null) { - if (logger.isDebugEnabled()) { - logger.debug("Field should be numeric but got '{}'. Will tell processor to ignore.", - new Object[] {fields[0]}); - } - } - - boolean hashOneIsNotEmpty = !fields[1].isEmpty(); - boolean hashTwoIsNotEmpty = !fields[2].isEmpty(); - - if (isNumber && hashOneIsNotEmpty && hashTwoIsNotEmpty) { - return true; - } - } - return false; - } - - @Override - public String getHash(String line) { - if (isValidHash(line)) { - return line.split(",", 2)[0]; - } else { - return null; - } - } - - @Override - public String getMatch(String line) { - if (isValidHash(line)) { - String[] match = line.split(",", 2); - // Because the file can be malformed and contain an unammed match, - // if match has a filename... - if (match.length == 2) { - // Return it. - return match[1]; - } - } - // Or return null - return null; - } -} diff --git a/nifi-nar-bundles/nifi-cybersecurity-bundle/nifi-cybersecurity-processors/src/main/java/org/apache/nifi/processors/cybersecurity/matchers/TLSHHashMatcher.java b/nifi-nar-bundles/nifi-cybersecurity-bundle/nifi-cybersecurity-processors/src/main/java/org/apache/nifi/processors/cybersecurity/matchers/TLSHHashMatcher.java deleted file mode 100644 index 73a140a5bb..0000000000 --- a/nifi-nar-bundles/nifi-cybersecurity-bundle/nifi-cybersecurity-processors/src/main/java/org/apache/nifi/processors/cybersecurity/matchers/TLSHHashMatcher.java +++ /dev/null @@ -1,117 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.nifi.processors.cybersecurity.matchers; - - -import com.idealista.tlsh.digests.Digest; -import com.idealista.tlsh.digests.DigestBuilder; -import org.apache.nifi.logging.ComponentLog; - -import java.io.BufferedReader; -import java.io.File; -import java.io.FileInputStream; -import java.io.IOException; -import java.io.InputStreamReader; - -import static org.apache.nifi.processors.cybersecurity.CompareFuzzyHash.HASH_LIST_FILE; - -public class TLSHHashMatcher implements FuzzyHashMatcher { - - ComponentLog logger; - - public TLSHHashMatcher(ComponentLog logger) { - this.logger = logger; - } - - @Override - public BufferedReader getReader(String source) throws IOException { - - File file = new File(source); - - FileInputStream fileInputStream = new FileInputStream(file); - BufferedReader reader = new BufferedReader(new InputStreamReader(fileInputStream)); - - return reader; - } - - @Override - public boolean matchExceedsThreshold(double similarity, double matchThreshold) { - if (similarity <= matchThreshold) { - return true; - } else { - return false; - } - } - - @Override - public double getSimilarity(String inputHash, String existingHash) { - String[] hashToCompare = existingHash.split("\t", 2); - // This will return null in case it fails validation - if (isValidHash(inputHash) && isValidHash(hashToCompare[0])) { - Digest inputDigest = new DigestBuilder().withHash(inputHash).build(); - Digest existingHashDigest = new DigestBuilder().withHash(hashToCompare[0]).build(); - - return inputDigest.calculateDifference(existingHashDigest, true); - } else { - return Double.NaN; - } - } - - @Override - public boolean isValidHash(String stringFromHashList) { - String[] hashToCompare = stringFromHashList.split("\t", 2); - // This will return null in case it fails validation - if (hashToCompare.length > 0) { - // Because DigestBuilder raises all sort of exceptions, so in order to keep the onTrigger loop a - // bit cleaner, we capture them here and return NaN to the loop above, otherwise simply return the - // similarity score. - try { - Digest digest = new DigestBuilder().withHash(hashToCompare[0]).build(); - return true; - } catch (ArrayIndexOutOfBoundsException | StringIndexOutOfBoundsException | NumberFormatException e) { - logger.error("Got {} while processing the string '{}'. This usually means the file " + - "defined by '{}' property contains invalid entries.", - new Object[]{e.getCause(), hashToCompare[0], HASH_LIST_FILE.getDisplayName()}); - } - } - return false; - } - - @Override - public String getHash(String line) { - if (isValidHash(line)) { - return line.split("\t", 2)[0]; - } else { - return null; - } - } - - @Override - public String getMatch(String line) { - if (isValidHash(line)) { - String[] match = line.split("\t", 2); - // Because the file can be malformed and contain an unammed match, - // if match has a filename... - if (match.length == 2) { - // Return it. - return match[1]; - } - } - // Or return null - return null; - } -} diff --git a/nifi-nar-bundles/nifi-cybersecurity-bundle/nifi-cybersecurity-processors/src/main/resources/META-INF/services/org.apache.nifi.processor.Processor b/nifi-nar-bundles/nifi-cybersecurity-bundle/nifi-cybersecurity-processors/src/main/resources/META-INF/services/org.apache.nifi.processor.Processor deleted file mode 100644 index e7cb5f7fca..0000000000 --- a/nifi-nar-bundles/nifi-cybersecurity-bundle/nifi-cybersecurity-processors/src/main/resources/META-INF/services/org.apache.nifi.processor.Processor +++ /dev/null @@ -1,16 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -org.apache.nifi.processors.cybersecurity.FuzzyHashContent -org.apache.nifi.processors.cybersecurity.CompareFuzzyHash \ No newline at end of file diff --git a/nifi-nar-bundles/nifi-cybersecurity-bundle/nifi-cybersecurity-processors/src/main/resources/docs/org/apache/nifi/processors/cybersecurity/CompareFuzzyHash/additionalDetails.html b/nifi-nar-bundles/nifi-cybersecurity-bundle/nifi-cybersecurity-processors/src/main/resources/docs/org/apache/nifi/processors/cybersecurity/CompareFuzzyHash/additionalDetails.html deleted file mode 100644 index ae2d5d3296..0000000000 --- a/nifi-nar-bundles/nifi-cybersecurity-bundle/nifi-cybersecurity-processors/src/main/resources/docs/org/apache/nifi/processors/cybersecurity/CompareFuzzyHash/additionalDetails.html +++ /dev/null @@ -1,45 +0,0 @@ - - - - - - CompareFuzzyHash - - - - - -

Description:

-

This Processor compares a an attribute containing a Fuzzy Hash (TLSH or SSDeep) value and compares it against a list - of hashes of the same family (i.e. TLSH is compared with a list of TLSH hashes), routing them to match or non-match - depending on a user configured threshold for similarity. -

- -

It is important to note that:

- -
    -
  • TLSH similarity increases as product of its comparison function decreases (i.e. 0 indicates nearly identical files)
    • -
  • SSDeep similarity directly relates to the product of its comparison function (e.g. 99 indicates nearly identical files
    • -
-

Based on the above, this processor when referring to "exceed the score" may be referring to: - -

    -
  • a value equal or lower than the configured threshold (in case of TLSH)
    • -
  • a value equal or higher than the configured threshold (in case of SSDeep)
    • -
- - \ No newline at end of file diff --git a/nifi-nar-bundles/nifi-cybersecurity-bundle/nifi-cybersecurity-processors/src/test/java/org/apache/nifi/processors/cybersecurity/TestCompareFuzzyHash.java b/nifi-nar-bundles/nifi-cybersecurity-bundle/nifi-cybersecurity-processors/src/test/java/org/apache/nifi/processors/cybersecurity/TestCompareFuzzyHash.java deleted file mode 100644 index 35e702a31a..0000000000 --- a/nifi-nar-bundles/nifi-cybersecurity-bundle/nifi-cybersecurity-processors/src/test/java/org/apache/nifi/processors/cybersecurity/TestCompareFuzzyHash.java +++ /dev/null @@ -1,381 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.nifi.processors.cybersecurity; - - -import org.apache.nifi.processors.cybersecurity.matchers.FuzzyHashMatcher; -import org.apache.nifi.processors.cybersecurity.matchers.SSDeepHashMatcher; -import org.apache.nifi.util.MockFlowFile; -import org.apache.nifi.util.TestRunner; -import org.apache.nifi.util.TestRunners; -import org.junit.jupiter.api.AfterEach; -import org.junit.jupiter.api.Test; - -import java.util.Arrays; -import java.util.HashMap; -import java.util.List; -import java.util.Map; - -import static org.junit.jupiter.api.Assertions.assertTrue; - - -public class TestCompareFuzzyHash { - String ssdeepInput = "48:c1xs8Z/m6H0eRH31S8p8bHENANkPrNy4tkPytwPyh2jTytxPythPytNdPytDgYyF:OuO/mg3HFSRHEb44RNMi6uHU2hcq3"; - String tlshInput = "EB519EA4A8F95171A2A409C1DEEB9872AF55C137E00A5289F1CCD0CE4F6CCD784BB4B7"; - - final CompareFuzzyHash proc = new CompareFuzzyHash(); - final private TestRunner runner = TestRunners.newTestRunner(proc); - - @AfterEach - public void stop() { - runner.shutdown(); - } - - @Test - public void testSsdeepCompareFuzzyHash() { - double matchingSimilarity = 80; - runner.setProperty(CompareFuzzyHash.HASH_ALGORITHM, CompareFuzzyHash.allowableValueSSDEEP.getValue()); - runner.setProperty(CompareFuzzyHash.ATTRIBUTE_NAME, "fuzzyhash.value"); - runner.setProperty(CompareFuzzyHash.HASH_LIST_FILE, "src/test/resources/ssdeep.list"); - runner.setProperty(CompareFuzzyHash.MATCH_THRESHOLD, String.valueOf(matchingSimilarity)); - runner.setProperty(CompareFuzzyHash.MATCHING_MODE, CompareFuzzyHash.singleMatch.getValue()); - - Map attributes = new HashMap<>(); - attributes.put("fuzzyhash.value", ssdeepInput); - - runner.enqueue("bogus".getBytes(), attributes); - runner.run(); - - runner.assertQueueEmpty(); - runner.assertAllFlowFilesTransferred(CompareFuzzyHash.REL_FOUND, 1); - - final MockFlowFile outFile = runner.getFlowFilesForRelationship(CompareFuzzyHash.REL_FOUND).get(0); - - - outFile.assertAttributeEquals( - "fuzzyhash.value.0.match", - "\"nifi/nifi-nar-bundles/nifi-beats-bundle/nifi-beats-processors/pom.xml\"" - ); - double similarity = Double.valueOf(outFile.getAttribute("fuzzyhash.value.0.similarity")); - assertTrue(similarity >= matchingSimilarity); - - outFile.assertAttributeNotExists("fuzzyhash.value.1.match"); - } - - @Test - public void testSsdeepCompareFuzzyHashMultipleMatches() { - double matchingSimilarity = 80; - runner.setProperty(CompareFuzzyHash.HASH_ALGORITHM, CompareFuzzyHash.allowableValueSSDEEP.getValue()); - runner.setProperty(CompareFuzzyHash.ATTRIBUTE_NAME, "fuzzyhash.value"); - runner.setProperty(CompareFuzzyHash.HASH_LIST_FILE, "src/test/resources/ssdeep.list"); - runner.setProperty(CompareFuzzyHash.MATCH_THRESHOLD, String.valueOf(matchingSimilarity)); - runner.setProperty(CompareFuzzyHash.MATCHING_MODE, CompareFuzzyHash.multiMatch.getValue()); - - Map attributes = new HashMap<>(); - attributes.put("fuzzyhash.value", ssdeepInput ); - - runner.enqueue("bogus".getBytes(), attributes); - runner.run(); - - runner.assertQueueEmpty(); - runner.assertAllFlowFilesTransferred(CompareFuzzyHash.REL_FOUND, 1); - - final MockFlowFile outFile = runner.getFlowFilesForRelationship(CompareFuzzyHash.REL_FOUND).get(0); - - - outFile.assertAttributeEquals("fuzzyhash.value.0.match", - "\"nifi/nifi-nar-bundles/nifi-beats-bundle/nifi-beats-processors/pom.xml\"" - ); - - double similarity = Double.valueOf(outFile.getAttribute("fuzzyhash.value.0.similarity")); - assertTrue(similarity >= matchingSimilarity); - - outFile.assertAttributeEquals("fuzzyhash.value.1.match", - "\"nifi/nifi-nar-bundles/nifi-lumberjack-bundle/nifi-lumberjack-processors/pom.xml\"" - ); - similarity = Double.valueOf(outFile.getAttribute("fuzzyhash.value.1.similarity")); - assertTrue(similarity >= matchingSimilarity); - } - - @Test - public void testSsdeepCompareFuzzyHashWithBlankHashList() { - double matchingSimilarity = 80; - runner.setProperty(CompareFuzzyHash.HASH_ALGORITHM, CompareFuzzyHash.allowableValueSSDEEP.getValue()); - runner.setProperty(CompareFuzzyHash.ATTRIBUTE_NAME, "fuzzyhash.value"); - runner.setProperty(CompareFuzzyHash.HASH_LIST_FILE, "src/test/resources/blank_ssdeep.list"); - runner.setProperty(CompareFuzzyHash.MATCH_THRESHOLD, String.valueOf(matchingSimilarity)); - - Map attributes = new HashMap<>(); - attributes.put("fuzzyhash.value", "6:hERjIfhRrlB63J0FDw1NBQmEH68xwMSELN:hZrlB62IwMS"); - - runner.enqueue("bogus".getBytes(), attributes); - runner.run(); - - runner.assertQueueEmpty(); - runner.assertAllFlowFilesTransferred(CompareFuzzyHash.REL_NOT_FOUND, 1); - - final MockFlowFile outFile = runner.getFlowFilesForRelationship(CompareFuzzyHash.REL_NOT_FOUND).get(0); - } - - @Test - public void testSsdeepCompareFuzzyHashWithInvalidHashList() { - // This is different from "BlankHashList series of tests in that the file lacks headers and as such is totally - // invalid - double matchingSimilarity = 80; - runner.setProperty(CompareFuzzyHash.HASH_ALGORITHM, CompareFuzzyHash.allowableValueSSDEEP.getValue()); - runner.setProperty(CompareFuzzyHash.ATTRIBUTE_NAME, "fuzzyhash.value"); - runner.setProperty(CompareFuzzyHash.HASH_LIST_FILE, "src/test/resources/empty.list"); - runner.setProperty(CompareFuzzyHash.MATCH_THRESHOLD, String.valueOf(matchingSimilarity)); - - Map attributes = new HashMap<>(); - attributes.put("fuzzyhash.value", "6:hERjIfhRrlB63J0FDw1NBQmEH68xwMSELN:hZrlB62IwMS"); - - runner.enqueue("bogus".getBytes(), attributes); - runner.run(); - - runner.assertQueueEmpty(); - runner.assertAllFlowFilesTransferred(CompareFuzzyHash.REL_NOT_FOUND, 1); - - final MockFlowFile outFile = runner.getFlowFilesForRelationship(CompareFuzzyHash.REL_NOT_FOUND).get(0); - - outFile.assertAttributeNotExists("fuzzyhash.value.0.match"); - } - - @Test - public void testSsdeepCompareFuzzyHashWithInvalidHash() { - double matchingSimilarity = 80; - runner.setProperty(CompareFuzzyHash.HASH_ALGORITHM, CompareFuzzyHash.allowableValueSSDEEP.getValue()); - runner.setProperty(CompareFuzzyHash.ATTRIBUTE_NAME, "fuzzyhash.value"); - runner.setProperty(CompareFuzzyHash.HASH_LIST_FILE, "src/test/resources/ssdeep.list"); - runner.setProperty(CompareFuzzyHash.MATCH_THRESHOLD, String.valueOf(matchingSimilarity)); - runner.setProperty(CompareFuzzyHash.MATCHING_MODE, CompareFuzzyHash.singleMatch.getValue()); - - Map attributes = new HashMap<>(); - attributes.put("fuzzyhash.value", "Test test test chocolate!"); - - runner.enqueue("bogus".getBytes(), attributes); - runner.run(); - - runner.assertQueueEmpty(); - runner.assertAllFlowFilesTransferred(CompareFuzzyHash.REL_FAILURE, 1); - - final MockFlowFile outFile = runner.getFlowFilesForRelationship(CompareFuzzyHash.REL_FAILURE).get(0); - - outFile.assertAttributeNotExists("fuzzyhash.value.0.match"); - } - - - @Test - public void testTLSHCompareFuzzyHash() { - double matchingSimilarity = 200; - runner.setProperty(CompareFuzzyHash.HASH_ALGORITHM, CompareFuzzyHash.allowableValueTLSH.getValue()); - runner.setProperty(CompareFuzzyHash.ATTRIBUTE_NAME, "fuzzyhash.value"); - runner.setProperty(CompareFuzzyHash.HASH_LIST_FILE, "src/test/resources/tlsh.list"); - runner.setProperty(CompareFuzzyHash.MATCH_THRESHOLD, String.valueOf(matchingSimilarity)); - runner.setProperty(CompareFuzzyHash.MATCHING_MODE, CompareFuzzyHash.singleMatch.getValue()); - - Map attributes = new HashMap<>(); - attributes.put("fuzzyhash.value", tlshInput); - - runner.enqueue("bogus".getBytes(), attributes); - runner.run(); - - runner.assertQueueEmpty(); - runner.assertAllFlowFilesTransferred(CompareFuzzyHash.REL_FOUND, 1); - - final MockFlowFile outFile = runner.getFlowFilesForRelationship(CompareFuzzyHash.REL_FOUND).get(0); - - outFile.assertAttributeEquals( - "fuzzyhash.value.0.match", - "nifi-nar-bundles/nifi-lumberjack-bundle/nifi-lumberjack-processors/pom.xml" - ); - double similarity = Double.valueOf(outFile.getAttribute("fuzzyhash.value.0.similarity")); - assertTrue(similarity <= matchingSimilarity); - - outFile.assertAttributeNotExists("fuzzyhash.value.1.match"); - } - - @Test - public void testTLSHCompareFuzzyHashMultipleMatches() { - double matchingSimilarity = 200; - runner.setProperty(CompareFuzzyHash.HASH_ALGORITHM, CompareFuzzyHash.allowableValueTLSH.getValue()); - runner.setProperty(CompareFuzzyHash.ATTRIBUTE_NAME, "fuzzyhash.value"); - runner.setProperty(CompareFuzzyHash.HASH_LIST_FILE, "src/test/resources/tlsh.list"); - runner.setProperty(CompareFuzzyHash.MATCH_THRESHOLD, String.valueOf(matchingSimilarity)); - runner.setProperty(CompareFuzzyHash.MATCHING_MODE, CompareFuzzyHash.multiMatch.getValue()); - - Map attributes = new HashMap<>(); - attributes.put("fuzzyhash.value", tlshInput); - - runner.enqueue("bogus".getBytes(), attributes); - runner.run(); - - runner.assertQueueEmpty(); - runner.assertAllFlowFilesTransferred(CompareFuzzyHash.REL_FOUND, 1); - - final MockFlowFile outFile = runner.getFlowFilesForRelationship(CompareFuzzyHash.REL_FOUND).get(0); - - outFile.assertAttributeEquals( - "fuzzyhash.value.0.match", - "nifi-nar-bundles/nifi-lumberjack-bundle/nifi-lumberjack-processors/pom.xml" - ); - double similarity = Double.valueOf(outFile.getAttribute("fuzzyhash.value.0.similarity")); - assertTrue(similarity <= matchingSimilarity); - - outFile.assertAttributeEquals( - "fuzzyhash.value.1.match", - "nifi-nar-bundles/nifi-beats-bundle/nifi-beats-processors/pom.xml" - ); - similarity = Double.valueOf(outFile.getAttribute("fuzzyhash.value.1.similarity")); - assertTrue(similarity <= matchingSimilarity); - } - - - @Test - public void testTLSHCompareFuzzyHashWithBlankFile() { - // This is different from "BlankHashList series of tests in that the file lacks headers and as such is totally - // invalid - double matchingSimilarity = 200; - runner.setProperty(CompareFuzzyHash.HASH_ALGORITHM, CompareFuzzyHash.allowableValueTLSH.getValue()); - runner.setProperty(CompareFuzzyHash.ATTRIBUTE_NAME, "fuzzyhash.value"); - runner.setProperty(CompareFuzzyHash.HASH_LIST_FILE, "src/test/resources/empty.list"); - runner.setProperty(CompareFuzzyHash.MATCH_THRESHOLD, String.valueOf(matchingSimilarity)); - - Map attributes = new HashMap<>(); - attributes.put("fuzzyhash.value", "E2F0818B7AE7173906A72221570E30979B11C0FC47B518A1E89D257E2343CEC02381ED"); - - runner.enqueue("bogus".getBytes(), attributes); - runner.run(); - - runner.assertQueueEmpty(); - runner.assertAllFlowFilesTransferred(CompareFuzzyHash.REL_NOT_FOUND, 1); - - final MockFlowFile outFile = runner.getFlowFilesForRelationship(CompareFuzzyHash.REL_NOT_FOUND).get(0); - - outFile.assertAttributeNotExists("fuzzyhash.value.0.match"); - } - - @Test - public void testTLSHCompareFuzzyHashWithEmptyHashList() { - double matchingSimilarity = 200; - runner.setProperty(CompareFuzzyHash.HASH_ALGORITHM, CompareFuzzyHash.allowableValueTLSH.getValue()); - runner.setProperty(CompareFuzzyHash.ATTRIBUTE_NAME, "fuzzyhash.value"); - runner.setProperty(CompareFuzzyHash.HASH_LIST_FILE, "src/test/resources/empty.list"); - runner.setProperty(CompareFuzzyHash.MATCH_THRESHOLD, String.valueOf(matchingSimilarity)); - - Map attributes = new HashMap<>(); - attributes.put("fuzzyhash.value", "E2F0818B7AE7173906A72221570E30979B11C0FC47B518A1E89D257E2343CEC02381ED"); - - runner.enqueue("bogus".getBytes(), attributes); - runner.run(); - - runner.assertQueueEmpty(); - runner.assertAllFlowFilesTransferred(CompareFuzzyHash.REL_NOT_FOUND, 1); - - final MockFlowFile outFile = runner.getFlowFilesForRelationship(CompareFuzzyHash.REL_NOT_FOUND).get(0); - - outFile.assertAttributeNotExists("fuzzyhash.value.0.match"); - } - - @Test - public void testTLSHCompareFuzzyHashWithInvalidHash() { - double matchingSimilarity = 200; - runner.setProperty(CompareFuzzyHash.HASH_ALGORITHM, CompareFuzzyHash.allowableValueTLSH.getValue()); - runner.setProperty(CompareFuzzyHash.ATTRIBUTE_NAME, "fuzzyhash.value"); - runner.setProperty(CompareFuzzyHash.HASH_LIST_FILE, "src/test/resources/empty.list"); - runner.setProperty(CompareFuzzyHash.MATCH_THRESHOLD, String.valueOf(matchingSimilarity)); - - Map attributes = new HashMap<>(); - attributes.put("fuzzyhash.value", "Test test test chocolate"); - - runner.enqueue("bogus".getBytes(), attributes); - runner.run(); - - runner.assertQueueEmpty(); - runner.assertAllFlowFilesTransferred(CompareFuzzyHash.REL_FAILURE, 1); - - final MockFlowFile outFile = runner.getFlowFilesForRelationship(CompareFuzzyHash.REL_FAILURE).get(0); - - outFile.assertAttributeNotExists("fuzzyhash.value.0.match"); - - } - - @Test - public void testMissingAttribute() { - double matchingSimilarity = 200; - runner.setProperty(CompareFuzzyHash.HASH_ALGORITHM, CompareFuzzyHash.allowableValueTLSH.getValue()); - runner.setProperty(CompareFuzzyHash.ATTRIBUTE_NAME, "fuzzyhash.value"); - runner.setProperty(CompareFuzzyHash.HASH_LIST_FILE, "src/test/resources/empty.list"); - runner.setProperty(CompareFuzzyHash.MATCH_THRESHOLD, String.valueOf(matchingSimilarity)); - runner.setProperty(CompareFuzzyHash.MATCHING_MODE, CompareFuzzyHash.multiMatch.getValue()); - - runner.enqueue("bogus".getBytes()); - runner.run(); - - runner.assertQueueEmpty(); - runner.assertAllFlowFilesTransferred(CompareFuzzyHash.REL_FAILURE, 1); - - final MockFlowFile outFile = runner.getFlowFilesForRelationship(CompareFuzzyHash.REL_FAILURE).get(0); - - outFile.assertAttributeNotExists("fuzzyhash.value.0.match"); - } - - @Test - public void testAttributeIsEmptyString() { - double matchingSimilarity = 200; - runner.setProperty(CompareFuzzyHash.HASH_ALGORITHM, CompareFuzzyHash.allowableValueTLSH.getValue()); - runner.setProperty(CompareFuzzyHash.ATTRIBUTE_NAME, "fuzzyhash.value"); - runner.setProperty(CompareFuzzyHash.HASH_LIST_FILE, "src/test/resources/empty.list"); - runner.setProperty(CompareFuzzyHash.MATCH_THRESHOLD, String.valueOf(matchingSimilarity)); - runner.setProperty(CompareFuzzyHash.MATCHING_MODE, CompareFuzzyHash.multiMatch.getValue()); - - Map attributes = new HashMap<>(); - attributes.put("fuzzyhash.value", ""); - runner.enqueue("bogus".getBytes(), attributes); - - runner.run(); - - runner.assertQueueEmpty(); - runner.assertAllFlowFilesTransferred(CompareFuzzyHash.REL_FAILURE, 1); - - final MockFlowFile outFile = runner.getFlowFilesForRelationship(CompareFuzzyHash.REL_FAILURE).get(0); - - outFile.assertAttributeNotExists("fuzzyhash.value.0.match"); - } - - @Test - public void testlooksLikeSpamSum() { - FuzzyHashMatcher matcher = new SSDeepHashMatcher(); - - List invalidPayloads = Arrays.asList( - "4AD:c1xs8Z/m6H0eRH31S8p8bHENANkPrNy4tkPytwPyh2jTytxPythPytNdPytDgYyF:OuO/mg3HFSRHEb44RNMi6uHU2hcq3", // invalidFirstField - ":c1xs8Z/m6H0eRH31S8p8bHENANkPrNy4tkPytwPyh2jTytxPythPytNdPytDgYyF:OuO/mg3HFSRHEb44RNMi6uHU2hcq3", // emptyFirstField - "48::OuO/mg3HFSRHEb44RNMi6uHU2hcq3", // emptySecondField - "48:c1xs8Z/m6H0eRH31S8p8bHENANkPrNy4tkPytwPyh2jTytxPythPytNdPytDgYyF:", // emptyThirdField - "48:c1xs8Z/m6H0eRH31S8p8bHENANkPrNy4tkPytwPyh2jTytxPythPytNdPytDgYyF", // withoutThirdField - "c1xs8Z/m6H0eRH31S8p8bHENANkPrNy4tkPytwPyh2jTytxPythPytNdPytDgYyF" // Just a simple string - ); - - for (String item : invalidPayloads) { - assertTrue(!matcher.isValidHash(item), "item '" + item + "' should have failed validation"); - } - - // Now test with a valid string - assertTrue(matcher.isValidHash(ssdeepInput)); - - } -} diff --git a/nifi-nar-bundles/nifi-cybersecurity-bundle/nifi-cybersecurity-processors/src/test/java/org/apache/nifi/processors/cybersecurity/TestFuzzyHashContent.java b/nifi-nar-bundles/nifi-cybersecurity-bundle/nifi-cybersecurity-processors/src/test/java/org/apache/nifi/processors/cybersecurity/TestFuzzyHashContent.java deleted file mode 100644 index a3085d518e..0000000000 --- a/nifi-nar-bundles/nifi-cybersecurity-bundle/nifi-cybersecurity-processors/src/test/java/org/apache/nifi/processors/cybersecurity/TestFuzzyHashContent.java +++ /dev/null @@ -1,97 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.nifi.processors.cybersecurity; - - -import org.apache.nifi.util.MockFlowFile; -import org.apache.nifi.util.TestRunner; -import org.apache.nifi.util.TestRunners; -import org.junit.jupiter.api.AfterEach; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; - -import static org.junit.jupiter.api.Assertions.assertEquals; - - -public class TestFuzzyHashContent { - - private TestRunner runner; - - @BeforeEach - public void init() { - runner = TestRunners.newTestRunner(FuzzyHashContent.class); - } - - @AfterEach - public void stop() { - runner.shutdown(); - } - - @Test - public void testSsdeepFuzzyHashContent() { - runner.setProperty(FuzzyHashContent.HASH_ALGORITHM, FuzzyHashContent.allowableValueSSDEEP.getValue()); - runner.enqueue("This is the a short and meaningless sample taste of 'test test test chocolate' " + - "an odd test string that is used within some of the NiFi test units. Once day the author of " + - "such strings may decide to tell the history behind this sentence and its true meaning...\n"); - runner.run(); - - runner.assertQueueEmpty(); - runner.assertAllFlowFilesTransferred(FuzzyHashContent.REL_SUCCESS, 1); - - final MockFlowFile outFile = runner.getFlowFilesForRelationship(FuzzyHashContent.REL_SUCCESS).get(0); - - assertEquals("6:hERjIfhRrlB63J0FDw1NBQmEH68xwMSELN:hZrlB62IwMS",outFile.getAttribute("fuzzyhash.value") ); - } - - @Test - public void testTLSHFuzzyHashInvalidContent() { - runner.setProperty(FuzzyHashContent.HASH_ALGORITHM, FuzzyHashContent.allowableValueTLSH.getValue()); - runner.enqueue("This is the a short and meaningless sample taste of 'test test test chocolate' " + - "an odd test string that is used within some of the NiFi test units. Once day the author of " + - "such strings may decide to tell the history behind this sentence and its true meaning...\n"); - runner.run(); - - runner.assertQueueEmpty(); - runner.assertAllFlowFilesTransferred(FuzzyHashContent.REL_FAILURE, 1); - - final MockFlowFile outFile = runner.getFlowFilesForRelationship(FuzzyHashContent.REL_FAILURE).get(0); - - outFile.assertAttributeNotExists("fuzzyhash.value"); - } - - @Test - public void testTLSHFuzzyHashValidContent() { - runner.setProperty(FuzzyHashContent.HASH_ALGORITHM, FuzzyHashContent.allowableValueTLSH.getValue()); - runner.enqueue("This is the a short and meaningless sample taste of 'test test test chocolate' " + - "an odd test string that is used within some of the NiFi test units. Once day the author of " + - "such strings may decide to tell the history behind this sentence and its true meaning...\n" + - "What is certain however, is that this section of the test unit requires at least 512 " + - "characters of data to produce the expect results.\n " + - "And yet... despite all the senseless verbosity, I still have to continue writing these somewhat " + - "meaningless words that do nothing but to remind all of us of Ipsum Lorem..." ); - runner.run(); - - runner.assertQueueEmpty(); - runner.assertAllFlowFilesTransferred(FuzzyHashContent.REL_SUCCESS, 1); - - final MockFlowFile outFile = runner.getFlowFilesForRelationship(FuzzyHashContent.REL_SUCCESS).get(0); - - assertEquals("E2F0818B7AE7173906A72221570E30979B11C0FC47B518A1E89D257E2343CEC02381ED", - outFile.getAttribute("fuzzyhash.value")); - } - -} diff --git a/nifi-nar-bundles/nifi-cybersecurity-bundle/nifi-cybersecurity-processors/src/test/resources/blank_ssdeep.list b/nifi-nar-bundles/nifi-cybersecurity-bundle/nifi-cybersecurity-processors/src/test/resources/blank_ssdeep.list deleted file mode 100644 index 0393719323..0000000000 --- a/nifi-nar-bundles/nifi-cybersecurity-bundle/nifi-cybersecurity-processors/src/test/resources/blank_ssdeep.list +++ /dev/null @@ -1 +0,0 @@ -ssdeep,1.0--blocksize:hash:hash,filename \ No newline at end of file diff --git a/nifi-nar-bundles/nifi-cybersecurity-bundle/nifi-cybersecurity-processors/src/test/resources/empty.list b/nifi-nar-bundles/nifi-cybersecurity-bundle/nifi-cybersecurity-processors/src/test/resources/empty.list deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/nifi-nar-bundles/nifi-cybersecurity-bundle/nifi-cybersecurity-processors/src/test/resources/ssdeep.list b/nifi-nar-bundles/nifi-cybersecurity-bundle/nifi-cybersecurity-processors/src/test/resources/ssdeep.list deleted file mode 100644 index 05b505e22f..0000000000 --- a/nifi-nar-bundles/nifi-cybersecurity-bundle/nifi-cybersecurity-processors/src/test/resources/ssdeep.list +++ /dev/null @@ -1,11 +0,0 @@ -ssdeep,1.0--blocksize:hash:hash,filename -96:KQhaGCVZGhr83h3bc0ok3892m12wzgnH5w2pw+sxNEI58:FIVkH4x73h39LH+2w+sxaD,"config.h" -96:EQOJvOl4ab3hhiNFXc4wwcweomr0cNJDBoqXjmAHKX8dEt001nfEhVIuX0dDcs:3mzpAsZpprbshfu3oujjdENdp21,"doc\README" -48:c1xs8Z/m6H0eRH31S8p8bHENRNkPSNy4tkPytwPytyYytxPythPytNdPytDgYyse:OuO/mg3HFSRHE+H4RNc6uHU2hqoMkh -96,MD9fHjsEuddrg31904l8bgx5ROg2MQZHZqpAlycowOsexbHDbk:MJwz/l2PqGqqbr2yk6pVgrwPV,"Whatever.txt-INVALID-DUE-TO-COMMA-AFTER-96" -48:c1xs8Z/m6H0eRH31S8p8bHENANkPrNy4tkPytwPyh2jTytxPythPytNdPytDgYyF:OuO/mg3HFSRHEb44RNMi6uHU2hcq3,"nifi/nifi-nar-bundles/nifi-beats-bundle/nifi-beats-processors/pom.xml" -6:hERjIfhRrlB63J0FDw1NBQmEH68xwMSELN:hZrlB62IwMS,"c:\this_is_valid_but_should_not_match" -96:MD9fHjsEuddrg31904l8bgx5ROg2MQZHZqpAlycowOsexbHDbk:MJwz/l2PqGqqbr2yk6pVgrwPV,"INSTALL" -48:c1xs8Z/m6H0eRH31S8p8bHENRNkPSNy4tkPytwPytyYytxPythPytNdPytDgYyse:OuO/mg3HFSRHE+H4RNc6uHU2hqoMkh,"nifi/nifi-nar-bundles/nifi-lumberjack-bundle/nifi-lumberjack-processors/pom.xml" -48:c1xs8Z/m6H0eRH31S8p8bHENRNkPSNy4tkPytwPytyYytxPythPytNdPytDgYyse:OuO/mg3HFSRHE+H4RNc6uHU2hqoMkh, - diff --git a/nifi-nar-bundles/nifi-cybersecurity-bundle/nifi-cybersecurity-processors/src/test/resources/tlsh.list b/nifi-nar-bundles/nifi-cybersecurity-bundle/nifi-cybersecurity-processors/src/test/resources/tlsh.list deleted file mode 100644 index 0f31c8ff63..0000000000 --- a/nifi-nar-bundles/nifi-cybersecurity-bundle/nifi-cybersecurity-processors/src/test/resources/tlsh.list +++ /dev/null @@ -1,9 +0,0 @@ -A4518DA4A8F9517162A409C1DEEA9872AF55C137E00A62C9F0CDD0CE4F6CCD784BB4B7 nifi-nar-bundles/nifi-lumberjack-bundle/nifi-lumberjack-processors/pom.xml - THERE SEEMS TO BE SOMETHING MISSING -6FF02BEF718027B0160B4391212923ED7F1A463D563B1549B86CF62973B197AD2731F Synthetic shorter-INVALID -EB519EA4A8F95171A2A409C1DEEB9872AF55C137E00A5289F1CCD0CE4F6CCD784BB4B7 -E2F0818B7AE7173906A72221570E30979B11C0FC47B518A1E89D257E2343CEC02381ED /this/is/also/valid/but/should/not/match -EB519EA4A8F95171A2A409C1DEEB9872AF55C137E00A5289F1CCD0CE4F6CCD784BB4B7 nifi-nar-bundles/nifi-beats-bundle/nifi-beats-processors/pom.xml -EB519EA4A8F95171A2A409C1DEEB9872AF55C137E00A5289F1CCD0CE4F6CCD784BB4B7 - -6FF02BEF718027B0160B4391212923ED7F1A463D563B1549B86CF62973B197AD2731F8 /this/is/valid/but/should/not/match diff --git a/nifi-nar-bundles/nifi-cybersecurity-bundle/pom.xml b/nifi-nar-bundles/nifi-cybersecurity-bundle/pom.xml deleted file mode 100644 index 66f00cf7e8..0000000000 --- a/nifi-nar-bundles/nifi-cybersecurity-bundle/pom.xml +++ /dev/null @@ -1,32 +0,0 @@ - - - - 4.0.0 - - - org.apache.nifi - nifi-nar-bundles - 2.0.0-SNAPSHOT - - - nifi-cybersecurity-bundle - pom - - - nifi-cybersecurity-processors - nifi-cybersecurity-nar - - diff --git a/nifi-nar-bundles/nifi-html-bundle/nifi-html-nar/pom.xml b/nifi-nar-bundles/nifi-html-bundle/nifi-html-nar/pom.xml deleted file mode 100644 index 1663817310..0000000000 --- a/nifi-nar-bundles/nifi-html-bundle/nifi-html-nar/pom.xml +++ /dev/null @@ -1,44 +0,0 @@ - - - - 4.0.0 - - - org.apache.nifi - nifi-html-bundle - 2.0.0-SNAPSHOT - - - nifi-html-nar - nar - - true - true - - - - - org.apache.nifi - nifi-standard-services-api-nar - 2.0.0-SNAPSHOT - nar - - - org.apache.nifi - nifi-html-processors - - - diff --git a/nifi-nar-bundles/nifi-html-bundle/nifi-html-nar/src/main/resources/META-INF/LICENSE b/nifi-nar-bundles/nifi-html-bundle/nifi-html-nar/src/main/resources/META-INF/LICENSE deleted file mode 100644 index c62123ee47..0000000000 --- a/nifi-nar-bundles/nifi-html-bundle/nifi-html-nar/src/main/resources/META-INF/LICENSE +++ /dev/null @@ -1,240 +0,0 @@ - - Apache License - Version 2.0, January 2004 - http://www.apache.org/licenses/ - - TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION - - 1. Definitions. - - "License" shall mean the terms and conditions for use, reproduction, - and distribution as defined by Sections 1 through 9 of this document. - - "Licensor" shall mean the copyright owner or entity authorized by - the copyright owner that is granting the License. - - "Legal Entity" shall mean the union of the acting entity and all - other entities that control, are controlled by, or are under common - control with that entity. For the purposes of this definition, - "control" means (i) the power, direct or indirect, to cause the - direction or management of such entity, whether by contract or - otherwise, or (ii) ownership of fifty percent (50%) or more of the - outstanding shares, or (iii) beneficial ownership of such entity. - - "You" (or "Your") shall mean an individual or Legal Entity - exercising permissions granted by this License. - - "Source" form shall mean the preferred form for making modifications, - including but not limited to software source code, documentation - source, and configuration files. - - "Object" form shall mean any form resulting from mechanical - transformation or translation of a Source form, including but - not limited to compiled object code, generated documentation, - and conversions to other media types. - - "Work" shall mean the work of authorship, whether in Source or - Object form, made available under the License, as indicated by a - copyright notice that is included in or attached to the work - (an example is provided in the Appendix below). - - "Derivative Works" shall mean any work, whether in Source or Object - form, that is based on (or derived from) the Work and for which the - editorial revisions, annotations, elaborations, or other modifications - represent, as a whole, an original work of authorship. For the purposes - of this License, Derivative Works shall not include works that remain - separable from, or merely link (or bind by name) to the interfaces of, - the Work and Derivative Works thereof. - - "Contribution" shall mean any work of authorship, including - the original version of the Work and any modifications or additions - to that Work or Derivative Works thereof, that is intentionally - submitted to Licensor for inclusion in the Work by the copyright owner - or by an individual or Legal Entity authorized to submit on behalf of - the copyright owner. For the purposes of this definition, "submitted" - means any form of electronic, verbal, or written communication sent - to the Licensor or its representatives, including but not limited to - communication on electronic mailing lists, source code control systems, - and issue tracking systems that are managed by, or on behalf of, the - Licensor for the purpose of discussing and improving the Work, but - excluding communication that is conspicuously marked or otherwise - designated in writing by the copyright owner as "Not a Contribution." - - "Contributor" shall mean Licensor and any individual or Legal Entity - on behalf of whom a Contribution has been received by Licensor and - subsequently incorporated within the Work. - - 2. Grant of Copyright License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - copyright license to reproduce, prepare Derivative Works of, - publicly display, publicly perform, sublicense, and distribute the - Work and such Derivative Works in Source or Object form. - - 3. Grant of Patent License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - (except as stated in this section) patent license to make, have made, - use, offer to sell, sell, import, and otherwise transfer the Work, - where such license applies only to those patent claims licensable - by such Contributor that are necessarily infringed by their - Contribution(s) alone or by combination of their Contribution(s) - with the Work to which such Contribution(s) was submitted. If You - institute patent litigation against any entity (including a - cross-claim or counterclaim in a lawsuit) alleging that the Work - or a Contribution incorporated within the Work constitutes direct - or contributory patent infringement, then any patent licenses - granted to You under this License for that Work shall terminate - as of the date such litigation is filed. - - 4. Redistribution. You may reproduce and distribute copies of the - Work or Derivative Works thereof in any medium, with or without - modifications, and in Source or Object form, provided that You - meet the following conditions: - - (a) You must give any other recipients of the Work or - Derivative Works a copy of this License; and - - (b) You must cause any modified files to carry prominent notices - stating that You changed the files; and - - (c) You must retain, in the Source form of any Derivative Works - that You distribute, all copyright, patent, trademark, and - attribution notices from the Source form of the Work, - excluding those notices that do not pertain to any part of - the Derivative Works; and - - (d) If the Work includes a "NOTICE" text file as part of its - distribution, then any Derivative Works that You distribute must - include a readable copy of the attribution notices contained - within such NOTICE file, excluding those notices that do not - pertain to any part of the Derivative Works, in at least one - of the following places: within a NOTICE text file distributed - as part of the Derivative Works; within the Source form or - documentation, if provided along with the Derivative Works; or, - within a display generated by the Derivative Works, if and - wherever such third-party notices normally appear. The contents - of the NOTICE file are for informational purposes only and - do not modify the License. You may add Your own attribution - notices within Derivative Works that You distribute, alongside - or as an addendum to the NOTICE text from the Work, provided - that such additional attribution notices cannot be construed - as modifying the License. - - You may add Your own copyright statement to Your modifications and - may provide additional or different license terms and conditions - for use, reproduction, or distribution of Your modifications, or - for any such Derivative Works as a whole, provided Your use, - reproduction, and distribution of the Work otherwise complies with - the conditions stated in this License. - - 5. Submission of Contributions. Unless You explicitly state otherwise, - any Contribution intentionally submitted for inclusion in the Work - by You to the Licensor shall be under the terms and conditions of - this License, without any additional terms or conditions. - Notwithstanding the above, nothing herein shall supersede or modify - the terms of any separate license agreement you may have executed - with Licensor regarding such Contributions. - - 6. Trademarks. This License does not grant permission to use the trade - names, trademarks, service marks, or product names of the Licensor, - except as required for reasonable and customary use in describing the - origin of the Work and reproducing the content of the NOTICE file. - - 7. Disclaimer of Warranty. Unless required by applicable law or - agreed to in writing, Licensor provides the Work (and each - Contributor provides its Contributions) on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - implied, including, without limitation, any warranties or conditions - of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A - PARTICULAR PURPOSE. You are solely responsible for determining the - appropriateness of using or redistributing the Work and assume any - risks associated with Your exercise of permissions under this License. - - 8. Limitation of Liability. In no event and under no legal theory, - whether in tort (including negligence), contract, or otherwise, - unless required by applicable law (such as deliberate and grossly - negligent acts) or agreed to in writing, shall any Contributor be - liable to You for damages, including any direct, indirect, special, - incidental, or consequential damages of any character arising as a - result of this License or out of the use or inability to use the - Work (including but not limited to damages for loss of goodwill, - work stoppage, computer failure or malfunction, or any and all - other commercial damages or losses), even if such Contributor - has been advised of the possibility of such damages. - - 9. Accepting Warranty or Additional Liability. While redistributing - the Work or Derivative Works thereof, You may choose to offer, - and charge a fee for, acceptance of support, warranty, indemnity, - or other liability obligations and/or rights consistent with this - License. However, in accepting such obligations, You may act only - on Your own behalf and on Your sole responsibility, not on behalf - of any other Contributor, and only if You agree to indemnify, - defend, and hold each Contributor harmless for any liability - incurred by, or claims asserted against, such Contributor by reason - of your accepting any such warranty or additional liability. - - END OF TERMS AND CONDITIONS - - APPENDIX: How to apply the Apache License to your work. - - To apply the Apache License to your work, attach the following - boilerplate notice, with the fields enclosed by brackets "[]" - replaced with your own identifying information. (Don't include - the brackets!) The text should be enclosed in the appropriate - comment syntax for the file format. We also recommend that a - file or class name and description of purpose be included on the - same "printed page" as the copyright notice for easier - identification within third-party archives. - - Copyright [yyyy] [name of copyright owner] - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - -APACHE NIFI SUBCOMPONENTS: - -The Apache NiFi project contains subcomponents with separate copyright -notices and license terms. Your use of the source code for the these -subcomponents is subject to the terms and conditions of the following -licenses. - -This product bundles 'jsoup' which is available under the MIT License. -For details see http://jsoup.org/ - - jsoup License - The jsoup code-base (include source and compiled packages) are distributed under the open source MIT license as described below. - - The MIT License - Copyright © 2009 - 2013 Jonathan Hedley (jonathan@hedley.net) - - Permission is hereby granted, free of charge, to any person - obtaining a copy of this software and associated documentation - files (the "Software"), to deal in the Software without - restriction, including without limitation the rights to use, - copy, modify, merge, publish, distribute, sublicense, and/or sell - copies of the Software, and to permit persons to whom the - Software is furnished to do so, subject to the following - conditions: - - The above copyright notice and this permission notice shall be - included in all copies or substantial portions of the Software. - - THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, - EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES - OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND - NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT - HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, - WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING - FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR - OTHER DEALINGS IN THE SOFTWARE. - diff --git a/nifi-nar-bundles/nifi-html-bundle/nifi-html-nar/src/main/resources/META-INF/NOTICE b/nifi-nar-bundles/nifi-html-bundle/nifi-html-nar/src/main/resources/META-INF/NOTICE deleted file mode 100644 index 1648eac249..0000000000 --- a/nifi-nar-bundles/nifi-html-bundle/nifi-html-nar/src/main/resources/META-INF/NOTICE +++ /dev/null @@ -1,19 +0,0 @@ -nifi-html-nar -Copyright 2015-2020 The Apache Software Foundation - -This product includes software developed at -The Apache Software Foundation (http://www.apache.org/). - -****************** -Apache Software License v2 -****************** - -The following binary components are provided under the Apache Software License v2 - - (ASLv2) Apache Commons Lang - The following NOTICE information applies: - Apache Commons Lang - Copyright 2001-2014 The Apache Software Foundation - - This product includes software from the Spring Framework, - under the Apache License 2.0 (see: StringUtils.containsWhitespace()) diff --git a/nifi-nar-bundles/nifi-html-bundle/nifi-html-processors/pom.xml b/nifi-nar-bundles/nifi-html-bundle/nifi-html-processors/pom.xml deleted file mode 100644 index dff44a96ad..0000000000 --- a/nifi-nar-bundles/nifi-html-bundle/nifi-html-processors/pom.xml +++ /dev/null @@ -1,67 +0,0 @@ - - - - 4.0.0 - - - org.apache.nifi - nifi-html-bundle - 2.0.0-SNAPSHOT - - - nifi-html-processors - Support for parsing HTML documents - - - - org.jsoup - jsoup - - - org.apache.commons - commons-lang3 - - - org.apache.nifi - nifi-api - - - org.apache.nifi - nifi-utils - 2.0.0-SNAPSHOT - - - org.apache.nifi - nifi-mock - 2.0.0-SNAPSHOT - test - - - - - - - org.apache.rat - apache-rat-plugin - - - src/test/resources/Weather.html - - - - - - diff --git a/nifi-nar-bundles/nifi-html-bundle/nifi-html-processors/src/main/java/org/apache/nifi/AbstractHTMLProcessor.java b/nifi-nar-bundles/nifi-html-bundle/nifi-html-processors/src/main/java/org/apache/nifi/AbstractHTMLProcessor.java deleted file mode 100644 index 902f215398..0000000000 --- a/nifi-nar-bundles/nifi-html-bundle/nifi-html-processors/src/main/java/org/apache/nifi/AbstractHTMLProcessor.java +++ /dev/null @@ -1,143 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.nifi; - -import org.apache.nifi.components.PropertyDescriptor; -import org.apache.nifi.components.ValidationContext; -import org.apache.nifi.components.ValidationResult; -import org.apache.nifi.components.Validator; -import org.apache.nifi.expression.ExpressionLanguageScope; -import org.apache.nifi.flowfile.FlowFile; -import org.apache.nifi.processor.AbstractProcessor; -import org.apache.nifi.processor.ProcessContext; -import org.apache.nifi.processor.ProcessSession; -import org.apache.nifi.processor.Relationship; -import org.apache.nifi.processor.io.InputStreamCallback; -import org.apache.nifi.processor.util.StandardValidators; -import org.jsoup.Jsoup; -import org.jsoup.nodes.Document; -import org.jsoup.select.Selector; - -import java.io.IOException; -import java.io.InputStream; -import java.util.concurrent.atomic.AtomicReference; - -public abstract class AbstractHTMLProcessor extends AbstractProcessor { - - protected static final String ELEMENT_HTML = "HTML"; - protected static final String ELEMENT_TEXT = "Text"; - protected static final String ELEMENT_DATA = "Data"; - protected static final String ELEMENT_ATTRIBUTE = "Attribute"; - - protected static final Validator CSS_SELECTOR_VALIDATOR = new Validator() { - @Override - public ValidationResult validate(final String subject, final String value, final ValidationContext context) { - if (context.isExpressionLanguageSupported(subject) && context.isExpressionLanguagePresent(value)) { - return new ValidationResult.Builder().subject(subject).input(value).explanation("Expression Language Present").valid(true).build(); - } - - String reason = null; - try { - Document doc = Jsoup.parse(""); - doc.select(value); - } catch (final Selector.SelectorParseException e) { - reason = "\"" + value + "\" is an invalid CSS selector"; - } - - return new ValidationResult.Builder().subject(subject).input(value).explanation(reason).valid(reason == null).build(); - } - }; - - public static final PropertyDescriptor URL = new PropertyDescriptor - .Builder().name("URL") - .description("Base URL for the HTML page being parsed." + - " This URL will be used to resolve an absolute URL" + - " when an attribute value is extracted from a HTML element.") - .required(true) - .addValidator(StandardValidators.NON_EMPTY_VALIDATOR) - .expressionLanguageSupported(ExpressionLanguageScope.FLOWFILE_ATTRIBUTES) - .build(); - - public static final PropertyDescriptor CSS_SELECTOR = new PropertyDescriptor - .Builder().name("CSS Selector") - .description("CSS selector syntax string used to extract the desired HTML element(s).") - .required(true) - .addValidator(CSS_SELECTOR_VALIDATOR) - .expressionLanguageSupported(ExpressionLanguageScope.FLOWFILE_ATTRIBUTES) - .build(); - - public static final PropertyDescriptor HTML_CHARSET = new PropertyDescriptor - .Builder().name("HTML Character Encoding") - .description("Character encoding of the input HTML") - .defaultValue("UTF-8") - .required(true) - .addValidator(StandardValidators.CHARACTER_SET_VALIDATOR) - .build(); - - public static final Relationship REL_ORIGINAL = new Relationship.Builder() - .name("original") - .description("The original HTML input") - .build(); - - public static final Relationship REL_SUCCESS = new Relationship.Builder() - .name("success") - .description("Successfully parsed HTML element") - .build(); - - public static final Relationship REL_INVALID_HTML = new Relationship.Builder() - .name("invalid html") - .description("The input HTML syntax is invalid") - .build(); - - public static final Relationship REL_NOT_FOUND = new Relationship.Builder() - .name("element not found") - .description("Element could not be found in the HTML document. The original HTML input will remain " + - "in the FlowFile content unchanged. Relationship '" + REL_ORIGINAL + "' will not be invoked " + - "in this scenario.") - .build(); - - /** - * Parses the Jsoup HTML document from the FlowFile input content. - * - * @param inputFlowFile Input FlowFile containing the HTML - * @param context ProcessContext - * @param session ProcessSession - * - * @return Jsoup Document - */ - protected Document parseHTMLDocumentFromFlowfile(final FlowFile inputFlowFile, final ProcessContext context, final ProcessSession session) { - final AtomicReference doc = new AtomicReference<>(); - session.read(inputFlowFile, new InputStreamCallback() { - @Override - public void process(InputStream inputStream) throws IOException { - final String baseUrl = getBaseUrl(inputFlowFile, context); - if (baseUrl == null || baseUrl.isEmpty()) { - throw new RuntimeException("Base URL was empty."); - } - doc.set(Jsoup.parse(inputStream, - context.getProperty(HTML_CHARSET).getValue(), - baseUrl)); - } - }); - return doc.get(); - } - - - protected String getBaseUrl(final FlowFile inputFlowFile, final ProcessContext context) { - return "http://localhost/"; - } -} diff --git a/nifi-nar-bundles/nifi-html-bundle/nifi-html-processors/src/main/java/org/apache/nifi/GetHTMLElement.java b/nifi-nar-bundles/nifi-html-bundle/nifi-html-processors/src/main/java/org/apache/nifi/GetHTMLElement.java deleted file mode 100644 index 37dab67b24..0000000000 --- a/nifi-nar-bundles/nifi-html-bundle/nifi-html-processors/src/main/java/org/apache/nifi/GetHTMLElement.java +++ /dev/null @@ -1,247 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.nifi; - -import org.apache.commons.lang3.StringUtils; -import org.apache.nifi.annotation.behavior.InputRequirement; -import org.apache.nifi.annotation.behavior.SupportsBatching; -import org.apache.nifi.components.PropertyDescriptor; -import org.apache.nifi.expression.ExpressionLanguageScope; -import org.apache.nifi.flowfile.FlowFile; -import org.apache.nifi.processor.ProcessContext; -import org.apache.nifi.processor.ProcessSession; -import org.apache.nifi.processor.Relationship; -import org.apache.nifi.processor.ProcessorInitializationContext; -import org.apache.nifi.annotation.behavior.WritesAttribute; -import org.apache.nifi.annotation.behavior.WritesAttributes; -import org.apache.nifi.annotation.documentation.CapabilityDescription; -import org.apache.nifi.annotation.documentation.SeeAlso; -import org.apache.nifi.annotation.documentation.Tags; -import org.apache.nifi.processor.exception.ProcessException; -import org.apache.nifi.processor.io.StreamCallback; -import org.apache.nifi.processor.util.StandardValidators; -import org.jsoup.nodes.Document; -import org.jsoup.nodes.Element; -import org.jsoup.select.Elements; - -import java.io.IOException; -import java.io.InputStream; -import java.io.OutputStream; -import java.nio.charset.StandardCharsets; -import java.util.ArrayList; -import java.util.List; -import java.util.Set; -import java.util.HashSet; -import java.util.Collections; - -@SupportsBatching -@Tags({"get", "html", "dom", "css", "element"}) -@InputRequirement(InputRequirement.Requirement.INPUT_REQUIRED) -@CapabilityDescription("Extracts HTML element values from the incoming flowfile's content using a CSS selector." + - " The incoming HTML is first converted into a HTML Document Object Model so that HTML elements may be selected" + - " in the similar manner that CSS selectors are used to apply styles to HTML. The resulting HTML DOM is then \"queried\"" + - " using the user defined CSS selector string. The result of \"querying\" the HTML DOM may produce 0-N results." + - " If no results are found the flowfile will be transferred to the \"element not found\" relationship to indicate" + - " so to the end user. If N results are found a new flowfile will be created and emitted for each result. The query result will" + - " either be placed in the content of the new flowfile or as an attribute of the new flowfile. By default the result is written to an" + - " attribute. This can be controlled by the \"Destination\" property. Resulting query values may also have data" + - " prepended or appended to them by setting the value of property \"Prepend Element Value\" or \"Append Element Value\"." + - " Prepended and appended values are treated as string values and concatenated to the result retrieved from the" + - " HTML DOM query operation. A more thorough reference for the CSS selector syntax can be found at" + - " \"http://jsoup.org/apidocs/org/jsoup/select/Selector.html\"") -@SeeAlso({ModifyHTMLElement.class, PutHTMLElement.class}) -@WritesAttributes({@WritesAttribute(attribute="HTMLElement", description="Flowfile attribute where the element result" + - " parsed from the HTML using the CSS selector syntax are placed if the destination is a flowfile attribute.")}) -public class GetHTMLElement - extends AbstractHTMLProcessor { - - public static final String HTML_ELEMENT_ATTRIBUTE_NAME = "HTMLElement"; - public static final String DESTINATION_ATTRIBUTE = "flowfile-attribute"; - public static final String DESTINATION_CONTENT = "flowfile-content"; - - public static final PropertyDescriptor PREPEND_ELEMENT_VALUE = new PropertyDescriptor - .Builder().name("Prepend Element Value") - .description("Prepends the specified value to the resulting Element") - .required(false) - .addValidator(StandardValidators.NON_EMPTY_VALIDATOR) - .expressionLanguageSupported(ExpressionLanguageScope.FLOWFILE_ATTRIBUTES) - .build(); - - public static final PropertyDescriptor APPEND_ELEMENT_VALUE = new PropertyDescriptor - .Builder().name("Append Element Value") - .description("Appends the specified value to the resulting Element") - .required(false) - .addValidator(StandardValidators.NON_EMPTY_VALIDATOR) - .expressionLanguageSupported(ExpressionLanguageScope.FLOWFILE_ATTRIBUTES) - .build(); - - public static final PropertyDescriptor ATTRIBUTE_KEY = new PropertyDescriptor - .Builder().name("Attribute Name") - .description(("When getting the value of a HTML element attribute this value is used as the key to determine" + - " which attribute on the selected element should be retrieved. This value is used when the \"Output Type\"" + - " is set to \"" + ELEMENT_ATTRIBUTE + "\"." + - " If this value is prefixed with 'abs:', then the extracted attribute value will be converted into" + - " an absolute URL form using the specified base URL.")) - .required(false) - .addValidator(StandardValidators.NON_EMPTY_VALIDATOR) - .expressionLanguageSupported(ExpressionLanguageScope.FLOWFILE_ATTRIBUTES) - .build(); - - public static final PropertyDescriptor OUTPUT_TYPE = new PropertyDescriptor.Builder() - .name("Output Type") - .description("Controls the type of DOM value that is retrieved from the HTML element.") - .required(true) - .addValidator(StandardValidators.NON_EMPTY_VALIDATOR) - .allowableValues(ELEMENT_HTML, ELEMENT_TEXT, ELEMENT_ATTRIBUTE, ELEMENT_DATA) - .defaultValue(ELEMENT_HTML) - .build(); - - public static final PropertyDescriptor DESTINATION = new PropertyDescriptor.Builder() - .name("Destination") - .description("Control if element extracted is written as a flowfile attribute or " + - "as flowfile content.") - .required(true) - .allowableValues(DESTINATION_ATTRIBUTE, DESTINATION_CONTENT) - .defaultValue(DESTINATION_ATTRIBUTE) - .build(); - - private List descriptors; - - private Set relationships; - - @Override - protected void init(final ProcessorInitializationContext context) { - final List descriptors = new ArrayList<>(); - descriptors.add(URL); - descriptors.add(CSS_SELECTOR); - descriptors.add(HTML_CHARSET); - descriptors.add(OUTPUT_TYPE); - descriptors.add(DESTINATION); - descriptors.add(PREPEND_ELEMENT_VALUE); - descriptors.add(APPEND_ELEMENT_VALUE); - descriptors.add(ATTRIBUTE_KEY); - this.descriptors = Collections.unmodifiableList(descriptors); - - final Set relationships = new HashSet<>(); - relationships.add(REL_ORIGINAL); - relationships.add(REL_SUCCESS); - relationships.add(REL_INVALID_HTML); - relationships.add(REL_NOT_FOUND); - this.relationships = Collections.unmodifiableSet(relationships); - } - - @Override - public Set getRelationships() { - return this.relationships; - } - - @Override - public final List getSupportedPropertyDescriptors() { - return descriptors; - } - - @Override - public void onTrigger(final ProcessContext context, final ProcessSession session) throws ProcessException { - final FlowFile flowFile = session.get(); - if ( flowFile == null ) { - return; - } - - final Document doc; - final Elements eles; - - try { - doc = parseHTMLDocumentFromFlowfile(flowFile, context, session); - eles = doc.select(context.getProperty(CSS_SELECTOR).evaluateAttributeExpressions(flowFile).getValue()); - } catch (final Exception ex) { - getLogger().error("Failed to extract HTML from {} due to {}; routing to {}", flowFile, ex, REL_INVALID_HTML, ex); - session.transfer(flowFile, REL_INVALID_HTML); - return; - } - - final String prependValue = context.getProperty(PREPEND_ELEMENT_VALUE).evaluateAttributeExpressions(flowFile).getValue(); - final String appendValue = context.getProperty(APPEND_ELEMENT_VALUE).evaluateAttributeExpressions(flowFile).getValue(); - final String outputType = context.getProperty(OUTPUT_TYPE).getValue(); - final String attributeKey = context.getProperty(ATTRIBUTE_KEY).evaluateAttributeExpressions(flowFile).getValue(); - - if (eles == null || eles.isEmpty()) { - // No element found - session.transfer(flowFile, REL_NOT_FOUND); - } else { - // Create a new FlowFile for each matching element. - for (final Element ele : eles) { - final String extractedElementValue = extractElementValue(prependValue, outputType, appendValue, ele, attributeKey); - - final FlowFile ff = session.create(flowFile); - FlowFile updatedFF = ff; - - switch (context.getProperty(DESTINATION).getValue()) { - case DESTINATION_ATTRIBUTE: - updatedFF = session.putAttribute(ff, HTML_ELEMENT_ATTRIBUTE_NAME, extractedElementValue); - break; - case DESTINATION_CONTENT: - updatedFF = session.write(ff, new StreamCallback() { - @Override - public void process(final InputStream inputStream, final OutputStream outputStream) throws IOException { - outputStream.write(extractedElementValue.getBytes(StandardCharsets.UTF_8)); - } - }); - - break; - } - - session.transfer(updatedFF, REL_SUCCESS); - } - - // Transfer the original HTML - session.transfer(flowFile, REL_ORIGINAL); - } - } - - - /** - * Extracts the HTML value based on the configuration values. - * - * @return value from the parsed HTML element - */ - private String extractElementValue(String prependValue, final String outputType, String appendValue, final Element ele, final String attrKey) { - if (StringUtils.isEmpty(prependValue)) { - prependValue = ""; - } - if (StringUtils.isEmpty(appendValue)) { - appendValue = ""; - } - - switch (outputType) { - case ELEMENT_HTML: - return prependValue + ele.html() + appendValue; - case ELEMENT_TEXT: - return prependValue + ele.text() + appendValue; - case ELEMENT_DATA: - return prependValue + ele.data() + appendValue; - case ELEMENT_ATTRIBUTE: - return prependValue + ele.attr(attrKey) + appendValue; - default: - return prependValue + ele.html() + appendValue; - } - } - - @Override - protected String getBaseUrl(FlowFile inputFlowFile, ProcessContext context) { - return context.getProperty(URL).evaluateAttributeExpressions(inputFlowFile).getValue(); - } -} diff --git a/nifi-nar-bundles/nifi-html-bundle/nifi-html-processors/src/main/java/org/apache/nifi/ModifyHTMLElement.java b/nifi-nar-bundles/nifi-html-bundle/nifi-html-processors/src/main/java/org/apache/nifi/ModifyHTMLElement.java deleted file mode 100644 index d864f71a2d..0000000000 --- a/nifi-nar-bundles/nifi-html-bundle/nifi-html-processors/src/main/java/org/apache/nifi/ModifyHTMLElement.java +++ /dev/null @@ -1,189 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.nifi; - -import org.apache.nifi.annotation.behavior.InputRequirement; -import org.apache.nifi.annotation.behavior.SupportsBatching; -import org.apache.nifi.annotation.behavior.WritesAttribute; -import org.apache.nifi.annotation.behavior.WritesAttributes; -import org.apache.nifi.annotation.documentation.CapabilityDescription; -import org.apache.nifi.annotation.documentation.SeeAlso; -import org.apache.nifi.annotation.documentation.Tags; -import org.apache.nifi.components.PropertyDescriptor; -import org.apache.nifi.expression.ExpressionLanguageScope; -import org.apache.nifi.flowfile.FlowFile; -import org.apache.nifi.processor.ProcessContext; -import org.apache.nifi.processor.ProcessSession; -import org.apache.nifi.processor.Relationship; -import org.apache.nifi.processor.ProcessorInitializationContext; -import org.apache.nifi.processor.exception.ProcessException; -import org.apache.nifi.processor.io.StreamCallback; -import org.apache.nifi.processor.util.StandardValidators; -import org.jsoup.nodes.Document; -import org.jsoup.nodes.Element; -import org.jsoup.select.Elements; - -import java.io.IOException; -import java.io.InputStream; -import java.io.OutputStream; -import java.nio.charset.StandardCharsets; -import java.util.List; -import java.util.ArrayList; -import java.util.Set; -import java.util.HashSet; -import java.util.Collections; - -@Tags({"modify", "html", "dom", "css", "element"}) -@SupportsBatching -@InputRequirement(InputRequirement.Requirement.INPUT_REQUIRED) -@CapabilityDescription("Modifies the value of an existing HTML element. The desired element to be modified is located by" + - " using CSS selector syntax. The incoming HTML is first converted into a HTML Document Object Model so that HTML elements may be selected" + - " in the similar manner that CSS selectors are used to apply styles to HTML. The resulting HTML DOM is then \"queried\"" + - " using the user defined CSS selector string to find the element the user desires to modify. If the HTML element is found" + - " the element's value is updated in the DOM using the value specified \"Modified Value\" property. All DOM elements" + - " that match the CSS selector will be updated. Once all of the DOM elements have been updated the DOM is rendered" + - " to HTML and the result replaces the flowfile content with the updated HTML. A more thorough reference for the" + - " CSS selector syntax can be found at" + - " \"http://jsoup.org/apidocs/org/jsoup/select/Selector.html\"") -@SeeAlso({GetHTMLElement.class, PutHTMLElement.class}) -@WritesAttributes({@WritesAttribute(attribute="NumElementsModified", description="Total number of HTML " + - "element modifications made")}) -public class ModifyHTMLElement extends AbstractHTMLProcessor { - - public static final String NUM_ELEMENTS_MODIFIED_ATTR = "NumElementsModified"; - - public static final PropertyDescriptor OUTPUT_TYPE = new PropertyDescriptor.Builder() - .name("Output Type") - .description("Controls whether the HTML element is output as " + - ELEMENT_HTML + "," + ELEMENT_TEXT + " or " + ELEMENT_DATA) - .required(true) - .allowableValues(ELEMENT_HTML, ELEMENT_TEXT, ELEMENT_ATTRIBUTE) - .defaultValue(ELEMENT_HTML) - .build(); - - public static final PropertyDescriptor MODIFIED_VALUE = new PropertyDescriptor - .Builder().name("Modified Value") - .description("Value to update the found HTML elements with") - .required(true) - .addValidator(StandardValidators.NON_EMPTY_VALIDATOR) - .expressionLanguageSupported(ExpressionLanguageScope.FLOWFILE_ATTRIBUTES) - .build(); - - public static final PropertyDescriptor ATTRIBUTE_KEY = new PropertyDescriptor - .Builder().name("Attribute Name") - .description(("When modifying the value of an element attribute this value is used as the key to determine" + - " which attribute on the selected element will be modified with the new value.")) - .required(false) - .addValidator(StandardValidators.NON_EMPTY_VALIDATOR) - .expressionLanguageSupported(ExpressionLanguageScope.FLOWFILE_ATTRIBUTES) - .build(); - - private List descriptors; - - private Set relationships; - - @Override - protected void init(final ProcessorInitializationContext context) { - final List descriptors = new ArrayList<>(); - descriptors.add(CSS_SELECTOR); - descriptors.add(HTML_CHARSET); - descriptors.add(OUTPUT_TYPE); - descriptors.add(MODIFIED_VALUE); - descriptors.add(ATTRIBUTE_KEY); - this.descriptors = Collections.unmodifiableList(descriptors); - - final Set relationships = new HashSet(); - relationships.add(REL_ORIGINAL); - relationships.add(REL_SUCCESS); - relationships.add(REL_INVALID_HTML); - relationships.add(REL_NOT_FOUND); - this.relationships = Collections.unmodifiableSet(relationships); - } - - @Override - public Set getRelationships() { - return this.relationships; - } - - @Override - public final List getSupportedPropertyDescriptors() { - return descriptors; - } - - /** - * This processor used to support URL property, but it has been removed - * since it's not required when altering HTML elements. - * Support URL as dynamic property so that existing data flow can stay in valid state without modification. - */ - @Override - protected PropertyDescriptor getSupportedDynamicPropertyDescriptor(final String propertyDescriptorName) { - return URL; - } - - @Override - public void onTrigger(final ProcessContext context, final ProcessSession session) throws ProcessException { - final FlowFile flowFile = session.get(); - if (flowFile == null) { - return; - } - - final Document doc; - final Elements eles; - try { - doc = parseHTMLDocumentFromFlowfile(flowFile, context, session); - eles = doc.select(context.getProperty(CSS_SELECTOR).evaluateAttributeExpressions(flowFile).getValue()); - } catch (Exception ex) { - getLogger().error("Failed to extract HTML from {} due to {}; routing to {}", flowFile, ex.toString(), REL_INVALID_HTML.getName(), ex); - session.transfer(flowFile, REL_INVALID_HTML); - return; - } - - final String modifiedValue = context.getProperty(MODIFIED_VALUE).evaluateAttributeExpressions(flowFile).getValue(); - - if (eles == null || eles.size() == 0) { - // No element found - session.transfer(flowFile, REL_NOT_FOUND); - } else { - for (Element ele : eles) { - switch (context.getProperty(OUTPUT_TYPE).getValue()) { - case ELEMENT_HTML: - ele.html(modifiedValue); - break; - case ELEMENT_ATTRIBUTE: - ele.attr(context.getProperty(ATTRIBUTE_KEY).evaluateAttributeExpressions(flowFile).getValue(), modifiedValue); - break; - case ELEMENT_TEXT: - ele.text(modifiedValue); - break; - } - } - - FlowFile ff = session.write(session.create(flowFile), new StreamCallback() { - @Override - public void process(InputStream in, OutputStream out) throws IOException { - out.write(doc.html().getBytes(StandardCharsets.UTF_8)); - } - }); - ff = session.putAttribute(ff, NUM_ELEMENTS_MODIFIED_ATTR, Integer.valueOf(eles.size()).toString()); - session.transfer(ff, REL_SUCCESS); - - // Transfer the original HTML - session.transfer(flowFile, REL_ORIGINAL); - } - } - -} diff --git a/nifi-nar-bundles/nifi-html-bundle/nifi-html-processors/src/main/java/org/apache/nifi/PutHTMLElement.java b/nifi-nar-bundles/nifi-html-bundle/nifi-html-processors/src/main/java/org/apache/nifi/PutHTMLElement.java deleted file mode 100644 index c3d4890443..0000000000 --- a/nifi-nar-bundles/nifi-html-bundle/nifi-html-processors/src/main/java/org/apache/nifi/PutHTMLElement.java +++ /dev/null @@ -1,175 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.nifi; - -import org.apache.nifi.annotation.behavior.InputRequirement; -import org.apache.nifi.annotation.behavior.SupportsBatching; -import org.apache.nifi.annotation.documentation.CapabilityDescription; -import org.apache.nifi.annotation.documentation.SeeAlso; -import org.apache.nifi.annotation.documentation.Tags; -import org.apache.nifi.components.PropertyDescriptor; -import org.apache.nifi.expression.ExpressionLanguageScope; -import org.apache.nifi.flowfile.FlowFile; -import org.apache.nifi.processor.ProcessContext; -import org.apache.nifi.processor.ProcessSession; -import org.apache.nifi.processor.Relationship; -import org.apache.nifi.processor.ProcessorInitializationContext; -import org.apache.nifi.processor.exception.ProcessException; -import org.apache.nifi.processor.io.StreamCallback; -import org.apache.nifi.processor.util.StandardValidators; -import org.jsoup.nodes.Document; -import org.jsoup.nodes.Element; -import org.jsoup.select.Elements; - -import java.io.IOException; -import java.io.InputStream; -import java.io.OutputStream; -import java.nio.charset.StandardCharsets; -import java.util.ArrayList; -import java.util.List; -import java.util.Set; -import java.util.HashSet; -import java.util.Collections; - -@Tags({"put", "html", "dom", "css", "element"}) -@SupportsBatching -@InputRequirement(InputRequirement.Requirement.INPUT_REQUIRED) -@CapabilityDescription("Places a new HTML element in the existing HTML DOM. The desired position for the new HTML element is specified by" + - " using CSS selector syntax. The incoming HTML is first converted into a HTML Document Object Model so that HTML DOM location may be located" + - " in a similar manner that CSS selectors are used to apply styles to HTML. The resulting HTML DOM is then \"queried\"" + - " using the user defined CSS selector string to find the position where the user desires to add the new HTML element." + - " Once the new HTML element is added to the DOM it is rendered to HTML and the result replaces the flowfile" + - " content with the updated HTML. A more thorough reference for the CSS selector syntax can be found at" + - " \"http://jsoup.org/apidocs/org/jsoup/select/Selector.html\"") -@SeeAlso({GetHTMLElement.class, ModifyHTMLElement.class}) -public class PutHTMLElement extends AbstractHTMLProcessor { - - public static final String APPEND_ELEMENT = "append-html"; - public static final String PREPEND_ELEMENT = "prepend-html"; - - public static final PropertyDescriptor PUT_LOCATION_TYPE = new PropertyDescriptor.Builder() - .name("Element Insert Location Type") - .description("Controls whether the new element is prepended or appended to the children of the " + - "Element located by the CSS selector. EX: prepended value 'Hi' inside of " + - "Element (using CSS Selector 'p') '

There

' would result in " + - "'

HiThere

'. Appending the value would result in '

ThereHi

'") - .required(true) - .allowableValues(APPEND_ELEMENT, PREPEND_ELEMENT) - .defaultValue(APPEND_ELEMENT) - .build(); - - public static final PropertyDescriptor PUT_VALUE = new PropertyDescriptor.Builder() - .name("Put Value") - .description("Value used when creating the new Element. Value should be a valid HTML element. " + - "The text should be supplied unencoded: characters like '<', '>', etc will be properly HTML " + - "encoded in the resulting output.") - .required(true) - .addValidator(StandardValidators.NON_EMPTY_VALIDATOR) - .expressionLanguageSupported(ExpressionLanguageScope.FLOWFILE_ATTRIBUTES) - .build(); - - private List descriptors; - - private Set relationships; - - @Override - protected void init(final ProcessorInitializationContext context) { - final List descriptors = new ArrayList(); - descriptors.add(CSS_SELECTOR); - descriptors.add(HTML_CHARSET); - descriptors.add(PUT_LOCATION_TYPE); - descriptors.add(PUT_VALUE); - this.descriptors = Collections.unmodifiableList(descriptors); - - final Set relationships = new HashSet(); - relationships.add(REL_ORIGINAL); - relationships.add(REL_SUCCESS); - relationships.add(REL_INVALID_HTML); - relationships.add(REL_NOT_FOUND); - this.relationships = Collections.unmodifiableSet(relationships); - } - - @Override - public Set getRelationships() { - return this.relationships; - } - - @Override - public final List getSupportedPropertyDescriptors() { - return descriptors; - } - - /** - * This processor used to support URL property, but it has been removed - * since it's not required when altering HTML elements. - * Support URL as dynamic property so that existing data flow can stay in valid state without modification. - */ - @Override - protected PropertyDescriptor getSupportedDynamicPropertyDescriptor(final String propertyDescriptorName) { - return URL; - } - - @Override - public void onTrigger(final ProcessContext context, final ProcessSession session) throws ProcessException { - final FlowFile flowFile = session.get(); - if (flowFile == null) { - return; - } - - final Document doc; - final Elements eles; - try { - doc = parseHTMLDocumentFromFlowfile(flowFile, context, session); - eles = doc.select(context.getProperty(CSS_SELECTOR).evaluateAttributeExpressions(flowFile).getValue()); - } catch (Exception ex) { - getLogger().error("Failed to extract HTML from {} due to {}; routing to {}", flowFile, ex.toString(), REL_INVALID_HTML.getName(), ex); - session.transfer(flowFile, REL_INVALID_HTML); - return; - } - - - if (eles == null || eles.isEmpty()) { - // No element found - session.transfer(flowFile, REL_NOT_FOUND); - } else { - final String putValue = context.getProperty(PUT_VALUE).evaluateAttributeExpressions(flowFile).getValue(); - - for (final Element ele : eles) { - switch (context.getProperty(PUT_LOCATION_TYPE).getValue()) { - case APPEND_ELEMENT: - ele.append(putValue); - break; - case PREPEND_ELEMENT: - ele.prepend(putValue); - break; - } - } - - FlowFile ff = session.write(session.create(flowFile), new StreamCallback() { - @Override - public void process(final InputStream in, final OutputStream out) throws IOException { - out.write(doc.html().getBytes(StandardCharsets.UTF_8)); - } - }); - - session.transfer(ff, REL_SUCCESS); - - // Transfer the original HTML - session.transfer(flowFile, REL_ORIGINAL); - } - } -} diff --git a/nifi-nar-bundles/nifi-html-bundle/nifi-html-processors/src/main/resources/META-INF/services/org.apache.nifi.processor.Processor b/nifi-nar-bundles/nifi-html-bundle/nifi-html-processors/src/main/resources/META-INF/services/org.apache.nifi.processor.Processor deleted file mode 100644 index aea106050c..0000000000 --- a/nifi-nar-bundles/nifi-html-bundle/nifi-html-processors/src/main/resources/META-INF/services/org.apache.nifi.processor.Processor +++ /dev/null @@ -1,17 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -org.apache.nifi.GetHTMLElement -org.apache.nifi.ModifyHTMLElement -org.apache.nifi.PutHTMLElement \ No newline at end of file diff --git a/nifi-nar-bundles/nifi-html-bundle/nifi-html-processors/src/test/java/org/apache/nifi/AbstractHTMLTest.java b/nifi-nar-bundles/nifi-html-bundle/nifi-html-processors/src/test/java/org/apache/nifi/AbstractHTMLTest.java deleted file mode 100644 index ba34be1d3f..0000000000 --- a/nifi-nar-bundles/nifi-html-bundle/nifi-html-processors/src/test/java/org/apache/nifi/AbstractHTMLTest.java +++ /dev/null @@ -1,26 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.nifi; - -public abstract class AbstractHTMLTest { - protected final String ATL_WEATHER_TEXT = "Atlanta Weather"; - protected final String GDR_WEATHER_TEXT = "Grand Rapids Weather"; - protected final String ATL_WEATHER_LINK = "http://w1.weather.gov/obhistory/KPDK.html"; - protected final String AUTHOR_NAME = "Apache NiFi Community"; - protected final String ATL_ID = "ATL"; - protected final String GDR_ID = "GDR"; -} diff --git a/nifi-nar-bundles/nifi-html-bundle/nifi-html-processors/src/test/java/org/apache/nifi/TestGetHTMLElement.java b/nifi-nar-bundles/nifi-html-bundle/nifi-html-processors/src/test/java/org/apache/nifi/TestGetHTMLElement.java deleted file mode 100644 index 36c81fed62..0000000000 --- a/nifi-nar-bundles/nifi-html-bundle/nifi-html-processors/src/test/java/org/apache/nifi/TestGetHTMLElement.java +++ /dev/null @@ -1,352 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.nifi; - -import java.io.File; -import java.io.IOException; -import java.nio.charset.StandardCharsets; -import java.util.HashMap; -import java.util.List; -import java.util.Map; - -import org.apache.nifi.util.MockFlowFile; -import org.apache.nifi.util.TestRunner; -import org.apache.nifi.util.TestRunners; -import org.jsoup.Jsoup; -import org.jsoup.nodes.Document; -import org.jsoup.select.Selector; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; - -import static org.junit.jupiter.api.Assertions.assertThrows; - -public class TestGetHTMLElement extends AbstractHTMLTest { - - private TestRunner testRunner; - - @BeforeEach - public void init() { - testRunner = TestRunners.newTestRunner(GetHTMLElement.class); - testRunner.setProperty(GetHTMLElement.URL, "http://localhost"); - testRunner.setProperty(GetHTMLElement.OUTPUT_TYPE, GetHTMLElement.ELEMENT_HTML); - testRunner.setProperty(GetHTMLElement.DESTINATION, GetHTMLElement.DESTINATION_CONTENT); - testRunner.setProperty(GetHTMLElement.HTML_CHARSET, "UTF-8"); - } - - @Test - public void testCSSSelectorSyntaxValidator() throws IOException { - Document doc = Jsoup.parse(new File("src/test/resources/Weather.html"), StandardCharsets.UTF_8.name()); - assertThrows(Selector.SelectorParseException.class, () -> doc.select("---invalidCssSelector")); - } - - @Test - public void testNoElementFound() throws Exception { - testRunner.setProperty(GetHTMLElement.CSS_SELECTOR, "b"); //Bold element is not present in sample HTML - - testRunner.enqueue(new File("src/test/resources/Weather.html").toPath()); - testRunner.run(); - - testRunner.assertTransferCount(GetHTMLElement.REL_SUCCESS, 0); - testRunner.assertTransferCount(GetHTMLElement.REL_INVALID_HTML, 0); - testRunner.assertTransferCount(GetHTMLElement.REL_NOT_FOUND, 1); - } - - @Test - public void testInvalidSelector() throws Exception { - testRunner.setProperty(GetHTMLElement.CSS_SELECTOR, "InvalidCSSSelectorSyntax"); - - testRunner.enqueue(new File("src/test/resources/Weather.html").toPath()); - testRunner.run(); - - testRunner.assertTransferCount(GetHTMLElement.REL_SUCCESS, 0); - testRunner.assertTransferCount(GetHTMLElement.REL_INVALID_HTML, 0); - testRunner.assertTransferCount(GetHTMLElement.REL_NOT_FOUND, 1); - } - - @Test - public void testSingleElementFound() throws Exception { - testRunner.setProperty(GetHTMLElement.CSS_SELECTOR, "head"); - - testRunner.enqueue(new File("src/test/resources/Weather.html").toPath()); - testRunner.run(); - - testRunner.assertTransferCount(GetHTMLElement.REL_SUCCESS, 1); - testRunner.assertTransferCount(GetHTMLElement.REL_INVALID_HTML, 0); - testRunner.assertTransferCount(GetHTMLElement.REL_ORIGINAL, 1); - testRunner.assertTransferCount(GetHTMLElement.REL_NOT_FOUND, 0); - } - - @Test - public void testMultipleElementFound() throws Exception { - testRunner.setProperty(GetHTMLElement.CSS_SELECTOR, "a"); - - testRunner.enqueue(new File("src/test/resources/Weather.html").toPath()); - testRunner.run(); - - testRunner.assertTransferCount(GetHTMLElement.REL_SUCCESS, 3); - testRunner.assertTransferCount(GetHTMLElement.REL_INVALID_HTML, 0); - testRunner.assertTransferCount(GetHTMLElement.REL_ORIGINAL, 1); - testRunner.assertTransferCount(GetHTMLElement.REL_NOT_FOUND, 0); - } - - @Test - public void testElementFoundWriteToAttribute() throws Exception { - testRunner.setProperty(GetHTMLElement.CSS_SELECTOR, "#" + ATL_ID); - testRunner.setProperty(GetHTMLElement.DESTINATION, GetHTMLElement.DESTINATION_ATTRIBUTE); - testRunner.setProperty(GetHTMLElement.OUTPUT_TYPE, GetHTMLElement.ELEMENT_ATTRIBUTE); - testRunner.setProperty(GetHTMLElement.ATTRIBUTE_KEY, "href"); - - testRunner.enqueue(new File("src/test/resources/Weather.html").toPath()); - testRunner.run(); - - testRunner.assertTransferCount(GetHTMLElement.REL_SUCCESS, 1); - testRunner.assertTransferCount(GetHTMLElement.REL_INVALID_HTML, 0); - testRunner.assertTransferCount(GetHTMLElement.REL_ORIGINAL, 1); - testRunner.assertTransferCount(GetHTMLElement.REL_NOT_FOUND, 0); - - List ffs = testRunner.getFlowFilesForRelationship(GetHTMLElement.REL_SUCCESS); - ffs.get(0).assertAttributeEquals(GetHTMLElement.HTML_ELEMENT_ATTRIBUTE_NAME, ATL_WEATHER_LINK); - } - - @Test - public void testElementFoundWriteToContent() throws Exception { - testRunner.setProperty(GetHTMLElement.CSS_SELECTOR, "#" + ATL_ID); - testRunner.setProperty(GetHTMLElement.DESTINATION, GetHTMLElement.DESTINATION_CONTENT); - testRunner.setProperty(GetHTMLElement.OUTPUT_TYPE, GetHTMLElement.ELEMENT_ATTRIBUTE); - testRunner.setProperty(GetHTMLElement.ATTRIBUTE_KEY, "href"); - - testRunner.enqueue(new File("src/test/resources/Weather.html").toPath()); - testRunner.run(); - - testRunner.assertTransferCount(GetHTMLElement.REL_SUCCESS, 1); - testRunner.assertTransferCount(GetHTMLElement.REL_INVALID_HTML, 0); - testRunner.assertTransferCount(GetHTMLElement.REL_ORIGINAL, 1); - testRunner.assertTransferCount(GetHTMLElement.REL_NOT_FOUND, 0); - - List ffs = testRunner.getFlowFilesForRelationship(GetHTMLElement.REL_SUCCESS); - ffs.get(0).assertContentEquals(ATL_WEATHER_LINK); - } - - @Test - public void testValidPrependValueToFoundElement() throws Exception { - final String PREPEND_VALUE = "TestPrepend"; - testRunner.setProperty(GetHTMLElement.PREPEND_ELEMENT_VALUE, PREPEND_VALUE); - testRunner.setProperty(GetHTMLElement.CSS_SELECTOR, "#" + ATL_ID); - testRunner.setProperty(GetHTMLElement.DESTINATION, GetHTMLElement.DESTINATION_CONTENT); - testRunner.setProperty(GetHTMLElement.OUTPUT_TYPE, GetHTMLElement.ELEMENT_ATTRIBUTE); - testRunner.setProperty(GetHTMLElement.ATTRIBUTE_KEY, "href"); - - testRunner.enqueue(new File("src/test/resources/Weather.html").toPath()); - testRunner.run(); - - testRunner.assertTransferCount(GetHTMLElement.REL_SUCCESS, 1); - testRunner.assertTransferCount(GetHTMLElement.REL_INVALID_HTML, 0); - testRunner.assertTransferCount(GetHTMLElement.REL_ORIGINAL, 1); - testRunner.assertTransferCount(GetHTMLElement.REL_NOT_FOUND, 0); - - List ffs = testRunner.getFlowFilesForRelationship(GetHTMLElement.REL_SUCCESS); - ffs.get(0).assertContentEquals(PREPEND_VALUE + ATL_WEATHER_LINK); - } - - @Test - public void testValidPrependValueToNotFoundElement() throws Exception { - final String PREPEND_VALUE = "TestPrepend"; - testRunner.setProperty(GetHTMLElement.PREPEND_ELEMENT_VALUE, PREPEND_VALUE); - testRunner.setProperty(GetHTMLElement.CSS_SELECTOR, "b"); - testRunner.setProperty(GetHTMLElement.DESTINATION, GetHTMLElement.DESTINATION_CONTENT); - testRunner.setProperty(GetHTMLElement.OUTPUT_TYPE, GetHTMLElement.ELEMENT_TEXT); - - testRunner.enqueue(new File("src/test/resources/Weather.html").toPath()); - testRunner.run(); - - testRunner.assertTransferCount(GetHTMLElement.REL_SUCCESS, 0); - testRunner.assertTransferCount(GetHTMLElement.REL_INVALID_HTML, 0); - testRunner.assertTransferCount(GetHTMLElement.REL_ORIGINAL, 0); - testRunner.assertTransferCount(GetHTMLElement.REL_NOT_FOUND, 1); - } - - @Test - public void testValidAppendValueToFoundElement() throws Exception { - final String APPEND_VALUE = "TestAppend"; - testRunner.setProperty(GetHTMLElement.APPEND_ELEMENT_VALUE, APPEND_VALUE); - testRunner.setProperty(GetHTMLElement.CSS_SELECTOR, "#" + ATL_ID); - testRunner.setProperty(GetHTMLElement.DESTINATION, GetHTMLElement.DESTINATION_CONTENT); - testRunner.setProperty(GetHTMLElement.OUTPUT_TYPE, GetHTMLElement.ELEMENT_ATTRIBUTE); - testRunner.setProperty(GetHTMLElement.ATTRIBUTE_KEY, "href"); - - testRunner.enqueue(new File("src/test/resources/Weather.html").toPath()); - testRunner.run(); - - testRunner.assertTransferCount(GetHTMLElement.REL_SUCCESS, 1); - testRunner.assertTransferCount(GetHTMLElement.REL_INVALID_HTML, 0); - testRunner.assertTransferCount(GetHTMLElement.REL_ORIGINAL, 1); - testRunner.assertTransferCount(GetHTMLElement.REL_NOT_FOUND, 0); - - List ffs = testRunner.getFlowFilesForRelationship(GetHTMLElement.REL_SUCCESS); - ffs.get(0).assertContentEquals(ATL_WEATHER_LINK + APPEND_VALUE); - } - - @Test - public void testValidAppendValueToNotFoundElement() throws Exception { - final String APPEND_VALUE = "TestAppend"; - testRunner.setProperty(GetHTMLElement.APPEND_ELEMENT_VALUE, APPEND_VALUE); - testRunner.setProperty(GetHTMLElement.CSS_SELECTOR, "b"); - testRunner.setProperty(GetHTMLElement.DESTINATION, GetHTMLElement.DESTINATION_CONTENT); - testRunner.setProperty(GetHTMLElement.OUTPUT_TYPE, GetHTMLElement.ELEMENT_TEXT); - - testRunner.enqueue(new File("src/test/resources/Weather.html").toPath()); - testRunner.run(); - - testRunner.assertTransferCount(GetHTMLElement.REL_SUCCESS, 0); - testRunner.assertTransferCount(GetHTMLElement.REL_INVALID_HTML, 0); - testRunner.assertTransferCount(GetHTMLElement.REL_ORIGINAL, 0); - testRunner.assertTransferCount(GetHTMLElement.REL_NOT_FOUND, 1); - } - - @Test - public void testExtractAttributeFromElement() throws Exception { - testRunner.setProperty(GetHTMLElement.CSS_SELECTOR, "meta[name=author]"); - testRunner.setProperty(GetHTMLElement.DESTINATION, GetHTMLElement.DESTINATION_CONTENT); - testRunner.setProperty(GetHTMLElement.OUTPUT_TYPE, GetHTMLElement.ELEMENT_ATTRIBUTE); - testRunner.setProperty(GetHTMLElement.ATTRIBUTE_KEY, "Content"); - - testRunner.enqueue(new File("src/test/resources/Weather.html").toPath()); - testRunner.run(); - - testRunner.assertTransferCount(GetHTMLElement.REL_SUCCESS, 1); - testRunner.assertTransferCount(GetHTMLElement.REL_INVALID_HTML, 0); - testRunner.assertTransferCount(GetHTMLElement.REL_ORIGINAL, 1); - testRunner.assertTransferCount(GetHTMLElement.REL_NOT_FOUND, 0); - - List ffs = testRunner.getFlowFilesForRelationship(GetHTMLElement.REL_SUCCESS); - ffs.get(0).assertContentEquals(AUTHOR_NAME); - } - - @Test - public void testExtractAttributeFromElementRelativeUrl() throws Exception { - testRunner.setProperty(GetHTMLElement.CSS_SELECTOR, "script"); - testRunner.setProperty(GetHTMLElement.DESTINATION, GetHTMLElement.DESTINATION_CONTENT); - testRunner.setProperty(GetHTMLElement.OUTPUT_TYPE, GetHTMLElement.ELEMENT_ATTRIBUTE); - testRunner.setProperty(GetHTMLElement.ATTRIBUTE_KEY, "src"); - - testRunner.enqueue(new File("src/test/resources/Weather.html").toPath()); - testRunner.run(); - - testRunner.assertTransferCount(GetHTMLElement.REL_SUCCESS, 1); - testRunner.assertTransferCount(GetHTMLElement.REL_INVALID_HTML, 0); - testRunner.assertTransferCount(GetHTMLElement.REL_ORIGINAL, 1); - testRunner.assertTransferCount(GetHTMLElement.REL_NOT_FOUND, 0); - - List ffs = testRunner.getFlowFilesForRelationship(GetHTMLElement.REL_SUCCESS); - ffs.get(0).assertContentEquals("js/scripts.js"); - } - - @Test - public void testExtractAttributeFromElementAbsoluteUrl() throws Exception { - testRunner.setProperty(GetHTMLElement.CSS_SELECTOR, "script"); - testRunner.setProperty(GetHTMLElement.DESTINATION, GetHTMLElement.DESTINATION_CONTENT); - testRunner.setProperty(GetHTMLElement.OUTPUT_TYPE, GetHTMLElement.ELEMENT_ATTRIBUTE); - testRunner.setProperty(GetHTMLElement.ATTRIBUTE_KEY, "abs:src"); - - testRunner.enqueue(new File("src/test/resources/Weather.html").toPath()); - testRunner.run(); - - testRunner.assertTransferCount(GetHTMLElement.REL_SUCCESS, 1); - testRunner.assertTransferCount(GetHTMLElement.REL_INVALID_HTML, 0); - testRunner.assertTransferCount(GetHTMLElement.REL_ORIGINAL, 1); - testRunner.assertTransferCount(GetHTMLElement.REL_NOT_FOUND, 0); - - List ffs = testRunner.getFlowFilesForRelationship(GetHTMLElement.REL_SUCCESS); - ffs.get(0).assertContentEquals("http://localhost/js/scripts.js"); - } - - @Test - public void testExtractAttributeFromElementAbsoluteUrlWithEL() throws Exception { - testRunner.setProperty(GetHTMLElement.CSS_SELECTOR, "script"); - testRunner.setProperty(GetHTMLElement.DESTINATION, GetHTMLElement.DESTINATION_CONTENT); - testRunner.setProperty(GetHTMLElement.OUTPUT_TYPE, GetHTMLElement.ELEMENT_ATTRIBUTE); - testRunner.setProperty(GetHTMLElement.ATTRIBUTE_KEY, "abs:src"); - testRunner.setProperty(GetHTMLElement.URL, "${contentUrl}"); - - final Map attributes = new HashMap<>(); - attributes.put("contentUrl", "https://example.com/a/b/c/Weather.html"); - testRunner.enqueue(new File("src/test/resources/Weather.html").toPath(), attributes); - testRunner.run(); - - testRunner.assertTransferCount(GetHTMLElement.REL_SUCCESS, 1); - testRunner.assertTransferCount(GetHTMLElement.REL_INVALID_HTML, 0); - testRunner.assertTransferCount(GetHTMLElement.REL_ORIGINAL, 1); - testRunner.assertTransferCount(GetHTMLElement.REL_NOT_FOUND, 0); - - List ffs = testRunner.getFlowFilesForRelationship(GetHTMLElement.REL_SUCCESS); - ffs.get(0).assertContentEquals("https://example.com/a/b/c/js/scripts.js"); - } - - @Test - public void testExtractAttributeFromElementAbsoluteUrlWithEmptyElResult() throws Exception { - testRunner.setProperty(GetHTMLElement.CSS_SELECTOR, "script"); - testRunner.setProperty(GetHTMLElement.DESTINATION, GetHTMLElement.DESTINATION_CONTENT); - testRunner.setProperty(GetHTMLElement.OUTPUT_TYPE, GetHTMLElement.ELEMENT_ATTRIBUTE); - testRunner.setProperty(GetHTMLElement.ATTRIBUTE_KEY, "abs:src"); - // Expression Language returns empty string because flow-file doesn't have contentUrl attribute. - testRunner.setProperty(GetHTMLElement.URL, "${contentUrl}"); - - testRunner.enqueue(new File("src/test/resources/Weather.html").toPath()); - testRunner.run(); - - testRunner.assertTransferCount(GetHTMLElement.REL_SUCCESS, 0); - testRunner.assertTransferCount(GetHTMLElement.REL_INVALID_HTML, 1); - testRunner.assertTransferCount(GetHTMLElement.REL_ORIGINAL, 0); - testRunner.assertTransferCount(GetHTMLElement.REL_NOT_FOUND, 0); - } - - @Test - public void testExtractTextFromElement() throws Exception { - testRunner.setProperty(GetHTMLElement.CSS_SELECTOR, "#" + ATL_ID); - testRunner.setProperty(GetHTMLElement.DESTINATION, GetHTMLElement.DESTINATION_CONTENT); - testRunner.setProperty(GetHTMLElement.OUTPUT_TYPE, GetHTMLElement.ELEMENT_TEXT); - - testRunner.enqueue(new File("src/test/resources/Weather.html").toPath()); - testRunner.run(); - - testRunner.assertTransferCount(GetHTMLElement.REL_SUCCESS, 1); - testRunner.assertTransferCount(GetHTMLElement.REL_INVALID_HTML, 0); - testRunner.assertTransferCount(GetHTMLElement.REL_ORIGINAL, 1); - testRunner.assertTransferCount(GetHTMLElement.REL_NOT_FOUND, 0); - - List ffs = testRunner.getFlowFilesForRelationship(GetHTMLElement.REL_SUCCESS); - ffs.get(0).assertContentEquals(ATL_WEATHER_TEXT); - } - - @Test - public void testExtractHTMLFromElement() throws Exception { - testRunner.setProperty(GetHTMLElement.CSS_SELECTOR, "#" + GDR_ID); - testRunner.setProperty(GetHTMLElement.DESTINATION, GetHTMLElement.DESTINATION_CONTENT); - testRunner.setProperty(GetHTMLElement.OUTPUT_TYPE, GetHTMLElement.ELEMENT_HTML); - - testRunner.enqueue(new File("src/test/resources/Weather.html").toPath()); - testRunner.run(); - - testRunner.assertTransferCount(GetHTMLElement.REL_SUCCESS, 1); - testRunner.assertTransferCount(GetHTMLElement.REL_INVALID_HTML, 0); - testRunner.assertTransferCount(GetHTMLElement.REL_ORIGINAL, 1); - testRunner.assertTransferCount(GetHTMLElement.REL_NOT_FOUND, 0); - - List ffs = testRunner.getFlowFilesForRelationship(GetHTMLElement.REL_SUCCESS); - ffs.get(0).assertContentEquals(GDR_WEATHER_TEXT); - } -} diff --git a/nifi-nar-bundles/nifi-html-bundle/nifi-html-processors/src/test/java/org/apache/nifi/TestModifyHTMLElement.java b/nifi-nar-bundles/nifi-html-bundle/nifi-html-processors/src/test/java/org/apache/nifi/TestModifyHTMLElement.java deleted file mode 100644 index ba21693892..0000000000 --- a/nifi-nar-bundles/nifi-html-bundle/nifi-html-processors/src/test/java/org/apache/nifi/TestModifyHTMLElement.java +++ /dev/null @@ -1,204 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.nifi; - -import org.apache.commons.lang3.StringUtils; -import org.apache.nifi.util.MockFlowFile; -import org.apache.nifi.util.TestRunner; -import org.apache.nifi.util.TestRunners; -import org.jsoup.Jsoup; -import org.jsoup.nodes.Document; -import org.jsoup.nodes.Element; -import org.jsoup.select.Elements; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; - -import java.io.File; -import java.util.List; - -import static org.junit.jupiter.api.Assertions.assertNotNull; -import static org.junit.jupiter.api.Assertions.assertTrue; - -public class TestModifyHTMLElement extends AbstractHTMLTest { - - private TestRunner testRunner; - - @BeforeEach - public void init() { - testRunner = TestRunners.newTestRunner(ModifyHTMLElement.class); - testRunner = TestRunners.newTestRunner(ModifyHTMLElement.class); - testRunner.setProperty(ModifyHTMLElement.URL, "http://localhost"); - testRunner.setProperty(ModifyHTMLElement.OUTPUT_TYPE, GetHTMLElement.ELEMENT_HTML); - testRunner.setProperty(ModifyHTMLElement.HTML_CHARSET, "UTF-8"); - } - - @Test - public void testModifyText() throws Exception { - final String MOD_VALUE = "Newly modified value to replace " + ATL_WEATHER_TEXT; - testRunner.setProperty(ModifyHTMLElement.CSS_SELECTOR, "#" + ATL_ID); - testRunner.setProperty(ModifyHTMLElement.OUTPUT_TYPE, ModifyHTMLElement.ELEMENT_TEXT); - testRunner.setProperty(ModifyHTMLElement.MODIFIED_VALUE, MOD_VALUE); - - testRunner.enqueue(new File("src/test/resources/Weather.html").toPath()); - testRunner.run(); - - testRunner.assertTransferCount(ModifyHTMLElement.REL_SUCCESS, 1); - testRunner.assertTransferCount(ModifyHTMLElement.REL_INVALID_HTML, 0); - testRunner.assertTransferCount(ModifyHTMLElement.REL_ORIGINAL, 1); - testRunner.assertTransferCount(ModifyHTMLElement.REL_NOT_FOUND, 0); - - List ffs = testRunner.getFlowFilesForRelationship(ModifyHTMLElement.REL_SUCCESS); - assertTrue(ffs.size() == 1); - String data = new String(testRunner.getContentAsByteArray(ffs.get(0))); - - //Contents will be the entire HTML doc. So lets use Jsoup again just the grab the element we want. - Document doc = Jsoup.parse(data); - Elements eles = doc.select("#" + ATL_ID); - Element ele = eles.get(0); - - assertTrue(StringUtils.equals(MOD_VALUE, ele.text())); - } - - @Test - public void testModifyHTMLWithExpressionLanguage() throws Exception { - - final String MOD_VALUE = "Newly modified value to replace " + ATL_WEATHER_TEXT; - - testRunner.setProperty(ModifyHTMLElement.CSS_SELECTOR, "#" + ATL_ID); - testRunner.setProperty(ModifyHTMLElement.OUTPUT_TYPE, ModifyHTMLElement.ELEMENT_TEXT); - testRunner.setProperty(ModifyHTMLElement.MODIFIED_VALUE, "${\" " + MOD_VALUE + " \":trim()}"); - - testRunner.enqueue(new File("src/test/resources/Weather.html").toPath()); - testRunner.run(); - - testRunner.assertTransferCount(ModifyHTMLElement.REL_SUCCESS, 1); - testRunner.assertTransferCount(ModifyHTMLElement.REL_INVALID_HTML, 0); - testRunner.assertTransferCount(ModifyHTMLElement.REL_ORIGINAL, 1); - testRunner.assertTransferCount(ModifyHTMLElement.REL_NOT_FOUND, 0); - - List ffs = testRunner.getFlowFilesForRelationship(ModifyHTMLElement.REL_SUCCESS); - assertTrue(ffs.size() == 1); - String data = new String(testRunner.getContentAsByteArray(ffs.get(0))); - - //Contents will be the entire HTML doc. So lets use Jsoup again just the grab the element we want. - Document doc = Jsoup.parse(data); - Elements eles = doc.select("#" + ATL_ID); - Element ele = eles.get(0); - - assertNotNull(ele.text()); - } - - @Test - public void testModifyHTML() throws Exception { - final String MOD_VALUE = "Newly modified HTML to replace " + GDR_WEATHER_TEXT; - testRunner.setProperty(ModifyHTMLElement.CSS_SELECTOR, "#" + GDR_ID); - testRunner.setProperty(ModifyHTMLElement.OUTPUT_TYPE, ModifyHTMLElement.ELEMENT_HTML); - testRunner.setProperty(ModifyHTMLElement.MODIFIED_VALUE, MOD_VALUE); - - testRunner.enqueue(new File("src/test/resources/Weather.html").toPath()); - testRunner.run(); - - testRunner.assertTransferCount(ModifyHTMLElement.REL_SUCCESS, 1); - testRunner.assertTransferCount(ModifyHTMLElement.REL_INVALID_HTML, 0); - testRunner.assertTransferCount(ModifyHTMLElement.REL_ORIGINAL, 1); - testRunner.assertTransferCount(ModifyHTMLElement.REL_NOT_FOUND, 0); - - List ffs = testRunner.getFlowFilesForRelationship(ModifyHTMLElement.REL_SUCCESS); - assertTrue(ffs.size() == 1); - String data = new String(testRunner.getContentAsByteArray(ffs.get(0))); - - //Contents will be the entire HTML doc. So lets use Jsoup again just the grab the element we want. - Document doc = Jsoup.parse(data); - Elements eles = doc.select("#" + GDR_ID); - Element ele = eles.get(0); - - assertTrue(StringUtils.equals(MOD_VALUE, ele.html())); - } - - @Test - public void testModifyAttribute() throws Exception { - final String MOD_VALUE = "http://localhost/newlink"; - testRunner.setProperty(ModifyHTMLElement.CSS_SELECTOR, "#" + GDR_ID); - testRunner.setProperty(ModifyHTMLElement.OUTPUT_TYPE, ModifyHTMLElement.ELEMENT_ATTRIBUTE); - testRunner.setProperty(ModifyHTMLElement.ATTRIBUTE_KEY, "href"); - testRunner.setProperty(ModifyHTMLElement.MODIFIED_VALUE, MOD_VALUE); - - testRunner.enqueue(new File("src/test/resources/Weather.html").toPath()); - testRunner.run(); - - testRunner.assertTransferCount(ModifyHTMLElement.REL_SUCCESS, 1); - testRunner.assertTransferCount(ModifyHTMLElement.REL_INVALID_HTML, 0); - testRunner.assertTransferCount(ModifyHTMLElement.REL_ORIGINAL, 1); - testRunner.assertTransferCount(ModifyHTMLElement.REL_NOT_FOUND, 0); - - List ffs = testRunner.getFlowFilesForRelationship(ModifyHTMLElement.REL_SUCCESS); - assertTrue(ffs.size() == 1); - String data = new String(testRunner.getContentAsByteArray(ffs.get(0))); - - //Contents will be the entire HTML doc. So lets use Jsoup again just the grab the element we want. - Document doc = Jsoup.parse(data); - Elements eles = doc.select("#" + GDR_ID); - Element ele = eles.get(0); - - assertTrue(StringUtils.equals(MOD_VALUE, ele.attr("href"))); - } - - @Test - public void testModifyElementNotFound() throws Exception { - final String MOD_VALUE = "http://localhost/newlink"; - testRunner.setProperty(ModifyHTMLElement.CSS_SELECTOR, "b"); - testRunner.setProperty(ModifyHTMLElement.OUTPUT_TYPE, ModifyHTMLElement.ELEMENT_HTML); - testRunner.setProperty(ModifyHTMLElement.MODIFIED_VALUE, MOD_VALUE); - - testRunner.enqueue(new File("src/test/resources/Weather.html").toPath()); - testRunner.run(); - - testRunner.assertTransferCount(ModifyHTMLElement.REL_SUCCESS, 0); - testRunner.assertTransferCount(ModifyHTMLElement.REL_INVALID_HTML, 0); - testRunner.assertTransferCount(ModifyHTMLElement.REL_ORIGINAL, 0); - testRunner.assertTransferCount(ModifyHTMLElement.REL_NOT_FOUND, 1); - } - - @Test - public void testModifyValueContainsHTMLCharacters() throws Exception { - final String MOD_VALUE = "Text that contains > and < characters"; - testRunner.setProperty(ModifyHTMLElement.CSS_SELECTOR, "#" + GDR_ID); - testRunner.setProperty(ModifyHTMLElement.OUTPUT_TYPE, ModifyHTMLElement.ELEMENT_HTML); - testRunner.setProperty(ModifyHTMLElement.MODIFIED_VALUE, MOD_VALUE); - - testRunner.enqueue(new File("src/test/resources/Weather.html").toPath()); - testRunner.run(); - - testRunner.assertTransferCount(ModifyHTMLElement.REL_SUCCESS, 1); - testRunner.assertTransferCount(ModifyHTMLElement.REL_INVALID_HTML, 0); - testRunner.assertTransferCount(ModifyHTMLElement.REL_ORIGINAL, 1); - testRunner.assertTransferCount(ModifyHTMLElement.REL_NOT_FOUND, 0); - - List ffs = testRunner.getFlowFilesForRelationship(ModifyHTMLElement.REL_SUCCESS); - assertTrue(ffs.size() == 1); - String data = new String(testRunner.getContentAsByteArray(ffs.get(0))); - - //Contents will be the entire HTML doc. So lets use Jsoup again just the grab the element we want. - Document doc = Jsoup.parse(data); - Elements eles = doc.select("#" + GDR_ID); - Element ele = eles.get(0); - - assertTrue(StringUtils.equals(MOD_VALUE, ele.text())); - assertTrue(StringUtils.equals(MOD_VALUE.replace(">", ">").replace("<", "<"), ele.html())); - } - -} diff --git a/nifi-nar-bundles/nifi-html-bundle/nifi-html-processors/src/test/java/org/apache/nifi/TestPutHTMLElement.java b/nifi-nar-bundles/nifi-html-bundle/nifi-html-processors/src/test/java/org/apache/nifi/TestPutHTMLElement.java deleted file mode 100644 index e9a7fcb926..0000000000 --- a/nifi-nar-bundles/nifi-html-bundle/nifi-html-processors/src/test/java/org/apache/nifi/TestPutHTMLElement.java +++ /dev/null @@ -1,127 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.nifi; - -import org.apache.commons.lang3.StringUtils; -import org.apache.nifi.util.MockFlowFile; -import org.apache.nifi.util.TestRunner; -import org.apache.nifi.util.TestRunners; -import org.jsoup.Jsoup; -import org.jsoup.nodes.Document; -import org.jsoup.nodes.Element; -import org.jsoup.select.Elements; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; - -import java.io.File; -import java.util.List; - -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertTrue; - -public class TestPutHTMLElement extends AbstractHTMLTest { - - private TestRunner testRunner; - - @BeforeEach - public void init() { - testRunner = TestRunners.newTestRunner(PutHTMLElement.class); - testRunner.setProperty(PutHTMLElement.URL, "http://localhost"); - } - - @Test - public void testAddNewElementToRoot() throws Exception { - final String MOD_VALUE = "

modified value

"; - testRunner.setProperty(PutHTMLElement.CSS_SELECTOR, "body"); - testRunner.setProperty(PutHTMLElement.PUT_LOCATION_TYPE, PutHTMLElement.PREPEND_ELEMENT); - testRunner.setProperty(PutHTMLElement.PUT_VALUE, MOD_VALUE); - - testRunner.enqueue(new File("src/test/resources/Weather.html").toPath()); - testRunner.run(); - - testRunner.assertTransferCount(PutHTMLElement.REL_SUCCESS, 1); - testRunner.assertTransferCount(PutHTMLElement.REL_INVALID_HTML, 0); - testRunner.assertTransferCount(PutHTMLElement.REL_ORIGINAL, 1); - testRunner.assertTransferCount(PutHTMLElement.REL_NOT_FOUND, 0); - - List ffs = testRunner.getFlowFilesForRelationship(PutHTMLElement.REL_SUCCESS); - assertEquals(1, ffs.size()); - String data = new String(testRunner.getContentAsByteArray(ffs.get(0))); - - //Contents will be the entire HTML doc. So lets use Jsoup again just the grab the element we want. - Document doc = Jsoup.parse(data); - Elements eles = doc.select("body > p"); - Element ele = eles.get(0); - - assertTrue(StringUtils.equals(MOD_VALUE.replace("

", "").replace("

", ""), ele.html())); - } - - @Test - public void testPrependPElementToDiv() throws Exception { - final String MOD_VALUE = "

modified value

"; - testRunner.setProperty(PutHTMLElement.CSS_SELECTOR, "#put"); - testRunner.setProperty(PutHTMLElement.PUT_LOCATION_TYPE, PutHTMLElement.PREPEND_ELEMENT); - testRunner.setProperty(PutHTMLElement.PUT_VALUE, MOD_VALUE); - - testRunner.enqueue(new File("src/test/resources/Weather.html").toPath()); - testRunner.run(); - - testRunner.assertTransferCount(PutHTMLElement.REL_SUCCESS, 1); - testRunner.assertTransferCount(PutHTMLElement.REL_INVALID_HTML, 0); - testRunner.assertTransferCount(PutHTMLElement.REL_ORIGINAL, 1); - testRunner.assertTransferCount(PutHTMLElement.REL_NOT_FOUND, 0); - - List ffs = testRunner.getFlowFilesForRelationship(PutHTMLElement.REL_SUCCESS); - assertEquals(1, ffs.size()); - String data = new String(testRunner.getContentAsByteArray(ffs.get(0))); - - //Contents will be the entire HTML doc. So lets use Jsoup again just the grab the element we want. - Document doc = Jsoup.parse(data); - Elements eles = doc.select("#put"); - Element ele = eles.get(0); - - assertEquals("

modified value

", ele.html()); - } - - @Test - public void testAppendPElementToDiv() throws Exception { - final String MOD_VALUE = "

modified value

"; - testRunner.setProperty(PutHTMLElement.CSS_SELECTOR, "#put"); - testRunner.setProperty(PutHTMLElement.PUT_LOCATION_TYPE, PutHTMLElement.APPEND_ELEMENT); - testRunner.setProperty(PutHTMLElement.PUT_VALUE, MOD_VALUE); - - testRunner.enqueue(new File("src/test/resources/Weather.html").toPath()); - testRunner.run(); - - testRunner.assertTransferCount(PutHTMLElement.REL_SUCCESS, 1); - testRunner.assertTransferCount(PutHTMLElement.REL_INVALID_HTML, 0); - testRunner.assertTransferCount(PutHTMLElement.REL_ORIGINAL, 1); - testRunner.assertTransferCount(PutHTMLElement.REL_NOT_FOUND, 0); - - List ffs = testRunner.getFlowFilesForRelationship(PutHTMLElement.REL_SUCCESS); - assertEquals(1, ffs.size()); - String data = new String(testRunner.getContentAsByteArray(ffs.get(0))); - - //Contents will be the entire HTML doc. So lets use Jsoup again just the grab the element we want. - Document doc = Jsoup.parse(data); - Elements eles = doc.select("#put"); - Element ele = eles.get(0); - - assertEquals("

modified value

", StringUtils.remove(ele.html(), "\n")); - } - -} diff --git a/nifi-nar-bundles/nifi-html-bundle/nifi-html-processors/src/test/resources/Weather.html b/nifi-nar-bundles/nifi-html-bundle/nifi-html-processors/src/test/resources/Weather.html deleted file mode 100644 index a715a1dd67..0000000000 --- a/nifi-nar-bundles/nifi-html-bundle/nifi-html-processors/src/test/resources/Weather.html +++ /dev/null @@ -1,25 +0,0 @@ - - - - NiFi HTML Parsing Demo - - - - - - - - - - -

Check out this weather! - Atlanta Weather -

-

I guess it could be colder ... - Grand Rapids Weather -

-
- - \ No newline at end of file diff --git a/nifi-nar-bundles/nifi-html-bundle/pom.xml b/nifi-nar-bundles/nifi-html-bundle/pom.xml deleted file mode 100644 index ae5422aaf1..0000000000 --- a/nifi-nar-bundles/nifi-html-bundle/pom.xml +++ /dev/null @@ -1,42 +0,0 @@ - - - - 4.0.0 - - - org.apache.nifi - nifi-nar-bundles - 2.0.0-SNAPSHOT - - - nifi-html-bundle - pom - - - nifi-html-processors - nifi-html-nar - - - - - - org.apache.nifi - nifi-html-processors - 2.0.0-SNAPSHOT - - - - diff --git a/nifi-nar-bundles/nifi-metrics-reporting-bundle/nifi-metrics-reporter-service-api-nar/pom.xml b/nifi-nar-bundles/nifi-metrics-reporting-bundle/nifi-metrics-reporter-service-api-nar/pom.xml deleted file mode 100644 index 31d649949e..0000000000 --- a/nifi-nar-bundles/nifi-metrics-reporting-bundle/nifi-metrics-reporter-service-api-nar/pom.xml +++ /dev/null @@ -1,39 +0,0 @@ - - - - - nifi-metrics-reporting-bundle - org.apache.nifi - 2.0.0-SNAPSHOT - - nar - 4.0.0 - - nifi-metrics-reporter-service-api-nar - - - org.apache.nifi - nifi-standard-services-api-nar - 2.0.0-SNAPSHOT - nar - - - org.apache.nifi - nifi-metrics-reporter-service-api - 2.0.0-SNAPSHOT - - - diff --git a/nifi-nar-bundles/nifi-metrics-reporting-bundle/nifi-metrics-reporter-service-api-nar/src/main/resources/META-INF/NOTICE b/nifi-nar-bundles/nifi-metrics-reporting-bundle/nifi-metrics-reporter-service-api-nar/src/main/resources/META-INF/NOTICE deleted file mode 100644 index f4828546c5..0000000000 --- a/nifi-nar-bundles/nifi-metrics-reporting-bundle/nifi-metrics-reporter-service-api-nar/src/main/resources/META-INF/NOTICE +++ /dev/null @@ -1,30 +0,0 @@ -nifi-metrics-reporter-service-api-nar -Copyright 2015-2020 The Apache Software Foundation - -This product includes software developed at -The Apache Software Foundation (http://www.apache.org/). - -****************** -Apache Software License v2 -****************** - -The following binary components are provided under the Apache Software License v2 - - (ASLv2) Dropwizard Metrics - The following NOTICE information applies: - Metrics - Copyright 2010-2013 Coda Hale and Yammer, Inc., 2014-2017 Dropwizard Team - This product includes software developed by Coda Hale and Yammer, Inc. - - This product includes code derived from the JSR-166 project (ThreadLocalRandom, Striped64, - LongAdder), which was released with the following comments: - - Written by Doug Lea with assistance from members of JCP JSR-166 - Expert Group and released to the public domain, as explained at - http://creativecommons.org/publicdomain/zero/1.0/ - - The derived work in the nifi-metrics module is adapted from - https://github.com/dropwizard/metrics/blob/v2.2.0/metrics-core/src/main/java/com/yammer/metrics/core/VirtualMachineMetrics.java - and can be found in - nifi-commons/nifi-metrics/src/main/java/org/apache/nifi/metrics/jvm/JvmMetrics.java - nifi-commons/nifi-metrics/src/main/java/org/apache/nifi/metrics/jvm/JmxJvmMetrics.java diff --git a/nifi-nar-bundles/nifi-metrics-reporting-bundle/nifi-metrics-reporter-service-api/pom.xml b/nifi-nar-bundles/nifi-metrics-reporting-bundle/nifi-metrics-reporter-service-api/pom.xml deleted file mode 100644 index 9a9010aa03..0000000000 --- a/nifi-nar-bundles/nifi-metrics-reporting-bundle/nifi-metrics-reporter-service-api/pom.xml +++ /dev/null @@ -1,32 +0,0 @@ - - - - - nifi-metrics-reporting-bundle - org.apache.nifi - 2.0.0-SNAPSHOT - - 4.0.0 - - nifi-metrics-reporter-service-api - - - - org.apache.nifi - nifi-metrics - - - diff --git a/nifi-nar-bundles/nifi-metrics-reporting-bundle/nifi-metrics-reporter-service-api/src/main/java/org/apache/nifi/metrics/reporting/reporter/service/MetricReporterService.java b/nifi-nar-bundles/nifi-metrics-reporting-bundle/nifi-metrics-reporter-service-api/src/main/java/org/apache/nifi/metrics/reporting/reporter/service/MetricReporterService.java deleted file mode 100644 index d87be15f83..0000000000 --- a/nifi-nar-bundles/nifi-metrics-reporting-bundle/nifi-metrics-reporter-service-api/src/main/java/org/apache/nifi/metrics/reporting/reporter/service/MetricReporterService.java +++ /dev/null @@ -1,40 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.nifi.metrics.reporting.reporter.service; - -import com.codahale.metrics.MetricRegistry; -import com.codahale.metrics.ScheduledReporter; -import org.apache.nifi.controller.ControllerService; -import org.apache.nifi.processor.exception.ProcessException; - -/** - * An interface for controller services used by MetricsReportingTask. In order to report to a new - * client, implement this interface and make sure to return the desired implementation of {@link ScheduledReporter}. - * - * @author Omer Hadari - */ -public interface MetricReporterService extends ControllerService { - - /** - * Create a reporter to a metric client (i.e. graphite). - * - * @param metricRegistry registry with the metrics to report. - * @return an instance of the reporter. - * @throws ProcessException if there was an error creating the reporter. - */ - ScheduledReporter createReporter(MetricRegistry metricRegistry) throws ProcessException; -} diff --git a/nifi-nar-bundles/nifi-metrics-reporting-bundle/nifi-metrics-reporting-nar/pom.xml b/nifi-nar-bundles/nifi-metrics-reporting-bundle/nifi-metrics-reporting-nar/pom.xml deleted file mode 100644 index d752cd98d6..0000000000 --- a/nifi-nar-bundles/nifi-metrics-reporting-bundle/nifi-metrics-reporting-nar/pom.xml +++ /dev/null @@ -1,40 +0,0 @@ - - - - - nifi-metrics-reporting-bundle - org.apache.nifi - 2.0.0-SNAPSHOT - - nar - 4.0.0 - - nifi-metrics-reporting-nar - - - - org.apache.nifi - nifi-metrics-reporting-task - 2.0.0-SNAPSHOT - - - org.apache.nifi - nifi-metrics-reporter-service-api-nar - 2.0.0-SNAPSHOT - nar - - - diff --git a/nifi-nar-bundles/nifi-metrics-reporting-bundle/nifi-metrics-reporting-nar/src/main/resources/META-INF/NOTICE b/nifi-nar-bundles/nifi-metrics-reporting-bundle/nifi-metrics-reporting-nar/src/main/resources/META-INF/NOTICE deleted file mode 100644 index 1eaddb63f7..0000000000 --- a/nifi-nar-bundles/nifi-metrics-reporting-bundle/nifi-metrics-reporting-nar/src/main/resources/META-INF/NOTICE +++ /dev/null @@ -1,30 +0,0 @@ -nifi-metrics-reporting-nar -Copyright 2015-2020 The Apache Software Foundation - -This product includes software developed at -The Apache Software Foundation (http://www.apache.org/). - -****************** -Apache Software License v2 -****************** - -The following binary components are provided under the Apache Software License v2 - - (ASLv2) Dropwizard Metrics - The following NOTICE information applies: - Metrics - Copyright 2010-2013 Coda Hale and Yammer, Inc., 2014-2017 Dropwizard Team - This product includes software developed by Coda Hale and Yammer, Inc. - - This product includes code derived from the JSR-166 project (ThreadLocalRandom, Striped64, - LongAdder), which was released with the following comments: - - Written by Doug Lea with assistance from members of JCP JSR-166 - Expert Group and released to the public domain, as explained at - http://creativecommons.org/publicdomain/zero/1.0/ - - The derived work in the nifi-metrics module is adapted from - https://github.com/dropwizard/metrics/blob/v2.2.0/metrics-core/src/main/java/com/yammer/metrics/core/VirtualMachineMetrics.java - and can be found in - nifi-commons/nifi-metrics/src/main/java/org/apache/nifi/metrics/jvm/JvmMetrics.java - nifi-commons/nifi-metrics/src/main/java/org/apache/nifi/metrics/jvm/JmxJvmMetrics.java diff --git a/nifi-nar-bundles/nifi-metrics-reporting-bundle/nifi-metrics-reporting-task/pom.xml b/nifi-nar-bundles/nifi-metrics-reporting-bundle/nifi-metrics-reporting-task/pom.xml deleted file mode 100644 index ad21f118f3..0000000000 --- a/nifi-nar-bundles/nifi-metrics-reporting-bundle/nifi-metrics-reporting-task/pom.xml +++ /dev/null @@ -1,54 +0,0 @@ - - - - - nifi-metrics-reporting-bundle - org.apache.nifi - 2.0.0-SNAPSHOT - - 4.0.0 - - nifi-metrics-reporting-task - - - - org.apache.nifi - nifi-utils - 2.0.0-SNAPSHOT - - - org.apache.nifi - nifi-metrics-reporter-service-api - 2.0.0-SNAPSHOT - provided - - - io.dropwizard.metrics - metrics-graphite - 4.2.19 - - - org.apache.nifi - nifi-metrics - - - org.apache.nifi - nifi-mock - 2.0.0-SNAPSHOT - test - - - diff --git a/nifi-nar-bundles/nifi-metrics-reporting-bundle/nifi-metrics-reporting-task/src/main/java/org/apache/nifi/metrics/FlowMetricSet.java b/nifi-nar-bundles/nifi-metrics-reporting-bundle/nifi-metrics-reporting-task/src/main/java/org/apache/nifi/metrics/FlowMetricSet.java deleted file mode 100644 index d6e2fbab8c..0000000000 --- a/nifi-nar-bundles/nifi-metrics-reporting-bundle/nifi-metrics-reporting-task/src/main/java/org/apache/nifi/metrics/FlowMetricSet.java +++ /dev/null @@ -1,95 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.nifi.metrics; - -import com.codahale.metrics.Gauge; -import com.codahale.metrics.Metric; -import com.codahale.metrics.MetricSet; -import org.apache.nifi.controller.status.ProcessGroupStatus; -import org.apache.nifi.controller.status.ProcessorStatus; - -import java.util.HashMap; -import java.util.Map; -import java.util.concurrent.atomic.AtomicReference; - -/** - * A metric set of NiFi instance related metrics. - * - * @author Omer Hadari - */ -public class FlowMetricSet implements MetricSet { - - - /** - * Reference to the process status that should be reported. Should be updated when the status changes. - */ - private final AtomicReference currentStatusReference; - - /** - * Create a metric set that will look at a given process status reference for deciding metrics. - * - * @param currentStatusReference a reference to the process status. - */ - public FlowMetricSet(AtomicReference currentStatusReference) { - this.currentStatusReference = currentStatusReference; - } - - /** - * Create a map of {@link Gauge}s for the {@link #currentStatusReference}. This methods reports the metrics as - * found in the reference. - * - * @return map between the metric name and a {@link Gauge} to it's value. - */ - @Override - public Map getMetrics() { - - Map metrics = new HashMap<>(); - - metrics.put(MetricNames.ACTIVE_THREADS, (Gauge) () -> currentStatusReference.get().getActiveThreadCount()); - metrics.put(MetricNames.BYTES_QUEUED, (Gauge) () -> currentStatusReference.get().getQueuedContentSize()); - metrics.put(MetricNames.BYTES_READ, (Gauge) () -> currentStatusReference.get().getBytesRead()); - metrics.put(MetricNames.BYTES_RECEIVED, (Gauge) () -> currentStatusReference.get().getBytesReceived()); - metrics.put(MetricNames.BYTES_SENT, (Gauge) () -> currentStatusReference.get().getBytesSent()); - metrics.put(MetricNames.BYTES_WRITTEN, (Gauge) () -> currentStatusReference.get().getBytesWritten()); - metrics.put(MetricNames.FLOW_FILES_RECEIVED, (Gauge) () -> currentStatusReference.get().getFlowFilesReceived()); - metrics.put(MetricNames.FLOW_FILES_QUEUED, (Gauge) () -> currentStatusReference.get().getQueuedCount()); - metrics.put(MetricNames.FLOW_FILES_SENT, (Gauge) () -> currentStatusReference.get().getFlowFilesSent()); - metrics.put(MetricNames.TOTAL_TASK_DURATION_NANOS, (Gauge) () -> calculateProcessingNanos(currentStatusReference.get())); - - return metrics; - } - - /** - * Calculate the total processing time of a process group. - * - * @param status the current process group status. - * @return the total amount of nanoseconds spent in each processor in the process group. - */ - private long calculateProcessingNanos(final ProcessGroupStatus status) { - long nanos = 0L; - - for (final ProcessorStatus procStats : status.getProcessorStatus()) { - nanos += procStats.getProcessingNanos(); - } - - for (final ProcessGroupStatus childGroupStatus : status.getProcessGroupStatus()) { - nanos += calculateProcessingNanos(childGroupStatus); - } - - return nanos; - } -} diff --git a/nifi-nar-bundles/nifi-metrics-reporting-bundle/nifi-metrics-reporting-task/src/main/java/org/apache/nifi/metrics/MetricNames.java b/nifi-nar-bundles/nifi-metrics-reporting-bundle/nifi-metrics-reporting-task/src/main/java/org/apache/nifi/metrics/MetricNames.java deleted file mode 100644 index fa06b8b984..0000000000 --- a/nifi-nar-bundles/nifi-metrics-reporting-bundle/nifi-metrics-reporting-task/src/main/java/org/apache/nifi/metrics/MetricNames.java +++ /dev/null @@ -1,35 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.nifi.metrics; - -/** - * The Metric names to send to Ambari. - */ -public interface MetricNames { - - // NiFi Metrics - String FLOW_FILES_RECEIVED = "FlowFilesReceivedLast5Minutes"; - String BYTES_RECEIVED = "BytesReceivedLast5Minutes"; - String FLOW_FILES_SENT = "FlowFilesSentLast5Minutes"; - String BYTES_SENT = "BytesSentLast5Minutes"; - String FLOW_FILES_QUEUED = "FlowFilesQueued"; - String BYTES_QUEUED = "BytesQueued"; - String BYTES_READ = "BytesReadLast5Minutes"; - String BYTES_WRITTEN = "BytesWrittenLast5Minutes"; - String ACTIVE_THREADS = "ActiveThreads"; - String TOTAL_TASK_DURATION_NANOS = "TotalTaskDurationNanoSeconds"; -} diff --git a/nifi-nar-bundles/nifi-metrics-reporting-bundle/nifi-metrics-reporting-task/src/main/java/org/apache/nifi/metrics/reporting/reporter/service/GraphiteMetricReporterService.java b/nifi-nar-bundles/nifi-metrics-reporting-bundle/nifi-metrics-reporting-task/src/main/java/org/apache/nifi/metrics/reporting/reporter/service/GraphiteMetricReporterService.java deleted file mode 100644 index 32c644938d..0000000000 --- a/nifi-nar-bundles/nifi-metrics-reporting-bundle/nifi-metrics-reporting-task/src/main/java/org/apache/nifi/metrics/reporting/reporter/service/GraphiteMetricReporterService.java +++ /dev/null @@ -1,181 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.nifi.metrics.reporting.reporter.service; - -import com.codahale.metrics.MetricRegistry; -import com.codahale.metrics.ScheduledReporter; -import com.codahale.metrics.graphite.Graphite; -import com.codahale.metrics.graphite.GraphiteReporter; -import com.codahale.metrics.graphite.GraphiteSender; -import org.apache.nifi.annotation.documentation.CapabilityDescription; -import org.apache.nifi.annotation.documentation.Tags; -import org.apache.nifi.annotation.lifecycle.OnDisabled; -import org.apache.nifi.annotation.lifecycle.OnEnabled; -import org.apache.nifi.components.PropertyDescriptor; -import org.apache.nifi.controller.AbstractControllerService; -import org.apache.nifi.controller.ConfigurationContext; -import org.apache.nifi.expression.ExpressionLanguageScope; -import org.apache.nifi.metrics.reporting.task.MetricsReportingTask; -import org.apache.nifi.processor.util.StandardValidators; - -import javax.net.SocketFactory; -import java.io.IOException; -import java.nio.charset.Charset; -import java.util.ArrayList; -import java.util.Collections; -import java.util.List; - -/** - * A controller service that provides metric reporters for graphite, can be used by {@link MetricsReportingTask}. - * - * @author Omer Hadari - */ -@Tags({"metrics", "reporting", "graphite"}) -@CapabilityDescription("A controller service that provides metric reporters for graphite. " + - "Used by MetricsReportingTask.") -public class GraphiteMetricReporterService extends AbstractControllerService implements MetricReporterService { - - /** - * Points to the hostname of the graphite listener. - */ - public static final PropertyDescriptor HOST = new PropertyDescriptor.Builder() - .name("host") - .displayName("Host") - .description("The hostname of the carbon listener") - .required(true) - .addValidator(StandardValidators.URI_VALIDATOR) - .expressionLanguageSupported(ExpressionLanguageScope.VARIABLE_REGISTRY) - .build(); - - /** - * Points to the port on which the graphite server listens. - */ - public static final PropertyDescriptor PORT = new PropertyDescriptor.Builder() - .name("port") - .displayName("Port") - .description("The port on which carbon listens") - .required(true) - .addValidator(StandardValidators.PORT_VALIDATOR) - .expressionLanguageSupported(ExpressionLanguageScope.VARIABLE_REGISTRY) - .build(); - - /** - * Points to the charset name that the graphite server expects. - */ - public static final PropertyDescriptor CHARSET = new PropertyDescriptor.Builder() - .name("charset") - .displayName("Charset") - .description("The charset used by the graphite server") - .required(true) - .defaultValue("UTF-8") - .addValidator(StandardValidators.CHARACTER_SET_VALIDATOR) - .build(); - - /** - * Prefix for all metric names sent by reporters - for separation of NiFi stats in graphite. - */ - protected static final PropertyDescriptor METRIC_NAME_PREFIX = new PropertyDescriptor.Builder() - .name("metric name prefix") - .displayName("Metric Name Prefix") - .description("A prefix that will be used for all metric names sent by reporters provided by this service.") - .required(true) - .defaultValue("nifi") - .addValidator(StandardValidators.NON_BLANK_VALIDATOR) - .expressionLanguageSupported(ExpressionLanguageScope.VARIABLE_REGISTRY) - .build(); - - /** - * List of property descriptors used by the service. - */ - private static final List properties; - - static { - final List props = new ArrayList<>(); - props.add(HOST); - props.add(PORT); - props.add(CHARSET); - props.add(METRIC_NAME_PREFIX); - properties = Collections.unmodifiableList(props); - } - - /** - * Graphite sender, a connection to the server. - */ - private GraphiteSender graphiteSender; - - /** - * The configured {@link #METRIC_NAME_PREFIX} value. - */ - private String metricNamePrefix; - - /** - * Create the {@link #graphiteSender} according to configuration. - * - * @param context used to access properties. - */ - @OnEnabled - public void onEnabled(final ConfigurationContext context) { - String host = context.getProperty(HOST).evaluateAttributeExpressions().getValue(); - int port = context.getProperty(PORT).evaluateAttributeExpressions().asInteger(); - Charset charset = Charset.forName(context.getProperty(CHARSET).getValue()); - graphiteSender = createSender(host, port, charset); - metricNamePrefix = context.getProperty(METRIC_NAME_PREFIX).evaluateAttributeExpressions().getValue(); - } - - /** - * Close the graphite sender. - * - * @throws IOException if failed to close the connection. - */ - @OnDisabled - public void shutdown() throws IOException { - try { - graphiteSender.close(); - } finally { - graphiteSender = null; - } - } - - /** - * Use the {@link #graphiteSender} in order to create a reporter. - * - * @param metricRegistry registry with the metrics to report. - * @return a reporter instance. - */ - @Override - public ScheduledReporter createReporter(MetricRegistry metricRegistry) { - return GraphiteReporter.forRegistry(metricRegistry).prefixedWith(metricNamePrefix).build(graphiteSender); - - } - - /** - * Create a sender. - * - * @param host the hostname of the server to connect to. - * @param port the port on which the server listens. - * @param charset the charset in which the server expects logs. - * @return The created sender. - */ - protected GraphiteSender createSender(String host, int port, Charset charset) { - return new Graphite(host, port, SocketFactory.getDefault(), charset); - } - - @Override - protected List getSupportedPropertyDescriptors() { - return properties; - } -} diff --git a/nifi-nar-bundles/nifi-metrics-reporting-bundle/nifi-metrics-reporting-task/src/main/java/org/apache/nifi/metrics/reporting/task/MetricsReportingTask.java b/nifi-nar-bundles/nifi-metrics-reporting-bundle/nifi-metrics-reporting-task/src/main/java/org/apache/nifi/metrics/reporting/task/MetricsReportingTask.java deleted file mode 100644 index dfd6f7916b..0000000000 --- a/nifi-nar-bundles/nifi-metrics-reporting-bundle/nifi-metrics-reporting-task/src/main/java/org/apache/nifi/metrics/reporting/task/MetricsReportingTask.java +++ /dev/null @@ -1,152 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.nifi.metrics.reporting.task; - -import com.codahale.metrics.MetricRegistry; -import com.codahale.metrics.ScheduledReporter; -import com.codahale.metrics.jvm.MemoryUsageGaugeSet; -import org.apache.nifi.annotation.documentation.CapabilityDescription; -import org.apache.nifi.annotation.documentation.Tags; -import org.apache.nifi.annotation.lifecycle.OnScheduled; -import org.apache.nifi.components.PropertyDescriptor; -import org.apache.nifi.controller.ConfigurationContext; -import org.apache.nifi.controller.status.ProcessGroupStatus; -import org.apache.nifi.expression.ExpressionLanguageScope; -import org.apache.nifi.metrics.FlowMetricSet; -import org.apache.nifi.metrics.reporting.reporter.service.MetricReporterService; -import org.apache.nifi.processor.util.StandardValidators; -import org.apache.nifi.reporting.AbstractReportingTask; -import org.apache.nifi.reporting.ReportingContext; -import org.apache.nifi.reporting.ReportingInitializationContext; - -import java.util.ArrayList; -import java.util.Collections; -import java.util.List; -import java.util.concurrent.atomic.AtomicReference; - -/** - * A reporting task for NiFi instance and JVM related metrics. - *

- * This task reports metrics to services according to a provided {@link ScheduledReporter}, reached by using a - * {@link MetricReporterService}. In order to report to different clients, simply use different implementations of - * the controller service. - * - * @author Omer Hadari - * @see MetricReporterService - */ -@Tags({"metrics", "reporting"}) -@CapabilityDescription("This reporting task reports a set of metrics regarding the JVM and the NiFi instance" + - "to a reporter. The reporter is provided by a MetricReporterService. It can be optionally used for a specific" + - "process group if a property with the group id is provided.") -public class MetricsReportingTask extends AbstractReportingTask { - - /** - * Points to the service which provides {@link ScheduledReporter} instances. - */ - protected static final PropertyDescriptor REPORTER_SERVICE = new PropertyDescriptor.Builder() - .name("metric reporter service") - .displayName("Metric Reporter Service") - .description("The service that provides a reporter for the gathered metrics") - .identifiesControllerService(MetricReporterService.class) - .required(true) - .build(); - - /** - * Metrics of the process group with this ID should be reported. If not specified, use the root process group. - */ - protected static final PropertyDescriptor PROCESS_GROUP_ID = new PropertyDescriptor.Builder() - .name("process group id") - .displayName("Process Group ID") - .description("The id of the process group to report. If not specified, metrics of the root process group" + - "are reported.") - .required(false) - .expressionLanguageSupported(ExpressionLanguageScope.VARIABLE_REGISTRY) - .addValidator(StandardValidators.NON_EMPTY_VALIDATOR) - .build(); - - /** - * Contains the metrics that should be reported. - */ - private MetricRegistry metricRegistry; - - /** - * Used for actually reporting metrics. - */ - private ScheduledReporter reporter; - - // Protected for testing sake. DO NOT ACCESS FOR OTHER PURPOSES. - /** - * Points to the most recent process group status seen by this task. - */ - protected AtomicReference currentStatusReference; - - /** - * Register all wanted metrics to {@link #metricRegistry}. - *

- * {@inheritDoc} - */ - @Override - protected void init(ReportingInitializationContext config) { - metricRegistry = new MetricRegistry(); - currentStatusReference = new AtomicReference<>(); - metricRegistry.registerAll(new MemoryUsageGaugeSet()); - metricRegistry.registerAll(new FlowMetricSet(currentStatusReference)); - } - - /** - * Populate {@link #reporter} using the {@link MetricReporterService}. If the reporter is active already, - * do nothing. - * - * @param context used for accessing the controller service. - */ - @OnScheduled - public void connect(ConfigurationContext context) { - if (reporter == null) { - reporter = ((MetricReporterService) context.getProperty(REPORTER_SERVICE).asControllerService()) - .createReporter(metricRegistry); - } - } - - /** - * Report the registered metrics. - * - * @param context used for getting the most recent {@link ProcessGroupStatus}. - */ - @Override - public void onTrigger(ReportingContext context) { - String groupId = context.getProperty(PROCESS_GROUP_ID).evaluateAttributeExpressions().getValue(); - - ProcessGroupStatus statusToReport = groupId == null - ? context.getEventAccess().getControllerStatus() - : context.getEventAccess().getGroupStatus(groupId); - - if (statusToReport != null) { - currentStatusReference.set(statusToReport); - reporter.report(); - } else { - getLogger().error("Process group with provided group id could not be found."); - } - } - - @Override - protected List getSupportedPropertyDescriptors() { - List properties = new ArrayList<>(); - properties.add(REPORTER_SERVICE); - properties.add(PROCESS_GROUP_ID); - return Collections.unmodifiableList(properties); - } -} diff --git a/nifi-nar-bundles/nifi-metrics-reporting-bundle/nifi-metrics-reporting-task/src/main/resources/META-INF/services/org.apache.nifi.controller.ControllerService b/nifi-nar-bundles/nifi-metrics-reporting-bundle/nifi-metrics-reporting-task/src/main/resources/META-INF/services/org.apache.nifi.controller.ControllerService deleted file mode 100644 index 0ddc70ac8e..0000000000 --- a/nifi-nar-bundles/nifi-metrics-reporting-bundle/nifi-metrics-reporting-task/src/main/resources/META-INF/services/org.apache.nifi.controller.ControllerService +++ /dev/null @@ -1,15 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -org.apache.nifi.metrics.reporting.reporter.service.GraphiteMetricReporterService \ No newline at end of file diff --git a/nifi-nar-bundles/nifi-metrics-reporting-bundle/nifi-metrics-reporting-task/src/main/resources/META-INF/services/org.apache.nifi.reporting.ReportingTask b/nifi-nar-bundles/nifi-metrics-reporting-bundle/nifi-metrics-reporting-task/src/main/resources/META-INF/services/org.apache.nifi.reporting.ReportingTask deleted file mode 100644 index 8c554a4252..0000000000 --- a/nifi-nar-bundles/nifi-metrics-reporting-bundle/nifi-metrics-reporting-task/src/main/resources/META-INF/services/org.apache.nifi.reporting.ReportingTask +++ /dev/null @@ -1,15 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -org.apache.nifi.metrics.reporting.task.MetricsReportingTask diff --git a/nifi-nar-bundles/nifi-metrics-reporting-bundle/nifi-metrics-reporting-task/src/test/java/org/apache/nifi/metrics/reporting/reporter/service/GraphiteMetricReporterServiceTest.java b/nifi-nar-bundles/nifi-metrics-reporting-bundle/nifi-metrics-reporting-task/src/test/java/org/apache/nifi/metrics/reporting/reporter/service/GraphiteMetricReporterServiceTest.java deleted file mode 100644 index dff6ca7780..0000000000 --- a/nifi-nar-bundles/nifi-metrics-reporting-bundle/nifi-metrics-reporting-task/src/test/java/org/apache/nifi/metrics/reporting/reporter/service/GraphiteMetricReporterServiceTest.java +++ /dev/null @@ -1,209 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.nifi.metrics.reporting.reporter.service; - -import com.codahale.metrics.Gauge; -import com.codahale.metrics.MetricRegistry; -import com.codahale.metrics.ScheduledReporter; -import com.codahale.metrics.graphite.GraphiteSender; -import org.apache.nifi.processor.Processor; -import org.apache.nifi.util.TestRunner; -import org.apache.nifi.util.TestRunners; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.extension.ExtendWith; -import org.mockito.Mock; -import org.mockito.junit.jupiter.MockitoExtension; - -import java.nio.charset.Charset; -import java.nio.charset.StandardCharsets; - -import static org.mockito.ArgumentMatchers.anyLong; -import static org.mockito.ArgumentMatchers.eq; -import static org.mockito.Mockito.verify; - -/** - * Test class for {@link GraphiteMetricReporterService}. - * - * @author Omer Hadari - */ -@ExtendWith(MockitoExtension.class) -public class GraphiteMetricReporterServiceTest { - - /** - * Service identifier for registerting the tested service to the tests runner. - */ - private static final String SERVICE_IDENTIFIER = "graphite-metric-reporter-service"; - - /** - * Sample host name for the {@link GraphiteMetricReporterService#HOST} property. - */ - private static final String TEST_HOST = "some-host"; - - /** - * Sample port for the {@link GraphiteMetricReporterService#PORT} property. - */ - private static final int TEST_PORT = 12345; - - /** - * Sample charset for the {@link GraphiteMetricReporterService#CHARSET} property. - */ - private static final Charset TEST_CHARSET = StandardCharsets.UTF_16LE; - - /** - * Sample prefix for metric names. - */ - private static final String METRIC_NAMES_PREFIX = "test-metric-name-prefix"; - - /** - * Sample metric for verifying that a graphite sender with the correct configuration is used. - */ - private static final String TEST_METRIC_NAME = "test-metric"; - - /** - * The fixed value of {@link #TEST_METRIC_NAME}. - */ - private static final int TEST_METRIC_VALUE = 2; - - /** - * Dummy processor for creating {@link #runner}. - */ - @Mock - private Processor processorDummy; - - /** - * Mock sender for verifying creation with the correct configuration. - */ - @Mock - private GraphiteSender graphiteSenderMock; - - /** - * Stub metric registry, that contains the test metrics. - */ - private MetricRegistry metricRegistryStub; - - /** - * Test runner for activating and configuring the service. - */ - private TestRunner runner; - - /** - * The test subject. - */ - private GraphiteMetricReporterService testedService; - - /** - * Instantiate the runner and mocks between tests. Register metrics to the {@link #metricRegistryStub}. - */ - @BeforeEach - public void setUp() throws Exception { - runner = TestRunners.newTestRunner(processorDummy); - testedService = new GraphiteMetricReporterService(); - - metricRegistryStub = new MetricRegistry(); - metricRegistryStub.register(TEST_METRIC_NAME, ((Gauge) () -> TEST_METRIC_VALUE)); - - } - - - /** - * Make sure that a correctly configured service can be activated. - */ - @Test - public void testGraphiteMetricReporterSanityConfiguration() throws Exception { - runner.addControllerService(SERVICE_IDENTIFIER, testedService); - setServiceProperties(TEST_HOST, TEST_PORT, TEST_CHARSET, METRIC_NAMES_PREFIX); - runner.enableControllerService(testedService); - - runner.assertValid(testedService); - } - - - /** - * Make sure that a correctly configured service provides a reporter for the matching configuration, and - * actually reports to the correct address. - */ - @Test - public void testCreateReporterUsesCorrectSender() throws Exception { - testedService = new TestableGraphiteMetricReporterService(); - runner.addControllerService(SERVICE_IDENTIFIER, testedService); - setServiceProperties(TEST_HOST, TEST_PORT, TEST_CHARSET, METRIC_NAMES_PREFIX); - runner.enableControllerService(testedService); - - ScheduledReporter createdReporter = testedService.createReporter(metricRegistryStub); - createdReporter.report(); - - String expectedMetricName = MetricRegistry.name(METRIC_NAMES_PREFIX, TEST_METRIC_NAME); - verify(graphiteSenderMock).send(eq(expectedMetricName), eq(String.valueOf(TEST_METRIC_VALUE)), anyLong()); - } - - /** - * Make sure that {@link GraphiteMetricReporterService#shutdown()} closes the connection to graphite. - */ - @Test - public void testShutdownClosesSender() throws Exception { - testedService = new TestableGraphiteMetricReporterService(); - runner.addControllerService(SERVICE_IDENTIFIER, testedService); - setServiceProperties(TEST_HOST, TEST_PORT, TEST_CHARSET, METRIC_NAMES_PREFIX); - runner.enableControllerService(testedService); - runner.disableControllerService(testedService); - - verify(graphiteSenderMock).close(); - } - - /** - * Set the test subject's properties. - * - * @param host populates {@link GraphiteMetricReporterService#HOST}. - * @param port populates {@link GraphiteMetricReporterService#PORT}. - * @param charset populates {@link GraphiteMetricReporterService#CHARSET}. - * @param metricNamesPrefix populates {@link GraphiteMetricReporterService#METRIC_NAME_PREFIX}. - */ - private void setServiceProperties(String host, int port, Charset charset, String metricNamesPrefix) { - runner.setProperty(testedService, GraphiteMetricReporterService.HOST, host); - runner.setProperty(testedService, GraphiteMetricReporterService.PORT, String.valueOf(port)); - runner.setProperty(testedService, GraphiteMetricReporterService.CHARSET, charset.name()); - runner.setProperty(testedService, GraphiteMetricReporterService.METRIC_NAME_PREFIX, metricNamesPrefix); - } - - /** - * This class is a patch. It overrides {@link GraphiteMetricReporterService#createSender(String, int, Charset)} - * so that it is possible to verify a correct creation of graphite senders according to property values. - */ - private class TestableGraphiteMetricReporterService extends GraphiteMetricReporterService { - - /** - * Overrides the actual methods in order to inject the mock {@link #graphiteSenderMock}. - *

- * If this method is called with the test property values, it returns the mock. Otherwise operate - * regularly. - * - * @param host the provided hostname. - * @param port the provided port. - * @param charset the provided graphite server charset. - * @return {@link #graphiteSenderMock} if all params were the constant test params, regular result otherwise. - */ - @Override - protected GraphiteSender createSender(String host, int port, Charset charset) { - if (TEST_HOST.equals(host) && TEST_PORT == port && TEST_CHARSET.equals(charset)) { - return graphiteSenderMock; - - } - return super.createSender(host, port, charset); - } - } -} diff --git a/nifi-nar-bundles/nifi-metrics-reporting-bundle/nifi-metrics-reporting-task/src/test/java/org/apache/nifi/metrics/reporting/task/MetricsReportingTaskTest.java b/nifi-nar-bundles/nifi-metrics-reporting-bundle/nifi-metrics-reporting-task/src/test/java/org/apache/nifi/metrics/reporting/task/MetricsReportingTaskTest.java deleted file mode 100644 index 22f1385d16..0000000000 --- a/nifi-nar-bundles/nifi-metrics-reporting-bundle/nifi-metrics-reporting-task/src/test/java/org/apache/nifi/metrics/reporting/task/MetricsReportingTaskTest.java +++ /dev/null @@ -1,256 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.nifi.metrics.reporting.task; - -import com.codahale.metrics.Metric; -import com.codahale.metrics.MetricRegistry; -import com.codahale.metrics.ScheduledReporter; -import com.codahale.metrics.jvm.MemoryUsageGaugeSet; -import org.apache.nifi.components.PropertyDescriptor; -import org.apache.nifi.controller.ConfigurationContext; -import org.apache.nifi.controller.ControllerService; -import org.apache.nifi.controller.status.ProcessGroupStatus; -import org.apache.nifi.metrics.FlowMetricSet; -import org.apache.nifi.metrics.reporting.reporter.service.MetricReporterService; -import org.apache.nifi.reporting.ReportingContext; -import org.apache.nifi.reporting.ReportingInitializationContext; -import org.apache.nifi.state.MockStateManager; -import org.apache.nifi.util.MockComponentLog; -import org.apache.nifi.util.MockConfigurationContext; -import org.apache.nifi.util.MockReportingContext; -import org.apache.nifi.util.MockReportingInitializationContext; -import org.apache.nifi.util.MockVariableRegistry; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.extension.ExtendWith; -import org.mockito.ArgumentCaptor; -import org.mockito.Mock; -import org.mockito.junit.jupiter.MockitoExtension; -import org.mockito.junit.jupiter.MockitoSettings; -import org.mockito.quality.Strictness; - -import java.util.Arrays; -import java.util.HashMap; -import java.util.List; -import java.util.Map; - -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertNull; -import static org.junit.jupiter.api.Assertions.assertTrue; -import static org.mockito.ArgumentMatchers.any; -import static org.mockito.Mockito.never; -import static org.mockito.Mockito.times; -import static org.mockito.Mockito.verify; -import static org.mockito.Mockito.when; - -/** - * Test class for {@link MetricsReportingTask}. - * - * @author Omer Hadari - */ -@ExtendWith(MockitoExtension.class) -@MockitoSettings(strictness = Strictness.LENIENT) -public class MetricsReportingTaskTest { - - /** - * Identifier for {@link #reporterServiceStub}. - */ - private static final String REPORTER_SERVICE_IDENTIFIER = "reporter-service"; - - /** - * Id for the group with status {@link #innerGroupStatus}. - */ - private static final String TEST_GROUP_ID = "test-process-group-id"; - - /** - * Id for the {@link #reportingInitContextStub}. - */ - private static final String TEST_INIT_CONTEXT_ID = "test-init-context-id"; - - /** - * Name for {@link #reportingInitContextStub}. - */ - private static final String TEST_INIT_CONTEXT_NAME = "test-init-context-name"; - - /** - * Id for the tested tested reporting task. - */ - private static final String TEST_TASK_ID = "test-task-id"; - - - /** - * Stub context, used by {@link MetricsReportingTask#onTrigger(ReportingContext)} for reaching the status. - */ - private MockReportingContext reportingContextStub; - - /** - * Stub context, used by {@link MetricsReportingTask#connect(ConfigurationContext)} for reaching the service. - */ - private MockConfigurationContext configurationContextStub; - - /** - * Stub service for providing {@link #reporterMock}, used for actual reporting - */ - @Mock - private MetricReporterService reporterServiceStub; - - /** - * Mock reporter, used for verifying actual reporting. - */ - @Mock - private ScheduledReporter reporterMock; - - /** - * A status for the "root" process group. - */ - private ProcessGroupStatus rootGroupStatus; - - /** - * Same as {@link #rootGroupStatus}, used when {@link MetricsReportingTask#PROCESS_GROUP_ID} is set. - */ - private ProcessGroupStatus innerGroupStatus; - - /** - * Stub initialization context for calling {@link MetricsReportingTask#initialize(ReportingInitializationContext)}. - */ - private MockReportingInitializationContext reportingInitContextStub; - - /** - * The test subject. - */ - private MetricsReportingTask testedReportingTask; - - /** - * Set up the test environment and mock behaviour. This includes registering {@link #reporterServiceStub} in the - * different contexts, overriding {@link MetricsReportingTask#currentStatusReference} and instantiating the test - * subject. - */ - @BeforeEach - public void setUp() throws Exception { - Map services = new HashMap<>(); - services.put(REPORTER_SERVICE_IDENTIFIER, reporterServiceStub); - testedReportingTask = new MetricsReportingTask(); - reportingContextStub = new MockReportingContext( - services, new MockStateManager(testedReportingTask), new MockVariableRegistry()); - - rootGroupStatus = new ProcessGroupStatus(); - innerGroupStatus = new ProcessGroupStatus(); - when(reporterServiceStub.createReporter(any())).thenReturn(reporterMock); - reportingContextStub.setProperty(MetricsReportingTask.REPORTER_SERVICE.getName(), REPORTER_SERVICE_IDENTIFIER); - reportingContextStub.addControllerService(reporterServiceStub, REPORTER_SERVICE_IDENTIFIER); - - configurationContextStub = new MockConfigurationContext(reportingContextStub.getProperties(), - reportingContextStub.getControllerServiceLookup()); - reportingInitContextStub = new MockReportingInitializationContext( - TEST_INIT_CONTEXT_ID, - TEST_INIT_CONTEXT_NAME, - new MockComponentLog(TEST_TASK_ID, testedReportingTask)); - } - - /** - * Make sure that in a single life cycle the correct metrics are registered, the correct {@link ProcessGroupStatus} - * is used and that metrics are actually reported. - */ - @Test - public void testValidLifeCycleReportsCorrectly() throws Exception { - reportingContextStub.getEventAccess().setProcessGroupStatus(rootGroupStatus); - - testedReportingTask.initialize(reportingInitContextStub); - testedReportingTask.connect(configurationContextStub); - testedReportingTask.onTrigger(reportingContextStub); - verify(reporterMock).report(); - - // Verify correct metrics are registered - ArgumentCaptor registryCaptor = ArgumentCaptor.forClass(MetricRegistry.class); - verify(reporterServiceStub).createReporter(registryCaptor.capture()); - MetricRegistry usedRegistry = registryCaptor.getValue(); - Map usedMetrics = usedRegistry.getMetrics(); - assertTrue(usedMetrics.keySet().containsAll(new MemoryUsageGaugeSet().getMetrics().keySet())); - assertTrue(usedMetrics.keySet() - .containsAll(new FlowMetricSet(testedReportingTask.currentStatusReference).getMetrics().keySet())); - - // Verify the most current ProcessGroupStatus is updated - assertEquals(testedReportingTask.currentStatusReference.get(), rootGroupStatus); - } - - /** - * Make sure that in a single life cycle the correct metrics are registered, the correct {@link ProcessGroupStatus} - * is used and that metrics are actually reported. - */ - @Test - public void testValidLifeCycleReportsCorrectlyProcessGroupSpecified() throws Exception { - reportingContextStub.setProperty(MetricsReportingTask.PROCESS_GROUP_ID.getName(), TEST_GROUP_ID); - reportingContextStub.getEventAccess().setProcessGroupStatus(TEST_GROUP_ID, innerGroupStatus); - - testedReportingTask.initialize(reportingInitContextStub); - testedReportingTask.connect(configurationContextStub); - testedReportingTask.onTrigger(reportingContextStub); - verify(reporterMock).report(); - - // Verify correct metrics are registered - ArgumentCaptor registryCaptor = ArgumentCaptor.forClass(MetricRegistry.class); - verify(reporterServiceStub).createReporter(registryCaptor.capture()); - MetricRegistry usedRegistry = registryCaptor.getValue(); - Map usedMetrics = usedRegistry.getMetrics(); - assertTrue(usedMetrics.keySet().containsAll(new MemoryUsageGaugeSet().getMetrics().keySet())); - assertTrue(usedMetrics.keySet() - .containsAll(new FlowMetricSet(testedReportingTask.currentStatusReference).getMetrics().keySet())); - - // Verify the most current ProcessGroupStatus is updated - assertEquals(testedReportingTask.currentStatusReference.get(), innerGroupStatus); - } - - /** - * Make sure that in a single life cycle the correct metrics are registered, the correct {@link ProcessGroupStatus} - * is used and that metrics are actually reported. - */ - @Test - public void testInvalidProcessGroupId() throws Exception { - reportingContextStub.setProperty(MetricsReportingTask.PROCESS_GROUP_ID.getName(), TEST_GROUP_ID + "-invalid"); - reportingContextStub.getEventAccess().setProcessGroupStatus(TEST_GROUP_ID, innerGroupStatus); - - testedReportingTask.initialize(reportingInitContextStub); - testedReportingTask.connect(configurationContextStub); - testedReportingTask.onTrigger(reportingContextStub); - verify(reporterMock, never()).report(); - assertNull(testedReportingTask.currentStatusReference.get()); - } - - /** - * Make sure that {@link MetricsReportingTask#connect(ConfigurationContext)} does not create a new reporter - * if there is already an active reporter. - */ - @Test - public void testConnectCreatesSingleReporter() throws Exception { - testedReportingTask.initialize(reportingInitContextStub); - testedReportingTask.connect(configurationContextStub); - testedReportingTask.connect(configurationContextStub); - - verify(reporterServiceStub, times(1)).createReporter(any()); - } - - /** - * Sanity check for registered properties. - */ - @Test - public void testGetSupportedPropertyDescriptorsSanity() throws Exception { - List expected = Arrays.asList( - MetricsReportingTask.REPORTER_SERVICE, - MetricsReportingTask.PROCESS_GROUP_ID); - assertEquals(expected, testedReportingTask.getSupportedPropertyDescriptors()); - } -} diff --git a/nifi-nar-bundles/nifi-metrics-reporting-bundle/pom.xml b/nifi-nar-bundles/nifi-metrics-reporting-bundle/pom.xml deleted file mode 100644 index 585e3a3f7e..0000000000 --- a/nifi-nar-bundles/nifi-metrics-reporting-bundle/pom.xml +++ /dev/null @@ -1,55 +0,0 @@ - - - - - nifi-nar-bundles - org.apache.nifi - 2.0.0-SNAPSHOT - - 4.0.0 - - nifi-metrics-reporting-bundle - pom - - nifi-metrics-reporting-task - nifi-metrics-reporting-nar - nifi-metrics-reporter-service-api - nifi-metrics-reporter-service-api-nar - - - - - - org.apache.nifi - nifi-metrics - 2.0.0-SNAPSHOT - - - org.apache.nifi - nifi-metrics-reporting-task - 2.0.0-SNAPSHOT - - - - - - - org.apache.nifi - nifi-api - provided - - - diff --git a/nifi-nar-bundles/nifi-riemann-bundle/nifi-riemann-nar/pom.xml b/nifi-nar-bundles/nifi-riemann-bundle/nifi-riemann-nar/pom.xml deleted file mode 100644 index 7dacd82026..0000000000 --- a/nifi-nar-bundles/nifi-riemann-bundle/nifi-riemann-nar/pom.xml +++ /dev/null @@ -1,36 +0,0 @@ - - - - 4.0.0 - - nifi-riemann-bundle - org.apache.nifi - 2.0.0-SNAPSHOT - - - true - true - - - nifi-riemann-nar - nar - - - org.apache.nifi - nifi-riemann-processors - - - \ No newline at end of file diff --git a/nifi-nar-bundles/nifi-riemann-bundle/nifi-riemann-nar/src/main/resources/META-INF/NOTICE b/nifi-nar-bundles/nifi-riemann-bundle/nifi-riemann-nar/src/main/resources/META-INF/NOTICE deleted file mode 100644 index 603d4bfa33..0000000000 --- a/nifi-nar-bundles/nifi-riemann-bundle/nifi-riemann-nar/src/main/resources/META-INF/NOTICE +++ /dev/null @@ -1,19 +0,0 @@ -nifi-riemann-nar -Copyright 2014-2023 The Apache Software Foundation - -This product includes software developed at -The Apache Software Foundation (http://www.apache.org/). - -=========================================== -Apache Software License v2 -=========================================== - -The following binary components are provided under the Apache Software License v2 - - (ASLv2) Apache Commons Lang - The following NOTICE information applies: - Apache Commons Lang - Copyright 2001-2015 The Apache Software Foundation - - This product includes software from the Spring Framework, - under the Apache License 2.0 (see: StringUtils.containsWhitespace()) diff --git a/nifi-nar-bundles/nifi-riemann-bundle/nifi-riemann-processors/pom.xml b/nifi-nar-bundles/nifi-riemann-bundle/nifi-riemann-processors/pom.xml deleted file mode 100644 index b5bf2252c4..0000000000 --- a/nifi-nar-bundles/nifi-riemann-bundle/nifi-riemann-processors/pom.xml +++ /dev/null @@ -1,55 +0,0 @@ - - - - 4.0.0 - - org.apache.nifi - nifi-riemann-bundle - 2.0.0-SNAPSHOT - - - nifi-riemann-processors - - - - io.riemann - riemann-java-client - - - org.apache.nifi - nifi-api - - - org.apache.nifi - nifi-utils - 2.0.0-SNAPSHOT - - - org.apache.nifi - nifi-properties - - - org.apache.commons - commons-lang3 - - - org.apache.nifi - nifi-mock - 2.0.0-SNAPSHOT - test - - - \ No newline at end of file diff --git a/nifi-nar-bundles/nifi-riemann-bundle/nifi-riemann-processors/src/main/java/org/apache/nifi/processors/riemann/PutRiemann.java b/nifi-nar-bundles/nifi-riemann-bundle/nifi-riemann-processors/src/main/java/org/apache/nifi/processors/riemann/PutRiemann.java deleted file mode 100644 index d9a54d0af8..0000000000 --- a/nifi-nar-bundles/nifi-riemann-bundle/nifi-riemann-processors/src/main/java/org/apache/nifi/processors/riemann/PutRiemann.java +++ /dev/null @@ -1,382 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.nifi.processors.riemann; - -import io.riemann.riemann.Proto; -import io.riemann.riemann.Proto.Event; -import io.riemann.riemann.client.RiemannClient; -import org.apache.commons.lang3.StringUtils; -import org.apache.nifi.annotation.behavior.DynamicProperty; -import org.apache.nifi.annotation.behavior.InputRequirement; -import org.apache.nifi.annotation.behavior.InputRequirement.Requirement; -import org.apache.nifi.annotation.behavior.SupportsBatching; -import org.apache.nifi.annotation.documentation.CapabilityDescription; -import org.apache.nifi.annotation.documentation.Tags; -import org.apache.nifi.annotation.lifecycle.OnScheduled; -import org.apache.nifi.annotation.lifecycle.OnStopped; -import org.apache.nifi.components.PropertyDescriptor; -import org.apache.nifi.components.PropertyValue; -import org.apache.nifi.components.Validator; -import org.apache.nifi.expression.ExpressionLanguageScope; -import org.apache.nifi.flowfile.FlowFile; -import org.apache.nifi.processor.AbstractProcessor; -import org.apache.nifi.processor.ProcessContext; -import org.apache.nifi.processor.ProcessSession; -import org.apache.nifi.processor.Relationship; -import org.apache.nifi.processor.exception.ProcessException; -import org.apache.nifi.processor.util.StandardValidators; - -import java.io.IOException; -import java.util.ArrayList; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.concurrent.TimeUnit; - -@Tags({"riemann", "monitoring", "metrics"}) -@DynamicProperty(name = "Custom Event Attribute", expressionLanguageScope = ExpressionLanguageScope.FLOWFILE_ATTRIBUTES, - description = "These values will be attached to the Riemann event as a custom attribute", - value = "Any value or expression") -@CapabilityDescription("Send events to Riemann (http://riemann.io) when FlowFiles pass through this processor. " + - "You can use events to notify Riemann that a FlowFile passed through, or you can attach a more " + - "meaningful metric, such as, the time a FlowFile took to get to this processor. All attributes attached to " + - "events support the NiFi Expression Language.") -@SupportsBatching -@InputRequirement(Requirement.INPUT_REQUIRED) -public class PutRiemann extends AbstractProcessor { - protected enum Transport { - TCP, UDP - } - - protected volatile RiemannClient riemannClient = null; - protected volatile Transport transport; - - public static final Relationship REL_SUCCESS = new Relationship.Builder() - .name("success") - .description("Metrics successfully written to Riemann") - .build(); - - public static final Relationship REL_FAILURE = new Relationship.Builder() - .name("failure") - .description("Metrics which failed to write to Riemann") - .build(); - - - public static final PropertyDescriptor RIEMANN_HOST = new PropertyDescriptor.Builder() - .name("Riemann Address") - .description("Hostname of Riemann server") - .required(true) - .addValidator(StandardValidators.NON_EMPTY_VALIDATOR) - .build(); - - public static final PropertyDescriptor RIEMANN_PORT = new PropertyDescriptor.Builder() - .name("Riemann Port") - .description("Port that Riemann is listening on") - .required(true) - .defaultValue("5555") - .addValidator(StandardValidators.PORT_VALIDATOR) - .build(); - - public static final PropertyDescriptor TRANSPORT_PROTOCOL = new PropertyDescriptor.Builder() - .name("Transport Protocol") - .description("Transport protocol to speak to Riemann in") - .required(true) - .allowableValues(new Transport[]{Transport.TCP, Transport.UDP}) - .defaultValue("TCP") - .build(); - - public static final PropertyDescriptor BATCH_SIZE = new PropertyDescriptor.Builder() - .name("Batch Size") - .description("Batch size for incoming FlowFiles") - .required(false) - .defaultValue("100") - .addValidator(StandardValidators.POSITIVE_INTEGER_VALIDATOR) - .build(); - - // Attributes Mappings - public static final PropertyDescriptor ATTR_SERVICE = new PropertyDescriptor.Builder() - .name("Service") - .description("Name of service associated to this event (e.g. FTP File Fetched)") - .required(false) - .expressionLanguageSupported(ExpressionLanguageScope.FLOWFILE_ATTRIBUTES) - .addValidator(Validator.VALID) - .build(); - - public static final PropertyDescriptor ATTR_STATE = new PropertyDescriptor.Builder() - .name("State") - .description("State of service associated to this event in string form (e.g. ok, warning, foo)") - .required(false) - .expressionLanguageSupported(ExpressionLanguageScope.FLOWFILE_ATTRIBUTES) - .addValidator(Validator.VALID) - .build(); - - public static final PropertyDescriptor ATTR_TIME = new PropertyDescriptor.Builder() - .name("Time") - .description("Time of event in unix epoch seconds (long), default: (current time)") - .required(false) - .expressionLanguageSupported(ExpressionLanguageScope.FLOWFILE_ATTRIBUTES) - .addValidator(Validator.VALID) - .build(); - - public static final PropertyDescriptor ATTR_HOST = new PropertyDescriptor.Builder() - .name("Host") - .description("A hostname associated to this event (e.g. nifi-app1)") - .required(false) - .defaultValue("${hostname()}") - .expressionLanguageSupported(ExpressionLanguageScope.FLOWFILE_ATTRIBUTES) - .addValidator(Validator.VALID) - .build(); - - public static final PropertyDescriptor ATTR_TTL = new PropertyDescriptor.Builder() - .name("TTL") - .description("Floating point value in seconds until Riemann considers this event as \"expired\"") - .required(false) - .addValidator(Validator.VALID) - .expressionLanguageSupported(ExpressionLanguageScope.FLOWFILE_ATTRIBUTES) - .build(); - - public static final PropertyDescriptor ATTR_METRIC = new PropertyDescriptor.Builder() - .name("Metric") - .description("Floating point number associated to this event") - .required(false) - .addValidator(Validator.VALID) - .expressionLanguageSupported(ExpressionLanguageScope.FLOWFILE_ATTRIBUTES) - .build(); - - public static final PropertyDescriptor ATTR_DESCRIPTION = new PropertyDescriptor.Builder() - .name("Description") - .description("Description associated to the event") - .required(false) - .expressionLanguageSupported(ExpressionLanguageScope.FLOWFILE_ATTRIBUTES) - .addValidator(Validator.VALID) - .build(); - - - public static final PropertyDescriptor ATTR_TAGS = new PropertyDescriptor.Builder() - .name("Tags") - .description("Comma separated list of tags associated to the event") - .required(false) - .expressionLanguageSupported(ExpressionLanguageScope.FLOWFILE_ATTRIBUTES) - .addValidator(Validator.VALID) - .build(); - - public static final PropertyDescriptor TIMEOUT = new PropertyDescriptor.Builder() - .name("Timeout") - .description("Timeout in milliseconds when writing events to Riemann") - .required(true) - .defaultValue("1000") - .addValidator(StandardValidators.POSITIVE_LONG_VALIDATOR) - .build(); - - private volatile List customAttributes = new ArrayList<>(); - private static final Set RELATIONSHIPS = new HashSet<>(); - private static final List LOCAL_PROPERTIES = new ArrayList<>(); - - private volatile int batchSize = -1; - private volatile long writeTimeout = 1000; - - static { - RELATIONSHIPS.add(REL_SUCCESS); - RELATIONSHIPS.add(REL_FAILURE); - LOCAL_PROPERTIES.add(RIEMANN_HOST); - LOCAL_PROPERTIES.add(RIEMANN_PORT); - LOCAL_PROPERTIES.add(TRANSPORT_PROTOCOL); - LOCAL_PROPERTIES.add(TIMEOUT); - LOCAL_PROPERTIES.add(BATCH_SIZE); - LOCAL_PROPERTIES.add(ATTR_DESCRIPTION); - LOCAL_PROPERTIES.add(ATTR_SERVICE); - LOCAL_PROPERTIES.add(ATTR_STATE); - LOCAL_PROPERTIES.add(ATTR_METRIC); - LOCAL_PROPERTIES.add(ATTR_TTL); - LOCAL_PROPERTIES.add(ATTR_TAGS); - LOCAL_PROPERTIES.add(ATTR_HOST); - LOCAL_PROPERTIES.add(ATTR_TIME); - } - - - @Override - public Set getRelationships() { - return RELATIONSHIPS; - } - - @Override - protected List getSupportedPropertyDescriptors() { - return LOCAL_PROPERTIES; - } - - @Override - protected PropertyDescriptor getSupportedDynamicPropertyDescriptor(final String propertyDescriptorName) { - return new PropertyDescriptor.Builder() - .name(propertyDescriptorName) - .expressionLanguageSupported(ExpressionLanguageScope.FLOWFILE_ATTRIBUTES) - .addValidator(Validator.VALID) - .required(false) - .dynamic(true) - .build(); - } - - @OnStopped - public final void cleanUpClient() { - if (riemannClient != null) { - this.riemannClient.close(); - } - this.riemannClient = null; - this.batchSize = -1; - this.customAttributes.clear(); - } - - @OnScheduled - public void onScheduled(ProcessContext context) throws ProcessException { - if (batchSize == -1) { - batchSize = context.getProperty(BATCH_SIZE).asInteger(); - } - if (riemannClient == null || !riemannClient.isConnected()) { - transport = Transport.valueOf(context.getProperty(TRANSPORT_PROTOCOL).getValue()); - String host = context.getProperty(RIEMANN_HOST).getValue().trim(); - int port = context.getProperty(RIEMANN_PORT).asInteger(); - writeTimeout = context.getProperty(TIMEOUT).asLong(); - RiemannClient client = null; - try { - switch (transport) { - case TCP: - client = RiemannClient.tcp(host, port); - break; - case UDP: - client = RiemannClient.udp(host, port); - break; - } - client.connect(); - riemannClient = client; - } catch (IOException e) { - if (client != null) { - client.close(); - } - context.yield(); - throw new ProcessException(String.format("Unable to connect to Riemann [%s:%d] (%s)\n%s", host, port, transport, e.getMessage())); - } - } - - if (customAttributes.size() == 0) { - for (Map.Entry property : context.getProperties().entrySet()) { - // only custom defined properties - if (!getSupportedPropertyDescriptors().contains(property.getKey())) { - customAttributes.add(property.getKey()); - } - } - } - } - - - @Override - public void onTrigger(ProcessContext context, ProcessSession session) throws ProcessException { - // Check if the client is currently connected, as a previous trigger could have detected a failure - // in the connection. - if (riemannClient == null || !riemannClient.isConnected()) { - // clean up the client and attempt to re-initialize the processor - cleanUpClient(); - onScheduled(context); - } - - List incomingFlowFiles = session.get(batchSize); - List successfulFlowFiles = new ArrayList<>(incomingFlowFiles.size()); - List eventsQueue = new ArrayList<>(incomingFlowFiles.size()); - for (FlowFile flowFile : incomingFlowFiles) { - try { - eventsQueue.add(FlowFileToEvent.fromAttributes(context, customAttributes, flowFile)); - successfulFlowFiles.add(flowFile); - } catch (NumberFormatException e) { - getLogger().warn("Unable to create Riemann event.", e); - session.transfer(flowFile, REL_FAILURE); - } - } - try { - if (transport == Transport.TCP) { - Proto.Msg returnMessage = riemannClient.sendEvents(eventsQueue).deref(writeTimeout, TimeUnit.MILLISECONDS); - if (returnMessage == null) { - context.yield(); - throw new ProcessException("Timed out writing to Riemann!"); - } - } else { - riemannClient.sendEvents(eventsQueue); - } - riemannClient.flush(); - session.transfer(successfulFlowFiles, REL_SUCCESS); - } catch (Exception e) { - context.yield(); - session.transfer(incomingFlowFiles); - throw new ProcessException("Failed writing to Riemann\n" + e.getMessage()); - } - } - - /** - * Converts a FlowFile into a Riemann Protobuf Event - */ - private static class FlowFileToEvent { - protected static Event fromAttributes(ProcessContext context, List customProperties, - FlowFile flowFile) { - Event.Builder builder = Event.newBuilder(); - - PropertyValue service = context.getProperty(ATTR_SERVICE).evaluateAttributeExpressions(flowFile); - if (StringUtils.isNotBlank(service.getValue())) { - builder.setService(service.getValue()); - } - PropertyValue description = context.getProperty(ATTR_DESCRIPTION).evaluateAttributeExpressions(flowFile); - if (StringUtils.isNotBlank(description.getValue())) { - builder.setDescription(description.getValue()); - } - PropertyValue metric = context.getProperty(ATTR_METRIC).evaluateAttributeExpressions(flowFile); - if (StringUtils.isNotBlank(metric.getValue())) { - builder.setMetricF(metric.asFloat()); - } - PropertyValue time = context.getProperty(ATTR_TIME).evaluateAttributeExpressions(flowFile); - if (StringUtils.isNotBlank(time.getValue())) { - builder.setTime(time.asLong()); - } - PropertyValue state = context.getProperty(ATTR_STATE).evaluateAttributeExpressions(flowFile); - if (StringUtils.isNotBlank(state.getValue())) { - builder.setState(state.getValue()); - } - PropertyValue ttl = context.getProperty(ATTR_TTL).evaluateAttributeExpressions(flowFile); - if (StringUtils.isNotBlank(ttl.getValue())) { - builder.setTtl(ttl.asFloat()); - } - PropertyValue host = context.getProperty(ATTR_HOST).evaluateAttributeExpressions(flowFile); - if (StringUtils.isNotBlank(host.getValue())) { - builder.setHost(host.getValue()); - } - PropertyValue tags = context.getProperty(ATTR_TAGS).evaluateAttributeExpressions(flowFile); - if (StringUtils.isNotBlank(tags.getValue())) { - String[] splitTags = tags.getValue().split(","); - for (String splitTag : splitTags) { - builder.addTags(splitTag.trim()); - } - } - PropertyValue customAttributeValue; - for (PropertyDescriptor customProperty : customProperties) { - customAttributeValue = context.getProperty(customProperty).evaluateAttributeExpressions(flowFile); - if (StringUtils.isNotBlank(customAttributeValue.getValue())) { - builder.addAttributes(Proto.Attribute.newBuilder() - .setKey(customProperty.getName()) - .setValue(customAttributeValue.getValue()) - .build()); - } - } - return builder.build(); - } - } -} diff --git a/nifi-nar-bundles/nifi-riemann-bundle/nifi-riemann-processors/src/main/resources/META-INF/services/org.apache.nifi.processor.Processor b/nifi-nar-bundles/nifi-riemann-bundle/nifi-riemann-processors/src/main/resources/META-INF/services/org.apache.nifi.processor.Processor deleted file mode 100644 index c53cea64f2..0000000000 --- a/nifi-nar-bundles/nifi-riemann-bundle/nifi-riemann-processors/src/main/resources/META-INF/services/org.apache.nifi.processor.Processor +++ /dev/null @@ -1,15 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -org.apache.nifi.processors.riemann.PutRiemann diff --git a/nifi-nar-bundles/nifi-riemann-bundle/nifi-riemann-processors/src/test/java/org/apache/nifi/processors/riemann/TestPutRiemann.java b/nifi-nar-bundles/nifi-riemann-bundle/nifi-riemann-processors/src/test/java/org/apache/nifi/processors/riemann/TestPutRiemann.java deleted file mode 100644 index 7082b9c185..0000000000 --- a/nifi-nar-bundles/nifi-riemann-bundle/nifi-riemann-processors/src/test/java/org/apache/nifi/processors/riemann/TestPutRiemann.java +++ /dev/null @@ -1,188 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.nifi.processors.riemann; - -import io.riemann.riemann.Proto; -import io.riemann.riemann.client.IPromise; -import io.riemann.riemann.client.RiemannClient; -import org.apache.nifi.processor.exception.ProcessException; -import org.apache.nifi.util.MockFlowFile; -import org.apache.nifi.util.TestRunner; -import org.apache.nifi.util.TestRunners; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; -import org.mockito.invocation.InvocationOnMock; -import org.mockito.stubbing.Answer; - -import java.util.HashMap; -import java.util.LinkedList; -import java.util.List; -import java.util.Map; -import java.util.Queue; -import java.util.concurrent.TimeUnit; - -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertThrows; -import static org.junit.jupiter.api.Assertions.assertTrue; -import static org.mockito.ArgumentMatchers.anyLong; -import static org.mockito.Mockito.any; -import static org.mockito.Mockito.anyList; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.when; - -public class TestPutRiemann { - // Holds incoming events to Riemann - private Queue eventStream = new LinkedList(); - - @BeforeEach - public void clearEventStream() { - eventStream.clear(); - } - - private TestRunner getTestRunner() { - return getTestRunner(false); - } - - private TestRunner getTestRunner(final boolean failOnWrite) { - RiemannClient riemannClient = mock(RiemannClient.class); - when(riemannClient.sendEvents(anyList())).thenAnswer(new Answer() { - @Override - public Object answer(InvocationOnMock invocationOnMock) throws Throwable { - List events = (List) invocationOnMock.getArguments()[0]; - for (Proto.Event event : events) { - eventStream.add(event); - } - IPromise iPromise = mock(IPromise.class); - if (!failOnWrite) { - when(iPromise.deref(anyLong(), any(TimeUnit.class))).thenReturn(Proto.Msg.getDefaultInstance()); - } else { - when(iPromise.deref(anyLong(), any(TimeUnit.class))).thenReturn(null); - } - return iPromise; - } - }); - when(riemannClient.isConnected()).thenReturn(true); - PutRiemann riemannProcessor = new PutRiemann(); - riemannProcessor.riemannClient = riemannClient; - riemannProcessor.transport = PutRiemann.Transport.TCP; - - TestRunner runner = TestRunners.newTestRunner(riemannProcessor); - runner.setProperty(PutRiemann.RIEMANN_HOST, "localhost"); - runner.setProperty(PutRiemann.RIEMANN_PORT, "5555"); - runner.setProperty(PutRiemann.TRANSPORT_PROTOCOL, "TCP"); - runner.setProperty(PutRiemann.BATCH_SIZE, "100"); - runner.setProperty(PutRiemann.ATTR_SERVICE, "nifi-test-service"); - runner.setProperty(PutRiemann.ATTR_HOST, "${riemann.host}"); - runner.setProperty(PutRiemann.ATTR_TTL, "5"); - runner.setProperty(PutRiemann.ATTR_DESCRIPTION, "test"); - runner.setProperty(PutRiemann.ATTR_TAGS, "tag1, tag2, tag3"); - runner.setProperty(PutRiemann.ATTR_METRIC, "${riemann.metric}"); - runner.setProperty("custom-attribute-1", "${custom.attribute.1}"); - runner.setProperty("custom-attribute-2", "${custom.attribute.2}"); - runner.setProperty("custom-attribute-3", "${custom.attribute.3}"); - return runner; - } - - - @Test - public void testBasicEvent() { - TestRunner runner = getTestRunner(); - Map attributes = new HashMap<>(); - attributes.put("riemann.metric", "42"); - attributes.put("riemann.host", "basic-host"); - MockFlowFile flowFile = new MockFlowFile(1); - flowFile.putAttributes(attributes); - runner.enqueue(flowFile); - runner.run(); - runner.assertAllFlowFilesTransferred(PutRiemann.REL_SUCCESS); - - Proto.Event event = eventStream.remove(); - assertEquals("nifi-test-service", event.getService()); - assertTrue(5.0 == event.getTtl()); - assertTrue(42.0 == event.getMetricF()); - assertEquals("basic-host", event.getHost()); - assertEquals("test", event.getDescription()); - assertEquals(3, event.getTagsCount()); - assertTrue(event.getTagsList().contains("tag1")); - assertTrue(event.getTagsList().contains("tag2")); - assertTrue(event.getTagsList().contains("tag3")); - assertEquals(0, event.getAttributesCount()); - } - - @Test - public void testBatchedEvents() { - // (2 batches) + (1 remaining event) - int iterations = Integer.parseInt(PutRiemann.BATCH_SIZE.getDefaultValue()) * 2 + 1; - TestRunner runner = getTestRunner(); - - for (int i = 0; i < iterations; i++) { - Map attributes = new HashMap<>(); - attributes.put("riemann.metric", Float.toString(i)); - attributes.put("riemann.host", "batch-host"); - attributes.put("custom.attribute.1", "attr1"); - attributes.put("custom.attribute.2", "attr2"); - attributes.put("custom.attribute.3", "attr3"); - MockFlowFile flowFile = new MockFlowFile(i); - flowFile.putAttributes(attributes); - runner.enqueue(flowFile); - } - runner.run(3); - runner.assertAllFlowFilesTransferred(PutRiemann.REL_SUCCESS); - - for (int i = 0; i < iterations; i++) { - Proto.Event event = eventStream.remove(); - assertEquals("nifi-test-service", event.getService()); - assertTrue(5.0 == event.getTtl()); - assertTrue(i == event.getMetricF()); - assertEquals("batch-host", event.getHost()); - assertEquals("test", event.getDescription()); - assertEquals(3, event.getTagsCount()); - assertEquals(3, event.getAttributesCount()); - assertTrue(event.getTagsList().contains("tag1")); - assertTrue(event.getTagsList().contains("tag2")); - assertTrue(event.getTagsList().contains("tag3")); - } - } - - @Test - public void testInvalidEvents() { - TestRunner runner = getTestRunner(); - MockFlowFile flowFile = new MockFlowFile(1); - Map attributes = new HashMap<>(); - attributes.put("riemann.metric", "NOT A NUMBER"); - flowFile.putAttributes(attributes); - runner.enqueue(flowFile); - runner.run(); - runner.assertAllFlowFilesTransferred(PutRiemann.REL_FAILURE); - } - - @Test - public void testFailedDeref() { - TestRunner runner = getTestRunner(true); - MockFlowFile flowFile = new MockFlowFile(1); - Map attributes = new HashMap<>(); - attributes.put("riemann.metric", "5"); - flowFile.putAttributes(attributes); - runner.enqueue(flowFile); - try { - assertThrows(AssertionError.class, () -> runner.run()); - } catch (ProcessException e) { - runner.assertQueueNotEmpty(); - throw e; - } - } -} diff --git a/nifi-nar-bundles/nifi-riemann-bundle/pom.xml b/nifi-nar-bundles/nifi-riemann-bundle/pom.xml deleted file mode 100644 index 810a66b81b..0000000000 --- a/nifi-nar-bundles/nifi-riemann-bundle/pom.xml +++ /dev/null @@ -1,54 +0,0 @@ - - - - 4.0.0 - - nifi-nar-bundles - org.apache.nifi - 2.0.0-SNAPSHOT - - nifi-riemann-bundle - pom - - nifi-riemann-processors - nifi-riemann-nar - - - - clojars.org - https://clojars.org/repo - - - - - - io.riemann - riemann-java-client - 0.5.3 - - - org.apache.nifi - nifi-riemann-processors - 2.0.0-SNAPSHOT - - - com.google.protobuf - protobuf-java - 3.23.0 - - - - \ No newline at end of file diff --git a/nifi-nar-bundles/nifi-rules-action-handler-bundle/nifi-rules-action-handler-nar/pom.xml b/nifi-nar-bundles/nifi-rules-action-handler-bundle/nifi-rules-action-handler-nar/pom.xml deleted file mode 100644 index 0157cec011..0000000000 --- a/nifi-nar-bundles/nifi-rules-action-handler-bundle/nifi-rules-action-handler-nar/pom.xml +++ /dev/null @@ -1,43 +0,0 @@ - - - - - nifi-rules-action-handler-bundle - org.apache.nifi - 2.0.0-SNAPSHOT - - 4.0.0 - - nifi-rules-action-handler-nar - nar - - true - true - - - - org.apache.nifi - nifi-rules-action-handler-service - 2.0.0-SNAPSHOT - - - org.apache.nifi - nifi-standard-services-api-nar - 2.0.0-SNAPSHOT - nar - - - diff --git a/nifi-nar-bundles/nifi-rules-action-handler-bundle/nifi-rules-action-handler-nar/src/main/resources/LICENSE b/nifi-nar-bundles/nifi-rules-action-handler-bundle/nifi-rules-action-handler-nar/src/main/resources/LICENSE deleted file mode 100644 index 1f84df0623..0000000000 --- a/nifi-nar-bundles/nifi-rules-action-handler-bundle/nifi-rules-action-handler-nar/src/main/resources/LICENSE +++ /dev/null @@ -1,236 +0,0 @@ - - Apache License - Version 2.0, January 2004 - http://www.apache.org/licenses/ - - TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION - - 1. Definitions. - - "License" shall mean the terms and conditions for use, reproduction, - and distribution as defined by Sections 1 through 9 of this document. - - "Licensor" shall mean the copyright owner or entity authorized by - the copyright owner that is granting the License. - - "Legal Entity" shall mean the union of the acting entity and all - other entities that control, are controlled by, or are under common - control with that entity. For the purposes of this definition, - "control" means (i) the power, direct or indirect, to cause the - direction or management of such entity, whether by contract or - otherwise, or (ii) ownership of fifty percent (50%) or more of the - outstanding shares, or (iii) beneficial ownership of such entity. - - "You" (or "Your") shall mean an individual or Legal Entity - exercising permissions granted by this License. - - "Source" form shall mean the preferred form for making modifications, - including but not limited to software source code, documentation - source, and configuration files. - - "Object" form shall mean any form resulting from mechanical - transformation or translation of a Source form, including but - not limited to compiled object code, generated documentation, - and conversions to other media types. - - "Work" shall mean the work of authorship, whether in Source or - Object form, made available under the License, as indicated by a - copyright notice that is included in or attached to the work - (an example is provided in the Appendix below). - - "Derivative Works" shall mean any work, whether in Source or Object - form, that is based on (or derived from) the Work and for which the - editorial revisions, annotations, elaborations, or other modifications - represent, as a whole, an original work of authorship. For the purposes - of this License, Derivative Works shall not include works that remain - separable from, or merely link (or bind by name) to the interfaces of, - the Work and Derivative Works thereof. - - "Contribution" shall mean any work of authorship, including - the original version of the Work and any modifications or additions - to that Work or Derivative Works thereof, that is intentionally - submitted to Licensor for inclusion in the Work by the copyright owner - or by an individual or Legal Entity authorized to submit on behalf of - the copyright owner. For the purposes of this definition, "submitted" - means any form of electronic, verbal, or written communication sent - to the Licensor or its representatives, including but not limited to - communication on electronic mailing lists, source code control systems, - and issue tracking systems that are managed by, or on behalf of, the - Licensor for the purpose of discussing and improving the Work, but - excluding communication that is conspicuously marked or otherwise - designated in writing by the copyright owner as "Not a Contribution." - - "Contributor" shall mean Licensor and any individual or Legal Entity - on behalf of whom a Contribution has been received by Licensor and - subsequently incorporated within the Work. - - 2. Grant of Copyright License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - copyright license to reproduce, prepare Derivative Works of, - publicly display, publicly perform, sublicense, and distribute the - Work and such Derivative Works in Source or Object form. - - 3. Grant of Patent License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - (except as stated in this section) patent license to make, have made, - use, offer to sell, sell, import, and otherwise transfer the Work, - where such license applies only to those patent claims licensable - by such Contributor that are necessarily infringed by their - Contribution(s) alone or by combination of their Contribution(s) - with the Work to which such Contribution(s) was submitted. If You - institute patent litigation against any entity (including a - cross-claim or counterclaim in a lawsuit) alleging that the Work - or a Contribution incorporated within the Work constitutes direct - or contributory patent infringement, then any patent licenses - granted to You under this License for that Work shall terminate - as of the date such litigation is filed. - - 4. Redistribution. You may reproduce and distribute copies of the - Work or Derivative Works thereof in any medium, with or without - modifications, and in Source or Object form, provided that You - meet the following conditions: - - (a) You must give any other recipients of the Work or - Derivative Works a copy of this License; and - - (b) You must cause any modified files to carry prominent notices - stating that You changed the files; and - - (c) You must retain, in the Source form of any Derivative Works - that You distribute, all copyright, patent, trademark, and - attribution notices from the Source form of the Work, - excluding those notices that do not pertain to any part of - the Derivative Works; and - - (d) If the Work includes a "NOTICE" text file as part of its - distribution, then any Derivative Works that You distribute must - include a readable copy of the attribution notices contained - within such NOTICE file, excluding those notices that do not - pertain to any part of the Derivative Works, in at least one - of the following places: within a NOTICE text file distributed - as part of the Derivative Works; within the Source form or - documentation, if provided along with the Derivative Works; or, - within a display generated by the Derivative Works, if and - wherever such third-party notices normally appear. The contents - of the NOTICE file are for informational purposes only and - do not modify the License. You may add Your own attribution - notices within Derivative Works that You distribute, alongside - or as an addendum to the NOTICE text from the Work, provided - that such additional attribution notices cannot be construed - as modifying the License. - - You may add Your own copyright statement to Your modifications and - may provide additional or different license terms and conditions - for use, reproduction, or distribution of Your modifications, or - for any such Derivative Works as a whole, provided Your use, - reproduction, and distribution of the Work otherwise complies with - the conditions stated in this License. - - 5. Submission of Contributions. Unless You explicitly state otherwise, - any Contribution intentionally submitted for inclusion in the Work - by You to the Licensor shall be under the terms and conditions of - this License, without any additional terms or conditions. - Notwithstanding the above, nothing herein shall supersede or modify - the terms of any separate license agreement you may have executed - with Licensor regarding such Contributions. - - 6. Trademarks. This License does not grant permission to use the trade - names, trademarks, service marks, or product names of the Licensor, - except as required for reasonable and customary use in describing the - origin of the Work and reproducing the content of the NOTICE file. - - 7. Disclaimer of Warranty. Unless required by applicable law or - agreed to in writing, Licensor provides the Work (and each - Contributor provides its Contributions) on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - implied, including, without limitation, any warranties or conditions - of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A - PARTICULAR PURPOSE. You are solely responsible for determining the - appropriateness of using or redistributing the Work and assume any - risks associated with Your exercise of permissions under this License. - - 8. Limitation of Liability. In no event and under no legal theory, - whether in tort (including negligence), contract, or otherwise, - unless required by applicable law (such as deliberate and grossly - negligent acts) or agreed to in writing, shall any Contributor be - liable to You for damages, including any direct, indirect, special, - incidental, or consequential damages of any character arising as a - result of this License or out of the use or inability to use the - Work (including but not limited to damages for loss of goodwill, - work stoppage, computer failure or malfunction, or any and all - other commercial damages or losses), even if such Contributor - has been advised of the possibility of such damages. - - 9. Accepting Warranty or Additional Liability. While redistributing - the Work or Derivative Works thereof, You may choose to offer, - and charge a fee for, acceptance of support, warranty, indemnity, - or other liability obligations and/or rights consistent with this - License. However, in accepting such obligations, You may act only - on Your own behalf and on Your sole responsibility, not on behalf - of any other Contributor, and only if You agree to indemnify, - defend, and hold each Contributor harmless for any liability - incurred by, or claims asserted against, such Contributor by reason - of your accepting any such warranty or additional liability. - - END OF TERMS AND CONDITIONS - - APPENDIX: How to apply the Apache License to your work. - - To apply the Apache License to your work, attach the following - boilerplate notice, with the fields enclosed by brackets "[]" - replaced with your own identifying information. (Don't include - the brackets!) The text should be enclosed in the appropriate - comment syntax for the file format. We also recommend that a - file or class name and description of purpose be included on the - same "printed page" as the copyright notice for easier - identification within third-party archives. - - Copyright [yyyy] [name of copyright owner] - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - - -nifi-rules-action-handler-nar includes subcomponents with separate copyright notices and -license terms. Your use of these subcomponents is subject to the terms -and conditions of the following licenses: - - The binary distribution of this product bundles the 'ASM' library which is available under a BSD style license. - - Copyright (c) 2000-2005 INRIA, France Telecom - All rights reserved. - - Redistribution and use in source and binary forms, with or without - modification, are permitted provided that the following conditions - are met: - 1. Redistributions of source code must retain the above copyright - notice, this list of conditions and the following disclaimer. - 2. Redistributions in binary form must reproduce the above copyright - notice, this list of conditions and the following disclaimer in the - documentation and/or other materials provided with the distribution. - 3. Neither the name of the copyright holders nor the names of its - contributors may be used to endorse or promote products derived from - this software without specific prior written permission. - - THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" - AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE - LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR - CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF - SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS - INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN - CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) - ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF - THE POSSIBILITY OF SUCH DAMAGE. \ No newline at end of file diff --git a/nifi-nar-bundles/nifi-rules-action-handler-bundle/nifi-rules-action-handler-nar/src/main/resources/NOTICE b/nifi-nar-bundles/nifi-rules-action-handler-bundle/nifi-rules-action-handler-nar/src/main/resources/NOTICE deleted file mode 100644 index 1f8db995a4..0000000000 --- a/nifi-nar-bundles/nifi-rules-action-handler-bundle/nifi-rules-action-handler-nar/src/main/resources/NOTICE +++ /dev/null @@ -1,18 +0,0 @@ -nifi-rules-action-handler-nar -Copyright 2014-2023 The Apache Software Foundation - -This product includes software developed at -The Apache Software Foundation (http://www.apache.org/). - -****************** -Apache Software License v2 -****************** - -The following binary components are provided under the Apache Software License v2 - - (ASLv2) MVEL (MVFLEX Expression Language) - - (ASLv2) Spring Framework (Core, Expression) - The following NOTICE information applies: - Spring Framework - Copyright (c) 2002-2019 Pivotal, Inc. diff --git a/nifi-nar-bundles/nifi-rules-action-handler-bundle/nifi-rules-action-handler-service/pom.xml b/nifi-nar-bundles/nifi-rules-action-handler-bundle/nifi-rules-action-handler-service/pom.xml deleted file mode 100644 index d7f1e5bf6e..0000000000 --- a/nifi-nar-bundles/nifi-rules-action-handler-bundle/nifi-rules-action-handler-service/pom.xml +++ /dev/null @@ -1,75 +0,0 @@ - - - - - nifi-rules-action-handler-bundle - org.apache.nifi - 2.0.0-SNAPSHOT - - 4.0.0 - - nifi-rules-action-handler-service - jar - - - org.apache.nifi - nifi-rules-engine-service-api - 2.0.0-SNAPSHOT - - - org.apache.nifi - nifi-api - - - org.apache.nifi - nifi-record-sink-api - 2.0.0-SNAPSHOT - - - org.apache.nifi - nifi-record - 2.0.0-SNAPSHOT - - - org.apache.nifi - nifi-record-serialization-service-api - 2.0.0-SNAPSHOT - - - org.apache.commons - commons-lang3 - - - org.apache.nifi - nifi-utils - 2.0.0-SNAPSHOT - - - org.mvel - mvel2 - 2.5.0.Final - - - org.springframework - spring-expression - - - org.apache.nifi - nifi-mock - test - - - diff --git a/nifi-nar-bundles/nifi-rules-action-handler-bundle/nifi-rules-action-handler-service/src/main/java/org/apache/nifi/rules/handlers/AbstractActionHandlerService.java b/nifi-nar-bundles/nifi-rules-action-handler-bundle/nifi-rules-action-handler-service/src/main/java/org/apache/nifi/rules/handlers/AbstractActionHandlerService.java deleted file mode 100644 index 4580ba07fe..0000000000 --- a/nifi-nar-bundles/nifi-rules-action-handler-bundle/nifi-rules-action-handler-service/src/main/java/org/apache/nifi/rules/handlers/AbstractActionHandlerService.java +++ /dev/null @@ -1,125 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.nifi.rules.handlers; - -import org.apache.commons.lang3.StringUtils; -import org.apache.nifi.annotation.lifecycle.OnEnabled; -import org.apache.nifi.components.PropertyDescriptor; -import org.apache.nifi.context.PropertyContext; -import org.apache.nifi.controller.AbstractControllerService; -import org.apache.nifi.controller.ConfigurationContext; -import org.apache.nifi.expression.ExpressionLanguageScope; -import org.apache.nifi.processor.util.StandardValidators; -import org.apache.nifi.reporting.InitializationException; -import org.apache.nifi.rules.Action; -import org.apache.nifi.rules.PropertyContextActionHandler; - -import java.util.Arrays; -import java.util.List; -import java.util.Map; -import java.util.stream.Collectors; - -public abstract class AbstractActionHandlerService extends AbstractControllerService implements PropertyContextActionHandler { - - protected List enforceActionTypes; - protected EnforceActionTypeLevel enforceActionTypeLevel; - - public enum DebugLevels { - trace, debug, info, warn, error - } - - public enum EnforceActionTypeLevel { - IGNORE, WARN, EXCEPTION - } - - public static final PropertyDescriptor ENFORCE_ACTION_TYPE = new PropertyDescriptor.Builder() - .name("action-handler-enforce-type") - .displayName("Enforce Action Type") - .required(false) - .description("The Action Type(s) that should be supported by this handler. If provided any other type an " + - "exception will be thrown. This can support a comma delimited list of types (e.g. ALERT,LOG)") - .addValidator(StandardValidators.NON_BLANK_VALIDATOR) - .expressionLanguageSupported(ExpressionLanguageScope.VARIABLE_REGISTRY) - .build(); - - public static final PropertyDescriptor ENFORCE_ACTION_TYPE_LEVEL = new PropertyDescriptor.Builder() - .name("action-handler-enforce-type-level") - .displayName("Enforce Level") - .required(false) - .description("If specific action types are enforced, this setting specifies whether the action should be ignored," + - " a warning should be logged or if an exception is thrown. Default is to ignore the received action.") - .addValidator(StandardValidators.NON_BLANK_VALIDATOR) - .allowableValues(EnforceActionTypeLevel.values()) - .defaultValue("IGNORE") - .build(); - - public void execute(Action action, Map facts) { - if (actionTypeNotSupported(action)) { - handleActionEnforcement(action); - } else { - executeAction(action, facts); - } - } - - public void execute(PropertyContext context, Action action, Map facts) { - if (actionTypeNotSupported(action)) { - handleActionEnforcement(action); - } else { - executeAction(context, action, facts); - } - } - - protected void executeAction(Action action, Map facts) { - throw new UnsupportedOperationException("This method is not supported by this handler."); - } - - protected void executeAction(PropertyContext propertyContext, Action action, Map facts) { - throw new UnsupportedOperationException("This method is not supported by this handler"); - } - - protected boolean actionTypeNotSupported(Action action) { - return enforceActionTypes != null && !enforceActionTypes.contains(action.getType()); - } - - protected void handleActionEnforcement(Action action) { - String message = "This Action Handler does not support actions with the provided type: " + action.getType(); - if (enforceActionTypeLevel.equals(EnforceActionTypeLevel.WARN)) { - getLogger().warn(message); - } else if (enforceActionTypeLevel.equals(EnforceActionTypeLevel.EXCEPTION)) { - throw new UnsupportedOperationException(message); - } else if (getLogger().isDebugEnabled()) { - getLogger().debug(message); - } - } - - @OnEnabled - public void onEnabled(final ConfigurationContext context) throws InitializationException { - String actionTypes = context.getProperty(ENFORCE_ACTION_TYPE).evaluateAttributeExpressions().getValue(); - if(StringUtils.isNotEmpty(actionTypes)){ - enforceActionTypes = Arrays.stream(actionTypes.split(",")) - .map(String::trim) - .filter(StringUtils::isNotEmpty) - .collect(Collectors.toList()); - } - String level = context.getProperty(ENFORCE_ACTION_TYPE_LEVEL).getValue(); - if(StringUtils.isNotEmpty(level)) { - enforceActionTypeLevel = EnforceActionTypeLevel.valueOf(level); - } - } - - -} diff --git a/nifi-nar-bundles/nifi-rules-action-handler-bundle/nifi-rules-action-handler-service/src/main/java/org/apache/nifi/rules/handlers/ActionHandlerLookup.java b/nifi-nar-bundles/nifi-rules-action-handler-bundle/nifi-rules-action-handler-service/src/main/java/org/apache/nifi/rules/handlers/ActionHandlerLookup.java deleted file mode 100644 index 2865193d01..0000000000 --- a/nifi-nar-bundles/nifi-rules-action-handler-bundle/nifi-rules-action-handler-service/src/main/java/org/apache/nifi/rules/handlers/ActionHandlerLookup.java +++ /dev/null @@ -1,124 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.nifi.rules.handlers; - -import org.apache.nifi.annotation.behavior.DynamicProperty; -import org.apache.nifi.annotation.documentation.CapabilityDescription; -import org.apache.nifi.annotation.documentation.Tags; -import org.apache.nifi.annotation.lifecycle.OnDisabled; -import org.apache.nifi.annotation.lifecycle.OnEnabled; -import org.apache.nifi.components.PropertyDescriptor; -import org.apache.nifi.components.ValidationContext; -import org.apache.nifi.components.ValidationResult; -import org.apache.nifi.context.PropertyContext; -import org.apache.nifi.controller.ConfigurationContext; -import org.apache.nifi.expression.ExpressionLanguageScope; -import org.apache.nifi.processor.exception.ProcessException; -import org.apache.nifi.processor.util.StandardValidators; -import org.apache.nifi.rules.Action; -import org.apache.nifi.rules.PropertyContextActionHandler; - -import java.util.ArrayList; -import java.util.Collection; -import java.util.Collections; -import java.util.HashMap; -import java.util.List; -import java.util.Map; - -@Tags({"rules", "rules engine", "action", "action handler","lookup"}) -@CapabilityDescription("Provides an Action Handler that can be used to dynamically select another Action Handler. " + -"This service will allow multiple ActionHandlers to be defined and registered by action type. When actions are provided the handlers can " + -"be dynamically determined and executed at runtime.") -@DynamicProperty(name = "actionType ", value = "Action Handler Service", expressionLanguageScope = ExpressionLanguageScope.NONE, description = "") -public class ActionHandlerLookup extends AbstractActionHandlerService{ - - private volatile Map actionHandlerMap; - - @Override - protected PropertyDescriptor getSupportedDynamicPropertyDescriptor(final String propertyDescriptorName) { - return new PropertyDescriptor.Builder() - .name(propertyDescriptorName) - .description("The Action handler to return when action type = '" + propertyDescriptorName + "'") - .identifiesControllerService(PropertyContextActionHandler.class) - .addValidator(StandardValidators.NON_BLANK_VALIDATOR) - .build(); - } - - @Override - protected Collection customValidate(ValidationContext context) { - final List results = new ArrayList<>(); - - int numDefinedServices = 0; - for (final PropertyDescriptor descriptor : context.getProperties().keySet()) { - if (descriptor.isDynamic()) { - numDefinedServices++; - } - - final String referencedId = context.getProperty(descriptor).getValue(); - if (this.getIdentifier().equals(referencedId)) { - results.add(new ValidationResult.Builder() - .subject(descriptor.getDisplayName()) - .explanation("the current service cannot be registered as an ActionHandler to lookup") - .valid(false) - .build()); - } - } - - if (numDefinedServices == 0) { - results.add(new ValidationResult.Builder() - .subject(this.getClass().getSimpleName()) - .explanation("at least one Action Handler must be defined via dynamic properties") - .valid(false) - .build()); - } - - return results; - } - - @OnEnabled - public void onEnabled(final ConfigurationContext context) { - final Map serviceMap = new HashMap<>(); - - for (final PropertyDescriptor descriptor : context.getProperties().keySet()) { - if (descriptor.isDynamic()) { - final PropertyContextActionHandler propertyContextActionHandler = context.getProperty(descriptor).asControllerService(PropertyContextActionHandler.class); - serviceMap.put(descriptor.getName(), propertyContextActionHandler); - } - } - - actionHandlerMap = Collections.unmodifiableMap(serviceMap); - } - - @OnDisabled - public void onDisabled() { - actionHandlerMap = null; - } - - @Override - public void execute(Action action, Map facts) { - execute(null,action,facts); - } - - @Override - public void execute(PropertyContext context, Action action, Map facts) { - PropertyContextActionHandler actionHandler = actionHandlerMap.get(action.getType()); - if (actionHandler == null) { - throw new ProcessException("No Action Handler was found for Action Type:" + action.getType()); - } - actionHandler.execute(context,action,facts); - } -} diff --git a/nifi-nar-bundles/nifi-rules-action-handler-bundle/nifi-rules-action-handler-service/src/main/java/org/apache/nifi/rules/handlers/AlertHandler.java b/nifi-nar-bundles/nifi-rules-action-handler-bundle/nifi-rules-action-handler-service/src/main/java/org/apache/nifi/rules/handlers/AlertHandler.java deleted file mode 100644 index 1687d17fdd..0000000000 --- a/nifi-nar-bundles/nifi-rules-action-handler-bundle/nifi-rules-action-handler-service/src/main/java/org/apache/nifi/rules/handlers/AlertHandler.java +++ /dev/null @@ -1,166 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.nifi.rules.handlers; - -import org.apache.nifi.annotation.documentation.CapabilityDescription; -import org.apache.nifi.annotation.documentation.Tags; -import org.apache.nifi.annotation.lifecycle.OnEnabled; -import org.apache.nifi.components.PropertyDescriptor; -import org.apache.nifi.context.PropertyContext; -import org.apache.nifi.controller.ConfigurationContext; -import org.apache.nifi.controller.ControllerServiceInitializationContext; -import org.apache.nifi.logging.ComponentLog; -import org.apache.nifi.processor.util.StandardValidators; -import org.apache.nifi.reporting.BulletinRepository; -import org.apache.nifi.reporting.InitializationException; -import org.apache.nifi.reporting.ReportingContext; -import org.apache.nifi.reporting.Severity; -import org.apache.nifi.rules.Action; - -import java.util.ArrayList; -import java.util.Collections; -import java.util.List; -import java.util.Map; -import java.util.Set; - -@Tags({"rules", "rules engine", "action", "action handler", "logging", "alerts", "bulletins"}) -@CapabilityDescription("Creates alerts as bulletins based on a provided action (usually created by a rules engine). " + - "Action objects executed with this Handler should contain \"category\", \"message\", and \"logLevel\" attributes.") -public class AlertHandler extends AbstractActionHandlerService { - - public static final PropertyDescriptor DEFAULT_LOG_LEVEL = new PropertyDescriptor.Builder() - .name("alert-default-log-level") - .displayName("Default Alert Log Level") - .required(true) - .description("The default Log Level that will be used to log an alert message" + - " if a log level was not provided in the received action's attributes.") - .allowableValues(DebugLevels.values()) - .defaultValue("info") - .build(); - - public static final PropertyDescriptor DEFAULT_CATEGORY = new PropertyDescriptor.Builder() - .name("alert-default-category") - .displayName("Default Category") - .required(true) - .description("The default category to use when logging alert message "+ - " if a category was not provided in the received action's attributes.") - .defaultValue("Rules Triggered Alert") - .addValidator(StandardValidators.NON_EMPTY_VALIDATOR) - .build(); - - public static final PropertyDescriptor DEFAULT_MESSAGE = new PropertyDescriptor.Builder() - .name("alert-default-message") - .displayName("Default Message") - .required(true) - .description("The default message to include in alert if an alert message was " + - "not provided in the received action's attributes") - .defaultValue("An alert was triggered by a rules-based action.") - .addValidator(StandardValidators.NON_EMPTY_VALIDATOR) - .build(); - - private static final PropertyDescriptor INCLUDE_FACTS = new PropertyDescriptor.Builder() - .name("alert-include-facts") - .displayName("Include Fact Data") - .required(true) - .description("If true, the alert message will include the facts which triggered this action. Default is false.") - .defaultValue("true") - .allowableValues("true", "false") - .build(); - - private List properties; - private String defaultCategory; - private String defaultLogLevel; - private String defaultMessage; - private Boolean includeFacts; - - @Override - protected void init(ControllerServiceInitializationContext config) throws InitializationException { - super.init(config); - final List properties = new ArrayList<>(); - properties.add(DEFAULT_LOG_LEVEL); - properties.add(DEFAULT_CATEGORY); - properties.add(DEFAULT_MESSAGE); - properties.add(INCLUDE_FACTS); - properties.add(ENFORCE_ACTION_TYPE); - properties.add(ENFORCE_ACTION_TYPE_LEVEL); - this.properties = Collections.unmodifiableList(properties); - } - - @Override - @OnEnabled - public void onEnabled(final ConfigurationContext context) throws InitializationException { - super.onEnabled(context); - defaultLogLevel = context.getProperty(DEFAULT_LOG_LEVEL).getValue().toUpperCase(); - defaultCategory = context.getProperty(DEFAULT_CATEGORY).getValue(); - defaultMessage = context.getProperty(DEFAULT_MESSAGE).getValue(); - includeFacts = context.getProperty(INCLUDE_FACTS).asBoolean(); - } - - @Override - protected List getSupportedPropertyDescriptors() { - return properties; - } - - @Override - protected void executeAction(PropertyContext propertyContext, Action action, Map facts) { - ComponentLog logger = getLogger(); - if (propertyContext instanceof ReportingContext) { - - ReportingContext context = (ReportingContext) propertyContext; - Map attributes = action.getAttributes(); - if (context.getBulletinRepository() != null) { - final String category = attributes.getOrDefault("category", defaultCategory); - final String message = getMessage(attributes.getOrDefault("message", defaultMessage), facts); - final String level = attributes.getOrDefault("severity", attributes.getOrDefault("logLevel", defaultLogLevel)); - Severity severity; - try { - severity = Severity.valueOf(level.toUpperCase()); - } catch (IllegalArgumentException iae) { - severity = Severity.INFO; - } - BulletinRepository bulletinRepository = context.getBulletinRepository(); - bulletinRepository.addBulletin(context.createBulletin(category, severity, message)); - - } else { - logger.warn("Bulletin Repository is not available which is unusual. Cannot send a bulletin."); - } - - } else { - logger.warn("Reporting context was not provided to create bulletins."); - } - } - - protected String getMessage(String alertMessage, Map facts){ - if (includeFacts) { - final StringBuilder message = new StringBuilder(alertMessage); - final Set fields = facts.keySet(); - message.append("\n"); - message.append("Alert Facts:\n"); - fields.forEach(field -> { - message.append("Field: "); - message.append(field); - message.append(", Value: "); - message.append(facts.get(field)); - message.append("\n"); - }); - return message.toString(); - }else{ - return alertMessage; - } - } - -} diff --git a/nifi-nar-bundles/nifi-rules-action-handler-bundle/nifi-rules-action-handler-service/src/main/java/org/apache/nifi/rules/handlers/ExpressionHandler.java b/nifi-nar-bundles/nifi-rules-action-handler-bundle/nifi-rules-action-handler-service/src/main/java/org/apache/nifi/rules/handlers/ExpressionHandler.java deleted file mode 100644 index 6a69d5eb24..0000000000 --- a/nifi-nar-bundles/nifi-rules-action-handler-bundle/nifi-rules-action-handler-service/src/main/java/org/apache/nifi/rules/handlers/ExpressionHandler.java +++ /dev/null @@ -1,131 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.nifi.rules.handlers; - -import org.apache.commons.lang3.StringUtils; -import org.apache.nifi.annotation.documentation.CapabilityDescription; -import org.apache.nifi.annotation.documentation.Tags; -import org.apache.nifi.annotation.lifecycle.OnEnabled; -import org.apache.nifi.components.PropertyDescriptor; -import org.apache.nifi.context.PropertyContext; -import org.apache.nifi.controller.ConfigurationContext; -import org.apache.nifi.controller.ControllerServiceInitializationContext; -import org.apache.nifi.reporting.InitializationException; -import org.apache.nifi.rules.Action; -import org.mvel2.MVEL; -import org.springframework.expression.Expression; -import org.springframework.expression.ExpressionParser; -import org.springframework.expression.spel.standard.SpelExpressionParser; -import org.springframework.expression.spel.support.StandardEvaluationContext; - -import java.util.ArrayList; -import java.util.Collections; -import java.util.List; -import java.util.Map; - - -@Tags({"rules", "rules engine", "action", "action handler", "expression language","MVEL","SpEL"}) -@CapabilityDescription("Executes an action containing an expression written in MVEL or SpEL. The action " + -"is usually created by a rules engine. Action objects executed with this Handler should contain \"command\" and \"type\" attributes.") -public class ExpressionHandler extends AbstractActionHandlerService { - - enum ExpresssionType { - MVEL, SPEL; - } - - public static final PropertyDescriptor DEFAULT_EXPRESSION_LANGUAGE_TYPE = new PropertyDescriptor.Builder() - .name("default-expression-language-type") - .displayName("Default Expression Language Type") - .required(true) - .description("If an expression language type is not provided as an attribute within an Action, the default expression language that " + - "should be used to compile and execute action. Supported languages are MVEL and Spring Expression Language (SpEL).") - .allowableValues(ExpresssionType.values()) - .defaultValue("MVEL") - .build(); - - private List properties; - private ExpresssionType type; - - @Override - protected void init(ControllerServiceInitializationContext config) throws InitializationException { - super.init(config); - final List properties = new ArrayList<>(); - properties.add(DEFAULT_EXPRESSION_LANGUAGE_TYPE); - properties.add(ENFORCE_ACTION_TYPE); - properties.add(ENFORCE_ACTION_TYPE_LEVEL); - this.properties = Collections.unmodifiableList(properties); - } - - @Override - @OnEnabled - public void onEnabled(final ConfigurationContext context) throws InitializationException { - super.onEnabled(context); - type = ExpresssionType.valueOf(context.getProperty(DEFAULT_EXPRESSION_LANGUAGE_TYPE).getValue()); - } - - @Override - protected List getSupportedPropertyDescriptors() { - return properties; - } - - @Override - protected void executeAction(PropertyContext propertyContext, Action action, Map facts) { - executeAction(action, facts); - } - - @Override - protected void executeAction(Action action, Map facts) { - Map attributes = action.getAttributes(); - final String command = attributes.get("command"); - if(StringUtils.isNotEmpty(command)) { - try { - final String type = attributes.getOrDefault("type",this.type.toString()); - ExpresssionType expresssionType = ExpresssionType.valueOf(type); - if (expresssionType.equals(ExpresssionType.MVEL)) { - executeMVEL(command, facts); - } else { - executeSPEL(command, facts); - } - } catch (Exception ex) { - getLogger().warn("Error occurred when attempting to execute expression. Action: {}, Facts - {}", action, facts, ex); - } - }else{ - getLogger().warn("Command attribute was not provided. Action: {}, Facts - {}", - new Object[]{action, facts}); - } - } - - private void executeMVEL(String command, Map facts) { - MVEL.executeExpression(MVEL.compileExpression(command), facts); - if(getLogger().isDebugEnabled()) { - getLogger().debug("Expression was executed successfully: {}: {}", new Object[]{type, command}); - } - } - - private void executeSPEL(String command, Map facts) { - final ExpressionParser parser = new SpelExpressionParser(); - StandardEvaluationContext context = new StandardEvaluationContext(); - context.setRootObject(facts); - context.setVariables(facts); - Expression expression = parser.parseExpression(command); - Object value = expression.getValue(context); - if(getLogger().isDebugEnabled()) { - getLogger().debug("Expression was executed successfully with result: {}. {}: {}", new Object[]{value, type, command}); - } - } - -} diff --git a/nifi-nar-bundles/nifi-rules-action-handler-bundle/nifi-rules-action-handler-service/src/main/java/org/apache/nifi/rules/handlers/LogHandler.java b/nifi-nar-bundles/nifi-rules-action-handler-bundle/nifi-rules-action-handler-service/src/main/java/org/apache/nifi/rules/handlers/LogHandler.java deleted file mode 100644 index b59bcbe3c2..0000000000 --- a/nifi-nar-bundles/nifi-rules-action-handler-bundle/nifi-rules-action-handler-service/src/main/java/org/apache/nifi/rules/handlers/LogHandler.java +++ /dev/null @@ -1,186 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.nifi.rules.handlers; - -import org.apache.commons.lang3.StringUtils; -import org.apache.nifi.annotation.documentation.CapabilityDescription; -import org.apache.nifi.annotation.documentation.Tags; -import org.apache.nifi.annotation.lifecycle.OnEnabled; -import org.apache.nifi.components.PropertyDescriptor; -import org.apache.nifi.context.PropertyContext; -import org.apache.nifi.controller.ConfigurationContext; -import org.apache.nifi.controller.ControllerServiceInitializationContext; -import org.apache.nifi.expression.ExpressionLanguageScope; -import org.apache.nifi.logging.ComponentLog; -import org.apache.nifi.logging.LogLevel; -import org.apache.nifi.processor.util.StandardValidators; -import org.apache.nifi.reporting.InitializationException; -import org.apache.nifi.rules.Action; - - -import java.util.ArrayList; -import java.util.Collections; -import java.util.List; -import java.util.Map; -import java.util.Set; - -@Tags({"rules", "rules engine", "action", "action handler", "logging"}) -@CapabilityDescription("Logs messages and fact information based on a provided action (usually created by a rules engine). " + - " Action objects executed with this Handler should contain \"logLevel\" and \"message\" attributes.") -public class LogHandler extends AbstractActionHandlerService { - - public static final PropertyDescriptor DEFAULT_LOG_LEVEL = new PropertyDescriptor.Builder() - .name("logger-default-log-level") - .displayName("Default Log Level") - .required(true) - .description("If a log level is not provided as an attribute within an Action, the default log level will be used.") - .allowableValues(DebugLevels.values()) - .defaultValue("info") - .build(); - - public static final PropertyDescriptor DEFAULT_LOG_MESSAGE = new PropertyDescriptor.Builder() - .name("logger-default-log-message") - .displayName("Default Log Message") - .required(true) - .description("If a log message is not provided as an attribute within an Action, the default log message will be used.") - .defaultValue("Rules Action Triggered Log.") - .addValidator(StandardValidators.NON_EMPTY_VALIDATOR) - .expressionLanguageSupported(ExpressionLanguageScope.VARIABLE_REGISTRY) - .build(); - - private static final PropertyDescriptor LOG_FACTS = new PropertyDescriptor.Builder() - .name("log-facts") - .displayName("Log Facts") - .required(true) - .description("If true, the log message will include the facts which triggered this log action.") - .defaultValue("true") - .allowableValues("true", "false") - .build(); - - private static final PropertyDescriptor LOG_PREFIX = new PropertyDescriptor.Builder() - .name("log-prefix") - .displayName("Log Prefix") - .required(false) - .description("Log prefix appended to the log lines. It helps to distinguish the output of multiple LogAttribute processors.") - .addValidator(StandardValidators.NON_EMPTY_VALIDATOR) - .expressionLanguageSupported(ExpressionLanguageScope.VARIABLE_REGISTRY) - .build(); - - private List properties; - private String logPrefix; - private Boolean logFacts; - private String defaultLogLevel; - private String defaultLogMessage; - - @Override - protected void init(ControllerServiceInitializationContext config) throws InitializationException { - super.init(config); - final List properties = new ArrayList<>(); - properties.add(LOG_PREFIX); - properties.add(LOG_FACTS); - properties.add(DEFAULT_LOG_LEVEL); - properties.add(DEFAULT_LOG_MESSAGE); - properties.add(ENFORCE_ACTION_TYPE); - properties.add(ENFORCE_ACTION_TYPE_LEVEL); - this.properties = Collections.unmodifiableList(properties); - } - - @Override - @OnEnabled - public void onEnabled(final ConfigurationContext context) throws InitializationException { - super.onEnabled(context); - logPrefix = context.getProperty(LOG_PREFIX).evaluateAttributeExpressions().getValue(); - logFacts = context.getProperty(LOG_FACTS).asBoolean(); - defaultLogLevel = context.getProperty(DEFAULT_LOG_LEVEL).getValue().toUpperCase(); - defaultLogMessage = context.getProperty(DEFAULT_LOG_MESSAGE).evaluateAttributeExpressions().getValue(); - } - - @Override - protected List getSupportedPropertyDescriptors() { - return properties; - } - - @Override - protected void executeAction(PropertyContext propertyContext, Action action, Map facts) { - executeAction(action, facts); - } - - @Override - protected void executeAction(Action action, Map facts) { - ComponentLog logger = getLogger(); - Map attributes = action.getAttributes(); - final String logLevel = attributes.get("logLevel"); - final LogLevel level = getLogLevel(logLevel, LogLevel.valueOf(defaultLogLevel)); - final String eventMessage = StringUtils.isNotEmpty(attributes.get("message")) ? attributes.get("message") : defaultLogMessage; - final String factsMessage = createFactsLogMessage(facts, eventMessage); - logger.log(level, factsMessage); - } - - private LogLevel getLogLevel(String logLevel, LogLevel defaultLevel) { - LogLevel level; - if (StringUtils.isNotEmpty(logLevel)) { - try { - level = LogLevel.valueOf(logLevel.toUpperCase()); - } catch (IllegalArgumentException iea) { - level = defaultLevel; - } - } else { - level = defaultLevel; - } - return level; - } - - protected String createFactsLogMessage(Map facts, String eventMessage) { - - final Set fields = facts.keySet(); - final StringBuilder message = new StringBuilder(); - String dashedLine; - - if (StringUtils.isBlank(logPrefix)) { - dashedLine = StringUtils.repeat('-', 50); - } else { - // abbreviate long lines - logPrefix = StringUtils.abbreviate(logPrefix, 40); - // center the logPrefix and pad with dashes - logPrefix = StringUtils.center(logPrefix, 40, '-'); - // five dashes on the left and right side, plus the dashed logPrefix - dashedLine = StringUtils.repeat('-', 5) + logPrefix + StringUtils.repeat('-', 5); - } - - message.append("\n"); - message.append(dashedLine); - message.append("\n"); - message.append("Log Message: "); - message.append(eventMessage); - message.append("\n"); - - if (logFacts) { - message.append("Log Facts:\n"); - fields.forEach(field -> { - message.append("Field: "); - message.append(field); - message.append(", Value: "); - message.append(facts.get(field)); - message.append("\n"); - }); - } - - return message.toString().trim(); - - } - -} diff --git a/nifi-nar-bundles/nifi-rules-action-handler-bundle/nifi-rules-action-handler-service/src/main/java/org/apache/nifi/rules/handlers/RecordSinkHandler.java b/nifi-nar-bundles/nifi-rules-action-handler-bundle/nifi-rules-action-handler-service/src/main/java/org/apache/nifi/rules/handlers/RecordSinkHandler.java deleted file mode 100644 index 55c0de4127..0000000000 --- a/nifi-nar-bundles/nifi-rules-action-handler-bundle/nifi-rules-action-handler-service/src/main/java/org/apache/nifi/rules/handlers/RecordSinkHandler.java +++ /dev/null @@ -1,160 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.nifi.rules.handlers; - -import org.apache.commons.lang3.math.NumberUtils; -import org.apache.nifi.annotation.documentation.CapabilityDescription; -import org.apache.nifi.annotation.documentation.Tags; -import org.apache.nifi.annotation.lifecycle.OnEnabled; -import org.apache.nifi.components.PropertyDescriptor; -import org.apache.nifi.context.PropertyContext; -import org.apache.nifi.controller.ConfigurationContext; -import org.apache.nifi.controller.ControllerServiceInitializationContext; -import org.apache.nifi.record.sink.RecordSinkService; -import org.apache.nifi.reporting.InitializationException; -import org.apache.nifi.rules.Action; -import org.apache.nifi.serialization.SimpleRecordSchema; -import org.apache.nifi.serialization.WriteResult; -import org.apache.nifi.serialization.record.DataType; -import org.apache.nifi.serialization.record.ListRecordSet; -import org.apache.nifi.serialization.record.MapRecord; -import org.apache.nifi.serialization.record.RecordField; -import org.apache.nifi.serialization.record.RecordFieldType; -import org.apache.nifi.serialization.record.RecordSchema; -import org.apache.nifi.serialization.record.RecordSet; - -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collections; -import java.util.List; -import java.util.Map; -import java.util.stream.Collectors; - -@Tags({"rules", "rules engine", "action", "action handler", "record", "record sink"}) -@CapabilityDescription("Sends fact information to sink based on a provided action (usually created by a rules engine)." + - " Action objects executed with this Handler should contain \"sendZeroResult\" attribute.") -public class RecordSinkHandler extends AbstractActionHandlerService{ - - static final PropertyDescriptor RECORD_SINK_SERVICE = new PropertyDescriptor.Builder() - .name("record-sink-service") - .displayName("Record Sink Service") - .description("Specifies the Controller Service used to support the SEND event action. If not set SEND events will be ignored.") - .identifiesControllerService(RecordSinkService.class) - .required(true) - .build(); - - private RecordSinkService recordSinkService; - private List properties; - - @Override - protected void init(ControllerServiceInitializationContext config) throws InitializationException { - super.init(config); - final List properties = new ArrayList<>(); - properties.add(RECORD_SINK_SERVICE); - properties.add(ENFORCE_ACTION_TYPE); - properties.add(ENFORCE_ACTION_TYPE_LEVEL); - this.properties = Collections.unmodifiableList(properties); - } - - @Override - protected List getSupportedPropertyDescriptors() { - return properties; - } - - @Override - @OnEnabled - public void onEnabled(final ConfigurationContext context) throws InitializationException { - super.onEnabled(context); - if(context.getProperty(RECORD_SINK_SERVICE).isSet()) { - recordSinkService = context.getProperty(RECORD_SINK_SERVICE).asControllerService(RecordSinkService.class); - } - } - - @Override - protected void executeAction(PropertyContext propertyContext, Action action, Map facts) { - executeAction(action, facts); - } - - @Override - protected void executeAction(Action action, Map facts) { - Map attributes = action.getAttributes(); - boolean sendZeroResults = attributes.containsKey("sentZeroResults") && Boolean.parseBoolean(attributes.get("sendZeroResults")); - final RecordSet recordSet = getRecordSet(facts); - - try { - WriteResult result = recordSinkService.sendData(recordSet, attributes, sendZeroResults); - if (getLogger().isDebugEnabled() && result != null) { - getLogger().debug("Records written to sink service: {}", new Object[]{result.getRecordCount()}); - } - } catch (Exception ex) { - getLogger().warn("Exception encountered when attempting to send metrics", ex); - } - } - - private RecordSet getRecordSet(Map metrics){ - List recordFields = metrics.entrySet().stream().map(entry -> - new RecordField(entry.getKey(),getDataType(String.valueOf(entry.getValue()))) - ).collect(Collectors.toList()); - final RecordSchema recordSchema = new SimpleRecordSchema(recordFields); - return new ListRecordSet(recordSchema, Arrays.asList( new MapRecord(recordSchema,metrics))); - } - - private DataType getDataType(final String value) { - if (value == null || value.isEmpty()) { - return null; - } - - if (NumberUtils.isParsable(value)) { - if (value.contains(".")) { - try { - final double doubleValue = Double.parseDouble(value); - - if (doubleValue == Double.POSITIVE_INFINITY || doubleValue == Double.NEGATIVE_INFINITY) { - return RecordFieldType.DECIMAL.getDecimalDataType(value.length() - 1, value.length() - 1 - value.indexOf(".")); - } - - if (doubleValue > Float.MAX_VALUE || doubleValue < Float.MIN_VALUE) { - return RecordFieldType.DOUBLE.getDataType(); - } - - return RecordFieldType.FLOAT.getDataType(); - } catch (final NumberFormatException nfe) { - return RecordFieldType.STRING.getDataType(); - } - } - - try { - final long longValue = Long.parseLong(value); - if (longValue > Integer.MAX_VALUE || longValue < Integer.MIN_VALUE) { - return RecordFieldType.LONG.getDataType(); - } - - return RecordFieldType.INT.getDataType(); - } catch (final NumberFormatException nfe) { - return RecordFieldType.STRING.getDataType(); - } - } - - if (value.equalsIgnoreCase("true") || value.equalsIgnoreCase("false")) { - return RecordFieldType.BOOLEAN.getDataType(); - } - - return RecordFieldType.STRING.getDataType(); - - } - -} diff --git a/nifi-nar-bundles/nifi-rules-action-handler-bundle/nifi-rules-action-handler-service/src/main/resources/META-INF/services/org.apache.nifi.controller.ControllerService b/nifi-nar-bundles/nifi-rules-action-handler-bundle/nifi-rules-action-handler-service/src/main/resources/META-INF/services/org.apache.nifi.controller.ControllerService deleted file mode 100644 index 461f818f3c..0000000000 --- a/nifi-nar-bundles/nifi-rules-action-handler-bundle/nifi-rules-action-handler-service/src/main/resources/META-INF/services/org.apache.nifi.controller.ControllerService +++ /dev/null @@ -1,19 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -org.apache.nifi.rules.handlers.ActionHandlerLookup -org.apache.nifi.rules.handlers.ExpressionHandler -org.apache.nifi.rules.handlers.LogHandler -org.apache.nifi.rules.handlers.RecordSinkHandler -org.apache.nifi.rules.handlers.AlertHandler \ No newline at end of file diff --git a/nifi-nar-bundles/nifi-rules-action-handler-bundle/nifi-rules-action-handler-service/src/main/resources/docs/org.apache.nifi.rules.handlers.AlertHandler/additionalDetails.html b/nifi-nar-bundles/nifi-rules-action-handler-bundle/nifi-rules-action-handler-service/src/main/resources/docs/org.apache.nifi.rules.handlers.AlertHandler/additionalDetails.html deleted file mode 100644 index ea1a49851a..0000000000 --- a/nifi-nar-bundles/nifi-rules-action-handler-bundle/nifi-rules-action-handler-service/src/main/resources/docs/org.apache.nifi.rules.handlers.AlertHandler/additionalDetails.html +++ /dev/null @@ -1,39 +0,0 @@ - - - - - - AlertHandler - - - - - -

Summary

-

- The AlertHandler is used to broadcast alerts (bulletins) as dictated by the action object. Action objects can include attributes to configure - the handler otherwise default values will be used. Possible attribute values are listed below. -

-

ExpressionHandler Service Attributes

- - - - - -
AttributeDescription
categoryThe category the alert should be grouped under.
logLevelLog Level for the alert. Possible values are trace, debug, info, warn, error.
messageMessage for the alert.
-
- - \ No newline at end of file diff --git a/nifi-nar-bundles/nifi-rules-action-handler-bundle/nifi-rules-action-handler-service/src/main/resources/docs/org.apache.nifi.rules.handlers.ExpressionHandler/additionalDetails.html b/nifi-nar-bundles/nifi-rules-action-handler-bundle/nifi-rules-action-handler-service/src/main/resources/docs/org.apache.nifi.rules.handlers.ExpressionHandler/additionalDetails.html deleted file mode 100644 index 8dff1bc188..0000000000 --- a/nifi-nar-bundles/nifi-rules-action-handler-bundle/nifi-rules-action-handler-service/src/main/resources/docs/org.apache.nifi.rules.handlers.ExpressionHandler/additionalDetails.html +++ /dev/null @@ -1,38 +0,0 @@ - - - - - - ExpressionHandler - - - - - -

Summary

-

- The ExpressionHandler is used to execute dynamic commands writtin in MVEL or SpEL expression language. Action objects must include attributes to configure - the handler otherwise an exception will be thrown. Possible attribute values are listed below. -

-

ExpressionHandler Service Attributes

- - - - -
AttributeDescription
typeThe expression language type of the command to be executed. Possible values are MVEL and SpEl (MVEL will be applied by default if type is not provided).
commandThe expression language command that should be executed
-
- - \ No newline at end of file diff --git a/nifi-nar-bundles/nifi-rules-action-handler-bundle/nifi-rules-action-handler-service/src/main/resources/docs/org.apache.nifi.rules.handlers.LogHandler/additionalDetails.html b/nifi-nar-bundles/nifi-rules-action-handler-bundle/nifi-rules-action-handler-service/src/main/resources/docs/org.apache.nifi.rules.handlers.LogHandler/additionalDetails.html deleted file mode 100644 index b9b487da23..0000000000 --- a/nifi-nar-bundles/nifi-rules-action-handler-bundle/nifi-rules-action-handler-service/src/main/resources/docs/org.apache.nifi.rules.handlers.LogHandler/additionalDetails.html +++ /dev/null @@ -1,38 +0,0 @@ - - - - - - LogHandler - - - - - -

Summary

-

- The LogHandler is used to execute actions that dictate to log a message and/or metrics. LogHandler can be invoked with any Action object. - Action objects can include attributes to configure the LogHandler or rely on the handler's default settings. Possible attribute values are listed below. -

-

LogHandler Service Attributes

- - - - -
AttributeDescription
logLevelLog Level for logged message. Possible values are trace, debug, info, warn, error.
messageMessage for log.
-
- - \ No newline at end of file diff --git a/nifi-nar-bundles/nifi-rules-action-handler-bundle/nifi-rules-action-handler-service/src/main/resources/docs/org.apache.nifi.rules.handlers.RecordSinkHandler/additionalDetails.html b/nifi-nar-bundles/nifi-rules-action-handler-bundle/nifi-rules-action-handler-service/src/main/resources/docs/org.apache.nifi.rules.handlers.RecordSinkHandler/additionalDetails.html deleted file mode 100644 index 4633f37a40..0000000000 --- a/nifi-nar-bundles/nifi-rules-action-handler-bundle/nifi-rules-action-handler-service/src/main/resources/docs/org.apache.nifi.rules.handlers.RecordSinkHandler/additionalDetails.html +++ /dev/null @@ -1,37 +0,0 @@ - - - - - - RecordSinkHandler - - - - - -

Summary

-

- The RecordSinkHandler is used to execute actions that send metrics information to a configured sink. RecordSinkHandler can be invoked with any Action object. - Action objects can include attributes to configure the handler. Possible attribute values are listed below. -

-

RecordSinkHandler Service Attributes

- - - -
AttributeDescription
sendZeroResultsAllow empty results to be sent to sink. Possible values are true and false (default is false).
-
- - \ No newline at end of file diff --git a/nifi-nar-bundles/nifi-rules-action-handler-bundle/nifi-rules-action-handler-service/src/test/java/org/apache/nifi/rules/handlers/MockComponentLog.java b/nifi-nar-bundles/nifi-rules-action-handler-bundle/nifi-rules-action-handler-service/src/test/java/org/apache/nifi/rules/handlers/MockComponentLog.java deleted file mode 100644 index 827ca3a0e9..0000000000 --- a/nifi-nar-bundles/nifi-rules-action-handler-bundle/nifi-rules-action-handler-service/src/test/java/org/apache/nifi/rules/handlers/MockComponentLog.java +++ /dev/null @@ -1,200 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.nifi.rules.handlers; - -import org.apache.nifi.logging.ComponentLog; -import org.apache.nifi.logging.LogLevel; -import org.apache.nifi.logging.LogMessage; - -public class MockComponentLog implements ComponentLog { - - String infoMessage; - String warnMessage; - String debugMessage; - String traceMessage; - String errorMessage; - - protected String convertMessage(String msg, Object[] os){ - String replaceMsg = msg; - for (Object o : os) { - replaceMsg = replaceMsg.replaceFirst("\\{\\}", os.toString()); - } - return replaceMsg; - } - - @Override - public void warn(String msg, Throwable t) { - warn(msg); - } - - @Override - public void warn(String msg, Object... os) { - warn(msg); - } - - @Override - public void warn(String msg) { - warnMessage = msg; - } - - @Override - public void warn(LogMessage logMessage) { - warnMessage = logMessage.getMessage(); - } - - @Override - public void trace(String msg, Throwable t) { - trace(msg); - } - - @Override - public void trace(String msg, Object... os) { - trace(msg); - } - - @Override - public void trace(String msg) { - traceMessage = msg; - } - - @Override - public void trace(LogMessage logMessage) { - traceMessage = logMessage.getMessage(); - } - - @Override - public boolean isWarnEnabled() { - return true; - } - - @Override - public boolean isTraceEnabled() { - return false; - } - - @Override - public boolean isInfoEnabled() { - return false; - } - - @Override - public boolean isErrorEnabled() { - return false; - } - - @Override - public boolean isDebugEnabled() { - return true; - } - - @Override - public void info(String msg, Throwable t) { - info(msg); - } - - @Override - public void info(String msg, Object... os) { - info(msg); - } - - @Override - public void info(String msg) { - infoMessage = msg; - } - - @Override - public void info(LogMessage message) { - infoMessage = message.getMessage(); - } - - @Override - public String getName() { - return null; - } - - @Override - public void error(String msg, Throwable t) { - error(msg); - } - - @Override - public void error(String msg, Object... os) { - error(convertMessage(msg, os)); - } - - @Override - public void error(String msg) { - errorMessage = msg; - } - - @Override - public void error(LogMessage message) { - errorMessage = message.getMessage(); - } - - @Override - public void debug(String msg, Throwable t) { - debug(msg); - } - - @Override - public void debug(String msg, Object... os) { - debug(convertMessage(msg, os)); - } - - @Override - public void debug(String msg) { - debugMessage = msg; - } - - @Override - public void debug(LogMessage message) { - debugMessage = message.getMessage(); - } - - @Override - public void log(LogLevel level, String msg, Throwable t) { - - } - - @Override - public void log(LogLevel level, String msg, Object... os) { - - } - - public String getInfoMessage() { - return infoMessage; - } - - public String getWarnMessage() { - return warnMessage; - } - - public String getDebugMessage() { - return debugMessage; - } - - public String getTraceMessage() { - return traceMessage; - } - - public String getErrorMessage() { - return errorMessage; - } - - -} diff --git a/nifi-nar-bundles/nifi-rules-action-handler-bundle/nifi-rules-action-handler-service/src/test/java/org/apache/nifi/rules/handlers/TestActionHandlerLookup.java b/nifi-nar-bundles/nifi-rules-action-handler-bundle/nifi-rules-action-handler-service/src/test/java/org/apache/nifi/rules/handlers/TestActionHandlerLookup.java deleted file mode 100644 index a015427e57..0000000000 --- a/nifi-nar-bundles/nifi-rules-action-handler-bundle/nifi-rules-action-handler-service/src/test/java/org/apache/nifi/rules/handlers/TestActionHandlerLookup.java +++ /dev/null @@ -1,135 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.nifi.rules.handlers; - -import org.apache.nifi.components.PropertyDescriptor; -import org.apache.nifi.context.PropertyContext; -import org.apache.nifi.processor.exception.ProcessException; -import org.apache.nifi.reporting.InitializationException; -import org.apache.nifi.rules.Action; -import org.apache.nifi.util.TestRunner; -import org.apache.nifi.util.TestRunners; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; - -import java.util.Arrays; -import java.util.HashMap; -import java.util.List; -import java.util.Map; - -import static org.junit.jupiter.api.Assertions.assertThrows; -import static org.junit.jupiter.api.Assertions.assertTrue; - -public class TestActionHandlerLookup { - - private MockPropertyActionHandler alertHandler; - private MockPropertyActionHandler logHandler; - private ActionHandlerLookup actionHandlerLookup; - private TestRunner runner; - - @BeforeEach - public void setup() throws InitializationException { - alertHandler = new MockPropertyActionHandler(); - logHandler = new MockPropertyActionHandler(); - actionHandlerLookup = new ActionHandlerLookup(); - - runner = TestRunners.newTestRunner(TestProcessor.class); - - final String alertIdentifier = "alert-handler"; - runner.addControllerService(alertIdentifier, alertHandler); - - final String logIdentifier = "log-handler"; - runner.addControllerService(logIdentifier, logHandler); - - runner.addControllerService("action-handler-lookup", actionHandlerLookup); - runner.setProperty(actionHandlerLookup, "ALERT", alertIdentifier); - runner.setProperty(actionHandlerLookup, "LOG", logIdentifier); - - runner.enableControllerService(alertHandler); - runner.enableControllerService(logHandler); - runner.enableControllerService(actionHandlerLookup); - - } - - @Test - public void testLookupAlert() { - final Map attributes = new HashMap<>(); - final Map metrics = new HashMap<>(); - attributes.put("logLevel", "INFO"); - attributes.put("message", "This should be not sent as an alert!"); - metrics.put("jvmHeap", "1000000"); - metrics.put("cpu", "90"); - final Action action = new Action(); - action.setType("ALERT"); - action.setAttributes(attributes); - actionHandlerLookup.execute(null, action, metrics); - assertTrue(alertHandler.getExecuteContextCalled()); - } - - @Test - public void testLookupLog() { - final Map attributes = new HashMap<>(); - final Map metrics = new HashMap<>(); - attributes.put("logLevel", "INFO"); - attributes.put("message", "This should be not sent as an alert!"); - metrics.put("jvmHeap", "1000000"); - metrics.put("cpu", "90"); - final Action action = new Action(); - action.setType("LOG"); - action.setAttributes(attributes); - actionHandlerLookup.execute(null, action, metrics); - assertTrue(logHandler.getExecuteContextCalled()); - } - - @Test - public void testLookupInvalidActionType() { - final Map attributes = new HashMap<>(); - final Map metrics = new HashMap<>(); - attributes.put("logLevel", "INFO"); - attributes.put("message", "This should be not sent as an alert!"); - metrics.put("jvmHeap", "1000000"); - metrics.put("cpu", "90"); - final Action action = new Action(); - action.setType("FAKE"); - assertThrows(ProcessException.class, () -> actionHandlerLookup.execute(null,action,metrics)); - } - - private static class MockPropertyActionHandler extends AbstractActionHandlerService { - - Boolean executeContextCalled = false; - - @Override - public void execute(Action action, Map facts) { - execute(null,action,facts); - } - - @Override - public void execute(PropertyContext context, Action action, Map facts) { - executeContextCalled = true; - } - - public Boolean getExecuteContextCalled() { - return executeContextCalled; - } - - @Override - protected List getSupportedPropertyDescriptors() { - return Arrays.asList(ENFORCE_ACTION_TYPE, ENFORCE_ACTION_TYPE_LEVEL); - } - } - -} diff --git a/nifi-nar-bundles/nifi-rules-action-handler-bundle/nifi-rules-action-handler-service/src/test/java/org/apache/nifi/rules/handlers/TestAlertHandler.java b/nifi-nar-bundles/nifi-rules-action-handler-bundle/nifi-rules-action-handler-service/src/test/java/org/apache/nifi/rules/handlers/TestAlertHandler.java deleted file mode 100644 index 3688940a0b..0000000000 --- a/nifi-nar-bundles/nifi-rules-action-handler-bundle/nifi-rules-action-handler-service/src/test/java/org/apache/nifi/rules/handlers/TestAlertHandler.java +++ /dev/null @@ -1,366 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.nifi.rules.handlers; - -import org.apache.commons.lang3.StringUtils; -import org.apache.nifi.components.PropertyDescriptor; -import org.apache.nifi.components.PropertyValue; -import org.apache.nifi.context.PropertyContext; -import org.apache.nifi.logging.ComponentLog; -import org.apache.nifi.reporting.Bulletin; -import org.apache.nifi.reporting.BulletinFactory; -import org.apache.nifi.reporting.BulletinRepository; -import org.apache.nifi.reporting.InitializationException; -import org.apache.nifi.reporting.ReportingContext; -import org.apache.nifi.reporting.Severity; -import org.apache.nifi.rules.Action; -import org.apache.nifi.util.MockBulletinRepository; -import org.apache.nifi.util.TestRunner; -import org.apache.nifi.util.TestRunners; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; -import org.mockito.Mockito; - -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; - -import static org.hamcrest.MatcherAssert.assertThat; -import static org.hamcrest.core.IsInstanceOf.instanceOf; -import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertFalse; -import static org.junit.jupiter.api.Assertions.assertThrows; -import static org.junit.jupiter.api.Assertions.assertTrue; -import static org.mockito.ArgumentMatchers.anyString; - -public class TestAlertHandler { - - private TestRunner runner; - private MockComponentLog mockComponentLog; - private ReportingContext reportingContext; - private AlertHandler alertHandler; - private MockAlertBulletinRepository mockAlertBulletinRepository; - - @BeforeEach - public void setup() throws InitializationException { - runner = TestRunners.newTestRunner(TestProcessor.class); - mockComponentLog = new MockComponentLog(); - AlertHandler handler = new MockAlertHandler(mockComponentLog); - mockAlertBulletinRepository = new MockAlertBulletinRepository(); - runner.addControllerService("MockAlertHandler", handler); - runner.enableControllerService(handler); - alertHandler = (AlertHandler) runner.getProcessContext() - .getControllerServiceLookup() - .getControllerService("MockAlertHandler"); - reportingContext = Mockito.mock(ReportingContext.class); - Mockito.when(reportingContext.getBulletinRepository()).thenReturn(mockAlertBulletinRepository); - Mockito.when(reportingContext.createBulletin(anyString(), Mockito.any(Severity.class), anyString())) - .thenAnswer(invocation -> - BulletinFactory.createBulletin(invocation.getArgument(0), invocation.getArgument(1).toString(), invocation.getArgument(2))); - } - - @Test - public void testValidService() { - runner.assertValid(alertHandler); - assertThat(alertHandler, instanceOf(AlertHandler.class)); - } - - @Test - public void testAlertNoReportingContext() { - - final Map attributes = new HashMap<>(); - final Map metrics = new HashMap<>(); - - attributes.put("logLevel", "INFO"); - attributes.put("message", "This should be not sent as an alert!"); - metrics.put("jvmHeap", "1000000"); - metrics.put("cpu", "90"); - - final Action action = new Action(); - action.setType("ALERT"); - action.setAttributes(attributes); - assertThrows(UnsupportedOperationException.class, () -> alertHandler.execute(action, metrics)); - } - - @Test - public void testAlertWithBulletinLevel() { - - final Map attributes = new HashMap<>(); - final Map metrics = new HashMap<>(); - - final String category = "Rules Alert"; - final String message = "This should be sent as an alert!"; - final String severity = "INFO"; - attributes.put("category", category); - attributes.put("message", message); - attributes.put("severity", severity); - metrics.put("jvmHeap", "1000000"); - metrics.put("cpu", "90"); - - final String expectedOutput = "This should be sent as an alert!\n" + - "Alert Facts:\n" + - "Field: cpu, Value: 90\n" + - "Field: jvmHeap, Value: 1000000\n"; - - final Action action = new Action(); - action.setType("ALERT"); - action.setAttributes(attributes); - alertHandler.execute(reportingContext, action, metrics); - BulletinRepository bulletinRepository = reportingContext.getBulletinRepository(); - List bulletins = bulletinRepository.findBulletinsForController(); - assertFalse(bulletins.isEmpty()); - Bulletin bulletin = bulletins.get(0); - assertEquals(bulletin.getCategory(), category); - assertEquals(bulletin.getMessage(), expectedOutput); - assertEquals(bulletin.getLevel(), severity); - } - - @Test - public void testAlertWithDefaultValues() { - - final Map attributes = new HashMap<>(); - final Map metrics = new HashMap<>(); - - final String category = "Rules Triggered Alert"; - final String message = "An alert was triggered by a rules based action."; - final String severity = "INFO"; - metrics.put("jvmHeap", "1000000"); - metrics.put("cpu", "90"); - - final String expectedOutput = "An alert was triggered by a rules-based action.\n" + - "Alert Facts:\n" + - "Field: cpu, Value: 90\n" + - "Field: jvmHeap, Value: 1000000\n"; - - final Action action = new Action(); - action.setType("ALERT"); - action.setAttributes(attributes); - alertHandler.execute(reportingContext, action, metrics); - BulletinRepository bulletinRepository = reportingContext.getBulletinRepository(); - List bulletins = bulletinRepository.findBulletinsForController(); - assertFalse(bulletins.isEmpty()); - Bulletin bulletin = bulletins.get(0); - assertEquals(bulletin.getCategory(), category); - assertEquals(bulletin.getMessage(), expectedOutput); - assertEquals(bulletin.getLevel(), severity); - } - - @Test - public void testInvalidContext(){ - final Map attributes = new HashMap<>(); - final Map metrics = new HashMap<>(); - - final String category = "Rules Alert"; - final String message = "This should be sent as an alert!"; - final String severity = "INFO"; - attributes.put("category", category); - attributes.put("message", message); - attributes.put("severity", severity); - metrics.put("jvmHeap", "1000000"); - metrics.put("cpu", "90"); - - final Action action = new Action(); - action.setType("ALERT"); - action.setAttributes(attributes); - PropertyContext fakeContext = new PropertyContext() { - @Override - public PropertyValue getProperty(PropertyDescriptor descriptor) { - return null; - } - - @Override - public Map getAllProperties() { - return null; - } - }; - alertHandler.execute(fakeContext, action, metrics); - final String debugMessage = mockComponentLog.getWarnMessage(); - assertTrue(StringUtils.isNotEmpty(debugMessage)); - assertEquals(debugMessage,"Reporting context was not provided to create bulletins."); - } - - @Test - public void testEmptyBulletinRepository(){ - final Map attributes = new HashMap<>(); - final Map metrics = new HashMap<>(); - - final String category = "Rules Alert"; - final String message = "This should be sent as an alert!"; - final String severity = "INFO"; - attributes.put("category", category); - attributes.put("message", message); - attributes.put("severity", severity); - metrics.put("jvmHeap", "1000000"); - metrics.put("cpu", "90"); - - final Action action = new Action(); - action.setType("ALERT"); - action.setAttributes(attributes); - ReportingContext fakeContext = Mockito.mock(ReportingContext.class); - Mockito.when(reportingContext.getBulletinRepository()).thenReturn(null); - alertHandler.execute(fakeContext, action, metrics); - final String warnMessage = mockComponentLog.getWarnMessage(); - assertTrue(StringUtils.isNotEmpty(warnMessage)); - assertEquals(warnMessage,"Bulletin Repository is not available which is unusual. Cannot send a bulletin."); - } - - @Test - public void testInvalidActionTypeException(){ - - runner.disableControllerService(alertHandler); - runner.setProperty(alertHandler, AlertHandler.ENFORCE_ACTION_TYPE, "ALERT"); - runner.setProperty(alertHandler, AlertHandler.ENFORCE_ACTION_TYPE_LEVEL, "EXCEPTION"); - runner.enableControllerService(alertHandler); - final Map attributes = new HashMap<>(); - final Map metrics = new HashMap<>(); - - final String category = "Rules Alert"; - final String message = "This should be sent as an alert!"; - final String severity = "INFO"; - attributes.put("category", category); - attributes.put("message", message); - attributes.put("severity", severity); - metrics.put("jvmHeap", "1000000"); - metrics.put("cpu", "90"); - - final Action action = new Action(); - action.setType("FAKE"); - action.setAttributes(attributes); - assertThrows(UnsupportedOperationException.class, () -> alertHandler.execute(reportingContext, action, metrics)); - } - - @Test - public void testInvalidActionTypeWarn(){ - - runner.disableControllerService(alertHandler); - runner.setProperty(alertHandler, AlertHandler.ENFORCE_ACTION_TYPE, "ALERT"); - runner.setProperty(alertHandler, AlertHandler.ENFORCE_ACTION_TYPE_LEVEL, "WARN"); - runner.enableControllerService(alertHandler); - final Map attributes = new HashMap<>(); - final Map metrics = new HashMap<>(); - - final String category = "Rules Alert"; - final String message = "This should be sent as an alert!"; - final String severity = "INFO"; - attributes.put("category", category); - attributes.put("message", message); - attributes.put("severity", severity); - metrics.put("jvmHeap", "1000000"); - metrics.put("cpu", "90"); - - final Action action = new Action(); - action.setType("FAKE"); - action.setAttributes(attributes); - - assertDoesNotThrow(() -> alertHandler.execute(reportingContext,action, metrics)); - - final String warnMessage = mockComponentLog.getWarnMessage(); - assertTrue(StringUtils.isNotEmpty(warnMessage)); - assertEquals("This Action Handler does not support actions with the provided type: FAKE",warnMessage); - } - - @Test - public void testInvalidActionTypeIgnore(){ - - runner.disableControllerService(alertHandler); - runner.setProperty(alertHandler, AlertHandler.ENFORCE_ACTION_TYPE, "ALERT"); - runner.setProperty(alertHandler, AlertHandler.ENFORCE_ACTION_TYPE_LEVEL, "IGNORE"); - runner.enableControllerService(alertHandler); - final Map attributes = new HashMap<>(); - final Map metrics = new HashMap<>(); - - final String category = "Rules Alert"; - final String message = "This should be sent as an alert!"; - final String severity = "INFO"; - attributes.put("category", category); - attributes.put("message", message); - attributes.put("severity", severity); - metrics.put("jvmHeap", "1000000"); - metrics.put("cpu", "90"); - - final Action action = new Action(); - action.setType("FAKE"); - action.setAttributes(attributes); - assertDoesNotThrow(() -> alertHandler.execute(reportingContext,action, metrics)); - - final String debugMessage = mockComponentLog.getDebugMessage(); - assertTrue(StringUtils.isNotEmpty(debugMessage)); - assertEquals("This Action Handler does not support actions with the provided type: FAKE",debugMessage); - } - - @Test - public void testValidActionType(){ - runner.disableControllerService(alertHandler); - runner.setProperty(alertHandler, AlertHandler.ENFORCE_ACTION_TYPE, "ALERT, LOG, "); - runner.enableControllerService(alertHandler); - final Map attributes = new HashMap<>(); - final Map metrics = new HashMap<>(); - - final String category = "Rules Alert"; - final String message = "This should be sent as an alert!"; - final String severity = "INFO"; - attributes.put("category", category); - attributes.put("message", message); - attributes.put("severity", severity); - metrics.put("jvmHeap", "1000000"); - metrics.put("cpu", "90"); - - final Action action = new Action(); - action.setType("ALERT"); - action.setAttributes(attributes); - assertDoesNotThrow(() -> alertHandler.execute(reportingContext,action, metrics)); - } - - private static class MockAlertHandler extends AlertHandler { - - private ComponentLog testLogger; - - public MockAlertHandler(ComponentLog testLogger) { - this.testLogger = testLogger; - } - - @Override - protected ComponentLog getLogger() { - return testLogger; - } - - } - - private static class MockAlertBulletinRepository extends MockBulletinRepository { - - List bulletinList; - - - public MockAlertBulletinRepository() { - bulletinList = new ArrayList<>(); - } - - @Override - public void addBulletin(Bulletin bulletin) { - bulletinList.add(bulletin); - } - - @Override - public List findBulletinsForController() { - return bulletinList; - } - - } - -} diff --git a/nifi-nar-bundles/nifi-rules-action-handler-bundle/nifi-rules-action-handler-service/src/test/java/org/apache/nifi/rules/handlers/TestExpressionHandler.java b/nifi-nar-bundles/nifi-rules-action-handler-bundle/nifi-rules-action-handler-service/src/test/java/org/apache/nifi/rules/handlers/TestExpressionHandler.java deleted file mode 100644 index e65e17469c..0000000000 --- a/nifi-nar-bundles/nifi-rules-action-handler-bundle/nifi-rules-action-handler-service/src/test/java/org/apache/nifi/rules/handlers/TestExpressionHandler.java +++ /dev/null @@ -1,236 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.nifi.rules.handlers; - -import org.apache.commons.lang3.StringUtils; -import org.apache.nifi.logging.ComponentLog; -import org.apache.nifi.reporting.InitializationException; -import org.apache.nifi.rules.Action; -import org.apache.nifi.util.TestRunner; -import org.apache.nifi.util.TestRunners; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; - -import java.util.HashMap; -import java.util.Map; - -import static org.hamcrest.MatcherAssert.assertThat; -import static org.hamcrest.core.IsInstanceOf.instanceOf; -import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertThrows; -import static org.junit.jupiter.api.Assertions.assertTrue; - -public class TestExpressionHandler { - - private TestRunner runner; - private MockComponentLog mockComponentLog; - private ExpressionHandler expressionHandler; - - @BeforeEach - public void setup() throws InitializationException { - runner = TestRunners.newTestRunner(TestProcessor.class); - mockComponentLog = new MockComponentLog(); - ExpressionHandler handler = new MockExpressionHandler(mockComponentLog); - runner.addControllerService("MockExpressionHandler", handler); - runner.enableControllerService(handler); - expressionHandler = (ExpressionHandler) runner.getProcessContext() - .getControllerServiceLookup() - .getControllerService("MockExpressionHandler"); - } - - @Test - public void testValidService() { - runner.assertValid(expressionHandler); - assertThat(expressionHandler, instanceOf(ExpressionHandler.class)); - } - - @Test - public void testMvelExpression(){ - - final Map attributes = new HashMap<>(); - final Map metrics = new HashMap<>(); - final String expectedMessage = "Expression was executed successfully:"; - - attributes.put("command","System.out.println(jvmHeap)"); - attributes.put("type","MVEL"); - metrics.put("jvmHeap","1000000"); - metrics.put("cpu","90"); - - final Action action = new Action(); - action.setType("EXPRESSION"); - action.setAttributes(attributes); - expressionHandler.execute(action, metrics); - String logMessage = mockComponentLog.getDebugMessage(); - assertTrue(StringUtils.isNotEmpty(logMessage)); - assertTrue(logMessage.startsWith(expectedMessage)); - } - - @Test - public void testSpelExpression(){ - final Map attributes = new HashMap<>(); - final Map metrics = new HashMap<>(); - final String expectedMessage = "Expression was executed successfully with result:"; - - attributes.put("command","#jvmHeap + ' is large'"); - attributes.put("type","SPEL"); - metrics.put("jvmHeap","1000000"); - metrics.put("cpu","90"); - - final Action action = new Action(); - action.setType("EXPRESSION"); - action.setAttributes(attributes); - expressionHandler.execute(action, metrics); - String logMessage = mockComponentLog.getDebugMessage(); - assertTrue(StringUtils.isNotEmpty(logMessage)); - assertTrue(logMessage.startsWith(expectedMessage)); - } - - @Test - public void testInvalidType() { - final Map attributes = new HashMap<>(); - final Map metrics = new HashMap<>(); - final String expectedMessage = "Error occurred when attempting to execute expression."; - - attributes.put("command","#jvmHeap + ' is large'"); - attributes.put("type","FAKE"); - metrics.put("jvmHeap","1000000"); - metrics.put("cpu","90"); - - final Action action = new Action(); - action.setType("EXPRESSION"); - action.setAttributes(attributes); - expressionHandler.execute(action, metrics); - String logMessage = mockComponentLog.getWarnMessage(); - assertTrue(StringUtils.isNotEmpty(logMessage)); - assertTrue(logMessage.startsWith(expectedMessage)); - } - - @Test - public void testNoCommandProvided() { - final Map attributes = new HashMap<>(); - final Map metrics = new HashMap<>(); - final String expectedMessage = "Command attribute was not provided."; - attributes.put("type","FAKE"); - metrics.put("jvmHeap","1000000"); - metrics.put("cpu","90"); - - final Action action = new Action(); - action.setType("EXPRESSION"); - action.setAttributes(attributes); - expressionHandler.execute(action, metrics); - String logMessage = mockComponentLog.getWarnMessage(); - assertTrue(StringUtils.isNotEmpty(logMessage)); - assertTrue(logMessage.startsWith(expectedMessage)); - } - - @Test - public void testInvalidActionTypeException() { - runner.disableControllerService(expressionHandler); - runner.setProperty(expressionHandler, AlertHandler.ENFORCE_ACTION_TYPE, "EXPRESSION"); - runner.setProperty(expressionHandler, AlertHandler.ENFORCE_ACTION_TYPE_LEVEL, "EXCEPTION"); - runner.enableControllerService(expressionHandler); - final Map attributes = new HashMap<>(); - final Map metrics = new HashMap<>(); - attributes.put("type","FAKE"); - metrics.put("jvmHeap","1000000"); - metrics.put("cpu","90"); - - final Action action = new Action(); - action.setType("FAKE"); - action.setAttributes(attributes); - - assertThrows(UnsupportedOperationException.class, () -> expressionHandler.execute(action, metrics)); - } - - @Test - public void testInvalidActionTypeWarning() { - runner.disableControllerService(expressionHandler); - runner.setProperty(expressionHandler, AlertHandler.ENFORCE_ACTION_TYPE, "EXPRESSION"); - runner.setProperty(expressionHandler, AlertHandler.ENFORCE_ACTION_TYPE_LEVEL, "WARN"); - runner.enableControllerService(expressionHandler); - final Map attributes = new HashMap<>(); - final Map metrics = new HashMap<>(); - attributes.put("type","FAKE"); - metrics.put("jvmHeap","1000000"); - metrics.put("cpu","90"); - - final Action action = new Action(); - action.setType("FAKE"); - action.setAttributes(attributes); - - assertDoesNotThrow(() -> expressionHandler.execute(action, metrics)); - - final String warnMessage = mockComponentLog.getWarnMessage(); - assertTrue(StringUtils.isNotEmpty(warnMessage)); - assertEquals("This Action Handler does not support actions with the provided type: FAKE",warnMessage); - } - - @Test - public void testInvalidActionTypeIgnore() { - runner.disableControllerService(expressionHandler); - runner.setProperty(expressionHandler, AlertHandler.ENFORCE_ACTION_TYPE, "EXPRESSION"); - runner.setProperty(expressionHandler, AlertHandler.ENFORCE_ACTION_TYPE_LEVEL, "IGNORE"); - runner.enableControllerService(expressionHandler); - final Map attributes = new HashMap<>(); - final Map metrics = new HashMap<>(); - attributes.put("type","FAKE"); - metrics.put("jvmHeap","1000000"); - metrics.put("cpu","90"); - - final Action action = new Action(); - action.setType("FAKE"); - action.setAttributes(attributes); - - assertDoesNotThrow(() -> expressionHandler.execute(action, metrics)); - - final String debugMessage = mockComponentLog.getDebugMessage(); - assertTrue(StringUtils.isNotEmpty(debugMessage)); - assertEquals("This Action Handler does not support actions with the provided type: FAKE",debugMessage); - - } - @Test - public void testValidActionType() { - runner.disableControllerService(expressionHandler); - runner.setProperty(expressionHandler, AlertHandler.ENFORCE_ACTION_TYPE, "EXPRESSION"); - runner.enableControllerService(expressionHandler); - final Map attributes = new HashMap<>(); - final Map metrics = new HashMap<>(); - attributes.put("type","FAKE"); - metrics.put("jvmHeap","1000000"); - metrics.put("cpu","90"); - - final Action action = new Action(); - action.setType("EXPRESSION"); - action.setAttributes(attributes); - assertDoesNotThrow(() -> expressionHandler.execute(action, metrics)); - } - - - private static class MockExpressionHandler extends ExpressionHandler{ - private ComponentLog testLogger; - - public MockExpressionHandler(ComponentLog testLogger) { - this.testLogger = testLogger; - } - - @Override - protected ComponentLog getLogger() { - return testLogger; - } - } -} diff --git a/nifi-nar-bundles/nifi-rules-action-handler-bundle/nifi-rules-action-handler-service/src/test/java/org/apache/nifi/rules/handlers/TestLogHandler.java b/nifi-nar-bundles/nifi-rules-action-handler-bundle/nifi-rules-action-handler-service/src/test/java/org/apache/nifi/rules/handlers/TestLogHandler.java deleted file mode 100644 index 671e485042..0000000000 --- a/nifi-nar-bundles/nifi-rules-action-handler-bundle/nifi-rules-action-handler-service/src/test/java/org/apache/nifi/rules/handlers/TestLogHandler.java +++ /dev/null @@ -1,269 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.nifi.rules.handlers; - -import org.apache.commons.lang3.StringUtils; -import org.apache.nifi.logging.ComponentLog; -import org.apache.nifi.reporting.InitializationException; -import org.apache.nifi.rules.Action; -import org.apache.nifi.util.TestRunner; -import org.apache.nifi.util.TestRunners; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; - -import java.util.HashMap; -import java.util.Map; - -import static org.hamcrest.MatcherAssert.assertThat; -import static org.hamcrest.core.IsInstanceOf.instanceOf; -import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertThrows; -import static org.junit.jupiter.api.Assertions.assertTrue; - -public class TestLogHandler { - - TestRunner runner; - MockComponentLog mockComponentLog; - LogHandler logHandler; - - @BeforeEach - public void setup() throws InitializationException { - runner = TestRunners.newTestRunner(TestProcessor.class); - mockComponentLog = new MockComponentLog(); - LogHandler handler = new MockLogHandler(mockComponentLog); - runner.addControllerService("MockLogHandler", handler); - runner.enableControllerService(handler); - logHandler = (LogHandler) runner.getProcessContext() - .getControllerServiceLookup() - .getControllerService("MockLogHandler"); - - } - - @Test - public void testValidService() { - runner.assertValid(logHandler); - assertThat(logHandler, instanceOf(LogHandler.class)); - } - - @Test - public void testWarningLogged() { - final Map attributes = new HashMap<>(); - final Map metrics = new HashMap<>(); - - final String expectedMessage = "--------------------------------------------------\n" + - "Log Message: This is a warning\n" + - "Log Facts:\n" + - "Field: cpu, Value: 90\n" + - "Field: jvmHeap, Value: 1000000"; - - - attributes.put("logLevel", "warn"); - attributes.put("message", "This is a warning"); - metrics.put("jvmHeap", "1000000"); - metrics.put("cpu", "90"); - final Action action = new Action(); - action.setType("LOG"); - action.setAttributes(attributes); - logHandler.execute(action, metrics); - String logMessage = mockComponentLog.getWarnMessage(); - assertTrue(StringUtils.isNotEmpty(logMessage)); - assertEquals(expectedMessage, logMessage); - } - - @Test - public void testNoLogAttributesProvided() { - - final Map attributes = new HashMap<>(); - final Map metrics = new HashMap<>(); - final String expectedMessage = "--------------------------------------------------\n" + - "Log Message: Rules Action Triggered Log.\n" + - "Log Facts:\n" + - "Field: cpu, Value: 90\n" + - "Field: jvmHeap, Value: 1000000"; - - metrics.put("jvmHeap", "1000000"); - metrics.put("cpu", "90"); - - final Action action = new Action(); - action.setType("LOG"); - action.setAttributes(attributes); - logHandler.execute(action, metrics); - String logMessage = mockComponentLog.getInfoMessage(); - assertTrue(StringUtils.isNotEmpty(logMessage)); - assertEquals(expectedMessage, logMessage); - - } - - @Test - public void testInvalidLogLevelProvided() { - final Map attributes = new HashMap<>(); - final Map metrics = new HashMap<>(); - - attributes.put("logLevel", "FAKE"); - - final String expectedMessage = "--------------------------------------------------\n" + - "Log Message: Rules Action Triggered Log.\n" + - "Log Facts:\n" + - "Field: cpu, Value: 90\n" + - "Field: jvmHeap, Value: 1000000"; - - metrics.put("jvmHeap", "1000000"); - metrics.put("cpu", "90"); - - final Action action = new Action(); - action.setType("LOG"); - action.setAttributes(attributes); - logHandler.execute(action, metrics); - String logMessage = mockComponentLog.getInfoMessage(); - assertTrue(StringUtils.isNotEmpty(logMessage)); - assertEquals(expectedMessage, logMessage); - - } - - @Test - public void testInvalidActionTypeException() { - runner.disableControllerService(logHandler); - runner.setProperty(logHandler, AlertHandler.ENFORCE_ACTION_TYPE, "LOG"); - runner.setProperty(logHandler, AlertHandler.ENFORCE_ACTION_TYPE_LEVEL, "EXCEPTION"); - runner.enableControllerService(logHandler); - - final Map attributes = new HashMap<>(); - final Map metrics = new HashMap<>(); - - attributes.put("logLevel", "FAKE"); - - final String expectedMessage = "--------------------------------------------------\n" + - "Log Message: Rules Action Triggered Log.\n" + - "Log Facts:\n" + - "Field: cpu, Value: 90\n" + - "Field: jvmHeap, Value: 1000000"; - - metrics.put("jvmHeap", "1000000"); - metrics.put("cpu", "90"); - - final Action action = new Action(); - action.setType("FAKE"); - action.setAttributes(attributes); - assertThrows(UnsupportedOperationException.class, () -> logHandler.execute(action, metrics)); - } - - @Test - public void testInvalidActionTypeWarning() { - runner.disableControllerService(logHandler); - runner.setProperty(logHandler, AlertHandler.ENFORCE_ACTION_TYPE, "LOG"); - runner.setProperty(logHandler, AlertHandler.ENFORCE_ACTION_TYPE_LEVEL, "WARN"); - runner.enableControllerService(logHandler); - - final Map attributes = new HashMap<>(); - final Map metrics = new HashMap<>(); - - attributes.put("logLevel", "FAKE"); - - final String expectedMessage = "--------------------------------------------------\n" + - "Log Message: Rules Action Triggered Log.\n" + - "Log Facts:\n" + - "Field: cpu, Value: 90\n" + - "Field: jvmHeap, Value: 1000000"; - - metrics.put("jvmHeap", "1000000"); - metrics.put("cpu", "90"); - - final Action action = new Action(); - action.setType("FAKE"); - action.setAttributes(attributes); - assertDoesNotThrow(() -> logHandler.execute(action, metrics)); - - final String warnMessage = mockComponentLog.getWarnMessage(); - assertTrue(StringUtils.isNotEmpty(warnMessage)); - assertEquals("This Action Handler does not support actions with the provided type: FAKE",warnMessage); - } - - @Test - public void testInvalidActionTypeDebug() { - runner.disableControllerService(logHandler); - runner.setProperty(logHandler, AlertHandler.ENFORCE_ACTION_TYPE, "LOG"); - runner.setProperty(logHandler, AlertHandler.ENFORCE_ACTION_TYPE_LEVEL, "IGNORE"); - runner.enableControllerService(logHandler); - - final Map attributes = new HashMap<>(); - final Map metrics = new HashMap<>(); - - attributes.put("logLevel", "FAKE"); - - final String expectedMessage = "--------------------------------------------------\n" + - "Log Message: Rules Action Triggered Log.\n" + - "Log Facts:\n" + - "Field: cpu, Value: 90\n" + - "Field: jvmHeap, Value: 1000000"; - - metrics.put("jvmHeap", "1000000"); - metrics.put("cpu", "90"); - - final Action action = new Action(); - action.setType("FAKE"); - action.setAttributes(attributes); - - assertDoesNotThrow(() -> logHandler.execute(action, metrics)); - - final String debugMessage = mockComponentLog.getDebugMessage(); - assertTrue(StringUtils.isNotEmpty(debugMessage)); - assertEquals("This Action Handler does not support actions with the provided type: FAKE",debugMessage); - } - - @Test - public void testValidActionType() { - runner.disableControllerService(logHandler); - runner.setProperty(logHandler, AlertHandler.ENFORCE_ACTION_TYPE, "LOG"); - runner.enableControllerService(logHandler); - - final Map attributes = new HashMap<>(); - final Map metrics = new HashMap<>(); - - attributes.put("logLevel", "FAKE"); - - final String expectedMessage = "--------------------------------------------------\n" + - "Log Message: Rules Action Triggered Log.\n" + - "Log Facts:\n" + - "Field: cpu, Value: 90\n" + - "Field: jvmHeap, Value: 1000000"; - - metrics.put("jvmHeap", "1000000"); - metrics.put("cpu", "90"); - - final Action action = new Action(); - action.setType("LOG"); - action.setAttributes(attributes); - - assertDoesNotThrow(() -> logHandler.execute(action, metrics)); - } - - private static class MockLogHandler extends LogHandler { - private ComponentLog testLogger; - - public MockLogHandler(ComponentLog testLogger) { - this.testLogger = testLogger; - } - - @Override - protected ComponentLog getLogger() { - return testLogger; - } - } - - -} diff --git a/nifi-nar-bundles/nifi-rules-action-handler-bundle/nifi-rules-action-handler-service/src/test/java/org/apache/nifi/rules/handlers/TestProcessor.java b/nifi-nar-bundles/nifi-rules-action-handler-bundle/nifi-rules-action-handler-service/src/test/java/org/apache/nifi/rules/handlers/TestProcessor.java deleted file mode 100644 index 43ad6fff00..0000000000 --- a/nifi-nar-bundles/nifi-rules-action-handler-bundle/nifi-rules-action-handler-service/src/test/java/org/apache/nifi/rules/handlers/TestProcessor.java +++ /dev/null @@ -1,46 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.nifi.rules.handlers; - -import org.apache.nifi.components.PropertyDescriptor; -import org.apache.nifi.processor.AbstractProcessor; -import org.apache.nifi.processor.ProcessContext; -import org.apache.nifi.processor.ProcessSession; -import org.apache.nifi.processor.exception.ProcessException; - -import java.util.ArrayList; -import java.util.List; - -public class TestProcessor extends AbstractProcessor { - - @Override - public void onTrigger(ProcessContext context, ProcessSession session) throws ProcessException { - } - - @Override - protected List getSupportedPropertyDescriptors() { - List properties = new ArrayList<>(); - properties.add(new PropertyDescriptor.Builder() - .name("log-action-handler-test") - .description("Logging Action Handler") - .identifiesControllerService(LogHandler.class) - .required(true) - .build()); - return properties; - } - -} \ No newline at end of file diff --git a/nifi-nar-bundles/nifi-rules-action-handler-bundle/nifi-rules-action-handler-service/src/test/java/org/apache/nifi/rules/handlers/TestRecordSinkHandler.java b/nifi-nar-bundles/nifi-rules-action-handler-bundle/nifi-rules-action-handler-service/src/test/java/org/apache/nifi/rules/handlers/TestRecordSinkHandler.java deleted file mode 100644 index fb5f342200..0000000000 --- a/nifi-nar-bundles/nifi-rules-action-handler-bundle/nifi-rules-action-handler-service/src/test/java/org/apache/nifi/rules/handlers/TestRecordSinkHandler.java +++ /dev/null @@ -1,152 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.nifi.rules.handlers; - -import org.apache.commons.lang3.StringUtils; -import org.apache.nifi.components.AbstractConfigurableComponent; -import org.apache.nifi.controller.ControllerServiceInitializationContext; -import org.apache.nifi.logging.ComponentLog; -import org.apache.nifi.record.sink.RecordSinkService; -import org.apache.nifi.reporting.InitializationException; -import org.apache.nifi.rules.Action; -import org.apache.nifi.serialization.WriteResult; -import org.apache.nifi.serialization.record.Record; -import org.apache.nifi.serialization.record.RecordSchema; -import org.apache.nifi.serialization.record.RecordSet; -import org.apache.nifi.util.TestRunner; -import org.apache.nifi.util.TestRunners; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; - -import java.io.IOException; -import java.math.BigDecimal; -import java.util.ArrayList; -import java.util.Collections; -import java.util.HashMap; -import java.util.List; -import java.util.Map; - -import static org.hamcrest.MatcherAssert.assertThat; -import static org.hamcrest.core.IsInstanceOf.instanceOf; -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertFalse; -import static org.junit.jupiter.api.Assertions.assertTrue; - -public class TestRecordSinkHandler { - private TestRunner runner; - private MockComponentLog mockComponentLog; - private RecordSinkHandler recordSinkHandler; - private MockRecordSinkService recordSinkService; - - @BeforeEach - public void setup() throws InitializationException { - runner = TestRunners.newTestRunner(TestProcessor.class); - mockComponentLog = new MockComponentLog(); - RecordSinkHandler handler = new MockRecordSinkHandler(mockComponentLog); - recordSinkService = new MockRecordSinkService(); - runner.addControllerService("MockRecordSinkService", recordSinkService); - runner.enableControllerService(recordSinkService); - runner.addControllerService("MockRecordSinkHandler", handler); - runner.setProperty(handler, MockRecordSinkHandler.RECORD_SINK_SERVICE,"MockRecordSinkService"); - runner.enableControllerService(handler); - recordSinkHandler = (RecordSinkHandler) runner.getProcessContext() - .getControllerServiceLookup() - .getControllerService("MockRecordSinkHandler"); - } - - @Test - public void testValidService() { - runner.assertValid(recordSinkHandler); - assertThat(recordSinkHandler, instanceOf(RecordSinkHandler.class)); - } - - @Test - public void testRecordSendViaSink() throws InitializationException, IOException { - final Map attributes = new HashMap<>(); - final Map metrics = new HashMap<>(); - final String expectedMessage = "Records written to sink service:"; - final BigDecimal bigDecimalValue = new BigDecimal(String.join("", Collections.nCopies(400, "1")) + ".2"); - - attributes.put("sendZeroResults","false"); - metrics.put("jvmHeap","1000000"); - metrics.put("cpu","90"); - metrics.put("custom", bigDecimalValue); - - final Action action = new Action(); - action.setType("SEND"); - action.setAttributes(attributes); - recordSinkHandler.execute(action, metrics); - String logMessage = mockComponentLog.getDebugMessage(); - List> rows = recordSinkService.getRows(); - assertTrue(StringUtils.isNotEmpty(logMessage)); - assertTrue(logMessage.startsWith(expectedMessage)); - assertFalse(rows.isEmpty()); - Map record = rows.get(0); - assertEquals("90", (record.get("cpu"))); - assertEquals("1000000", (record.get("jvmHeap"))); - assertEquals(bigDecimalValue, (record.get("custom"))); - } - - private static class MockRecordSinkHandler extends RecordSinkHandler { - private ComponentLog testLogger; - - public MockRecordSinkHandler(ComponentLog testLogger) { - this.testLogger = testLogger; - } - - @Override - protected ComponentLog getLogger() { - return testLogger; - } - } - - private static class MockRecordSinkService extends AbstractConfigurableComponent implements RecordSinkService { - - private List> rows = new ArrayList<>(); - - @Override - public WriteResult sendData(RecordSet recordSet, Map attributes, boolean sendZeroResults) throws IOException { - int numRecordsWritten = 0; - RecordSchema recordSchema = recordSet.getSchema(); - Record record; - while ((record = recordSet.next()) != null) { - Map row = new HashMap<>(); - final Record finalRecord = record; - recordSchema.getFieldNames().forEach((fieldName) -> row.put(fieldName, finalRecord.getValue(fieldName))); - rows.add(row); - numRecordsWritten++; - } - return WriteResult.of(numRecordsWritten, Collections.emptyMap()); - } - - @Override - public String getIdentifier() { - return "MockRecordSinkService"; - } - - @Override - public void initialize(ControllerServiceInitializationContext context) throws InitializationException { - } - - public List> getRows() { - return rows; - } - } - - -} diff --git a/nifi-nar-bundles/nifi-rules-action-handler-bundle/pom.xml b/nifi-nar-bundles/nifi-rules-action-handler-bundle/pom.xml deleted file mode 100644 index 4bdf1f36f9..0000000000 --- a/nifi-nar-bundles/nifi-rules-action-handler-bundle/pom.xml +++ /dev/null @@ -1,40 +0,0 @@ - - - - - nifi-nar-bundles - org.apache.nifi - 2.0.0-SNAPSHOT - - 4.0.0 - - nifi-rules-action-handler-bundle - pom - - nifi-rules-action-handler-nar - nifi-rules-action-handler-service - - - - - - org.apache.nifi - nifi-rules-action-handler-service - 2.0.0-SNAPSHOT - - - - diff --git a/nifi-nar-bundles/nifi-scripting-bundle/nifi-scripting-processors/pom.xml b/nifi-nar-bundles/nifi-scripting-bundle/nifi-scripting-processors/pom.xml index 3b098525ef..91a42c2450 100644 --- a/nifi-nar-bundles/nifi-scripting-bundle/nifi-scripting-processors/pom.xml +++ b/nifi-nar-bundles/nifi-scripting-bundle/nifi-scripting-processors/pom.xml @@ -69,11 +69,6 @@ nifi-record-sink-api provided - - org.apache.nifi - nifi-rules-engine-service-api - 2.0.0-SNAPSHOT - org.apache.ivy ivy diff --git a/nifi-nar-bundles/nifi-scripting-bundle/nifi-scripting-processors/src/main/java/org/apache/nifi/rules/engine/script/ScriptedRulesEngine.java b/nifi-nar-bundles/nifi-scripting-bundle/nifi-scripting-processors/src/main/java/org/apache/nifi/rules/engine/script/ScriptedRulesEngine.java deleted file mode 100644 index d200778793..0000000000 --- a/nifi-nar-bundles/nifi-scripting-bundle/nifi-scripting-processors/src/main/java/org/apache/nifi/rules/engine/script/ScriptedRulesEngine.java +++ /dev/null @@ -1,224 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.nifi.rules.engine.script; - -import org.apache.nifi.annotation.behavior.DynamicProperty; -import org.apache.nifi.annotation.behavior.Restricted; -import org.apache.nifi.annotation.behavior.Restriction; -import org.apache.nifi.annotation.documentation.CapabilityDescription; -import org.apache.nifi.annotation.documentation.Tags; -import org.apache.nifi.annotation.lifecycle.OnEnabled; -import org.apache.nifi.components.PropertyDescriptor; -import org.apache.nifi.components.RequiredPermission; -import org.apache.nifi.components.ValidationResult; -import org.apache.nifi.controller.ConfigurationContext; -import org.apache.nifi.expression.ExpressionLanguageScope; -import org.apache.nifi.logging.ComponentLog; -import org.apache.nifi.processor.exception.ProcessException; -import org.apache.nifi.rules.Action; -import org.apache.nifi.rules.engine.RulesEngineService; -import org.apache.nifi.script.AbstractScriptedControllerService; -import org.apache.nifi.script.ScriptingComponentHelper; - -import javax.script.Invocable; -import javax.script.ScriptContext; -import javax.script.ScriptEngine; -import javax.script.ScriptException; -import java.util.Collection; -import java.util.Collections; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.concurrent.atomic.AtomicReference; - -@Tags({"rules", "rules engine", "script", "invoke", "groovy", "python", "jython"}) -@CapabilityDescription("Allows the user to provide a scripted RulesEngineService for custom firing of rules depending on the supplied facts. The script must set a variable 'rulesEngine' to an " - + "implementation of RulesEngineService.") -@DynamicProperty(name = "Script Engine Binding property", value = "Binding property value passed to Script Runner", - expressionLanguageScope = ExpressionLanguageScope.VARIABLE_REGISTRY, - description = "Updates a script engine property specified by the Dynamic Property's key with the value specified by the Dynamic Property's value") -@Restricted( - restrictions = { - @Restriction( - requiredPermission = RequiredPermission.EXECUTE_CODE, - explanation = "Provides operator the ability to execute arbitrary code assuming all permissions that NiFi has.") - } -) -public class ScriptedRulesEngine extends AbstractScriptedControllerService implements RulesEngineService { - - protected final AtomicReference rulesEngine = new AtomicReference<>(); - - /** - * Returns a list of property descriptors supported by this processor. The list always includes properties such as - * script engine name, script file name, script body name, script arguments, and an external module path. If the - * scripted processor also defines supported properties, those are added to the list as well. - * - * @return a List of PropertyDescriptor objects supported by this processor - */ - @Override - protected List getSupportedPropertyDescriptors() { - synchronized (scriptingComponentHelper.isInitialized) { - if (!scriptingComponentHelper.isInitialized.get()) { - scriptingComponentHelper.createResources(); - } - } - - return Collections.unmodifiableList(scriptingComponentHelper.getDescriptors()); - } - - public void setup() { - if (scriptNeedsReload.get() || rulesEngine.get() == null) { - if (ScriptingComponentHelper.isFile(scriptingComponentHelper.getScriptPath())) { - scriptNeedsReload.set(!reloadScriptFile(scriptingComponentHelper.getScriptPath())); - } else { - scriptNeedsReload.set(!reloadScriptBody(scriptingComponentHelper.getScriptBody())); - } - } - } - - /** - * Reloads the script RulesEngineService. This must be called within the lock. - * - * @param scriptBody An input stream associated with the script content - * @return Whether the script was successfully reloaded - */ - protected boolean reloadScript(final String scriptBody) { - // note we are starting here with a fresh listing of validation - // results since we are (re)loading a new/updated script. any - // existing validation results are not relevant - final Collection results = new HashSet<>(); - - try { - // Create a single script engine, the Processor object is reused by each task - if (scriptRunner == null) { - scriptingComponentHelper.setupScriptRunners(1, scriptBody, getLogger()); - scriptRunner = scriptingComponentHelper.scriptRunnerQ.poll(); - } - - if (scriptRunner == null) { - throw new ProcessException("No script runner available!"); - } - // get the engine and ensure its invocable - ScriptEngine scriptEngine = scriptRunner.getScriptEngine(); - if (scriptEngine instanceof Invocable) { - final Invocable invocable = (Invocable) scriptEngine; - - // evaluate the script - scriptRunner.run(scriptEngine.getBindings(ScriptContext.ENGINE_SCOPE)); - - - // get configured processor from the script (if it exists) - final Object obj = scriptEngine.get("rulesEngine"); - if (obj != null) { - final ComponentLog logger = getLogger(); - - try { - // set the logger if the processor wants it - invocable.invokeMethod(obj, "setLogger", logger); - } catch (final NoSuchMethodException nsme) { - if (logger.isDebugEnabled()) { - logger.debug("Configured script RulesEngineService does not contain a setLogger method."); - } - } - - if (configurationContext != null) { - try { - // set the logger if the processor wants it - invocable.invokeMethod(obj, "setConfigurationContext", configurationContext); - } catch (final NoSuchMethodException nsme) { - if (logger.isDebugEnabled()) { - logger.debug("Configured script RulesEngineService does not contain a setConfigurationContext method."); - } - } - } - - // record the processor for use later - final RulesEngineService scriptedReader = invocable.getInterface(obj, RulesEngineService.class); - rulesEngine.set(scriptedReader); - - } else { - throw new ScriptException("No RecordReader was defined by the script."); - } - } - - } catch (final Exception ex) { - final ComponentLog logger = getLogger(); - final String message = "Unable to load script: " + ex.getLocalizedMessage(); - - logger.error(message, ex); - results.add(new ValidationResult.Builder() - .subject("ScriptValidation") - .valid(false) - .explanation("Unable to load script due to " + ex.getLocalizedMessage()) - .input(scriptingComponentHelper.getScriptPath()) - .build()); - } - - // store the updated validation results - validationResults.set(results); - - // return whether there was any issues loading the configured script - return results.isEmpty(); - } - - @Override - @OnEnabled - public void onEnabled(final ConfigurationContext context) { - synchronized (scriptingComponentHelper.isInitialized) { - if (!scriptingComponentHelper.isInitialized.get()) { - scriptingComponentHelper.createResources(); - } - } - super.onEnabled(context); - - // Call an non-interface method onEnabled(context), to allow a scripted RulesEngineService the chance to set up as necessary - if (scriptRunner != null) { - final ScriptEngine scriptEngine = scriptRunner.getScriptEngine(); - final Invocable invocable = (Invocable) scriptEngine; - if (configurationContext != null) { - try { - // Get the actual object from the script engine, versus the proxy stored in RulesEngineService. The object may have additional methods, - // where RulesEngineService is a proxied interface - final Object obj = scriptEngine.get("rulesEngine"); - if (obj != null) { - try { - invocable.invokeMethod(obj, "onEnabled", context); - } catch (final NoSuchMethodException nsme) { - if (getLogger().isDebugEnabled()) { - getLogger().debug("Configured script RulesEngineService does not contain an onEnabled() method."); - } - } - } else { - throw new ScriptException("No RulesEngineService was defined by the script."); - } - } catch (ScriptException se) { - throw new ProcessException("Error executing onEnabled(context) method: " + se.getMessage(), se); - } - } - } else { - throw new ProcessException("Error creating ScriptRunner"); - } - } - - @Override - public List fireRules(Map facts) { - if (rulesEngine.get() != null) { - return rulesEngine.get().fireRules(facts); - } - return null; - } -} diff --git a/nifi-nar-bundles/nifi-scripting-bundle/nifi-scripting-processors/src/main/java/org/apache/nifi/rules/handlers/script/ScriptedActionHandler.java b/nifi-nar-bundles/nifi-scripting-bundle/nifi-scripting-processors/src/main/java/org/apache/nifi/rules/handlers/script/ScriptedActionHandler.java deleted file mode 100644 index e423ba953e..0000000000 --- a/nifi-nar-bundles/nifi-scripting-bundle/nifi-scripting-processors/src/main/java/org/apache/nifi/rules/handlers/script/ScriptedActionHandler.java +++ /dev/null @@ -1,257 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.nifi.rules.handlers.script; - -import org.apache.nifi.annotation.behavior.DynamicProperty; -import org.apache.nifi.annotation.behavior.Restricted; -import org.apache.nifi.annotation.behavior.Restriction; -import org.apache.nifi.annotation.documentation.CapabilityDescription; -import org.apache.nifi.annotation.documentation.Tags; -import org.apache.nifi.annotation.lifecycle.OnEnabled; -import org.apache.nifi.components.PropertyDescriptor; -import org.apache.nifi.components.RequiredPermission; -import org.apache.nifi.components.ValidationResult; -import org.apache.nifi.context.PropertyContext; -import org.apache.nifi.controller.ConfigurationContext; -import org.apache.nifi.expression.ExpressionLanguageScope; -import org.apache.nifi.logging.ComponentLog; -import org.apache.nifi.processor.exception.ProcessException; -import org.apache.nifi.rules.Action; -import org.apache.nifi.rules.ActionHandler; -import org.apache.nifi.rules.PropertyContextActionHandler; -import org.apache.nifi.script.AbstractScriptedControllerService; -import org.apache.nifi.script.ScriptingComponentHelper; - -import javax.script.Invocable; -import javax.script.ScriptContext; -import javax.script.ScriptEngine; -import javax.script.ScriptException; -import java.util.Collection; -import java.util.Collections; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.concurrent.atomic.AtomicReference; - -@Tags({"rules", "rules engine", "action", "action handler", "script", "invoke", "groovy", "python", "jython"}) -@CapabilityDescription("Allows the user to provide a scripted ActionHandler for custom firing of rules depending on the supplied facts. The script must set a variable 'actionHandler' to an " - + "implementation of ActionHandler.") -@DynamicProperty(name = "Script Engine Binding property", value = "Binding property value passed to Script Runner", - expressionLanguageScope = ExpressionLanguageScope.VARIABLE_REGISTRY, - description = "Updates a script engine property specified by the Dynamic Property's key with the value specified by the Dynamic Property's value") -@Restricted( - restrictions = { - @Restriction( - requiredPermission = RequiredPermission.EXECUTE_CODE, - explanation = "Provides operator the ability to execute arbitrary code assuming all permissions that NiFi has.") - } -) -public class ScriptedActionHandler extends AbstractScriptedControllerService implements PropertyContextActionHandler { - - protected final AtomicReference actionHandler = new AtomicReference<>(); - - /** - * Returns a list of property descriptors supported by this processor. The list always includes properties such as - * script engine name, script file name, script body name, script arguments, and an external module path. If the - * scripted processor also defines supported properties, those are added to the list as well. - * - * @return a List of PropertyDescriptor objects supported by this processor - */ - @Override - protected List getSupportedPropertyDescriptors() { - synchronized (scriptingComponentHelper.isInitialized) { - if (!scriptingComponentHelper.isInitialized.get()) { - scriptingComponentHelper.createResources(); - } - } - - return Collections.unmodifiableList(scriptingComponentHelper.getDescriptors()); - } - - public void setup() { - if (scriptNeedsReload.get() || actionHandler.get() == null) { - if (ScriptingComponentHelper.isFile(scriptingComponentHelper.getScriptPath())) { - scriptNeedsReload.set(!reloadScriptFile(scriptingComponentHelper.getScriptPath())); - } else { - scriptNeedsReload.set(!reloadScriptBody(scriptingComponentHelper.getScriptBody())); - } - } - } - - /** - * Reloads the script ActionHandler. This must be called within the lock. - * - * @param scriptBody An input stream associated with the script content - * @return Whether the script was successfully reloaded - */ - protected boolean reloadScript(final String scriptBody) { - // note we are starting here with a fresh listing of validation - // results since we are (re)loading a new/updated script. any - // existing validation results are not relevant - final Collection results = new HashSet<>(); - - try { - // Create a single script engine, the Processor object is reused by each task - if (scriptRunner == null) { - scriptingComponentHelper.setupScriptRunners(1, scriptBody, getLogger()); - scriptRunner = scriptingComponentHelper.scriptRunnerQ.poll(); - } - - if (scriptRunner == null) { - throw new ProcessException("No script runner available!"); - } - // get the engine and ensure its invocable - ScriptEngine scriptEngine = scriptRunner.getScriptEngine(); - if (scriptEngine instanceof Invocable) { - final Invocable invocable = (Invocable) scriptEngine; - - // evaluate the script - scriptRunner.run(scriptEngine.getBindings(ScriptContext.ENGINE_SCOPE)); - - - // get configured processor from the script (if it exists) - final Object obj = scriptRunner.getScriptEngine().get("actionHandler"); - if (obj != null) { - final ComponentLog logger = getLogger(); - - try { - // set the logger if the processor wants it - invocable.invokeMethod(obj, "setLogger", logger); - } catch (final NoSuchMethodException nsme) { - if (logger.isDebugEnabled()) { - logger.debug("Configured script ActionHandler does not contain a setLogger method."); - } - } - - if (configurationContext != null) { - try { - // set the logger if the processor wants it - invocable.invokeMethod(obj, "setConfigurationContext", configurationContext); - } catch (final NoSuchMethodException nsme) { - if (logger.isDebugEnabled()) { - logger.debug("Configured script ActionHandler does not contain a setConfigurationContext method."); - } - } - } - - // record the processor for use later - final ActionHandler scriptedReader = invocable.getInterface(obj, ActionHandler.class); - actionHandler.set(scriptedReader); - - } else { - throw new ScriptException("No RecordReader was defined by the script."); - } - } - - } catch (final Exception ex) { - final ComponentLog logger = getLogger(); - final String message = "Unable to load script: " + ex.getLocalizedMessage(); - - logger.error(message, ex); - results.add(new ValidationResult.Builder() - .subject("ScriptValidation") - .valid(false) - .explanation("Unable to load script due to " + ex.getLocalizedMessage()) - .input(scriptingComponentHelper.getScriptPath()) - .build()); - } - - // store the updated validation results - validationResults.set(results); - - // return whether there was any issues loading the configured script - return results.isEmpty(); - } - - @Override - @OnEnabled - public void onEnabled(final ConfigurationContext context) { - synchronized (scriptingComponentHelper.isInitialized) { - if (!scriptingComponentHelper.isInitialized.get()) { - scriptingComponentHelper.createResources(); - } - } - super.onEnabled(context); - - // Call an non-interface method onEnabled(context), to allow a scripted ActionHandler the chance to set up as necessary - if (scriptRunner != null) { - final ScriptEngine scriptEngine = scriptRunner.getScriptEngine(); - final Invocable invocable = (Invocable) scriptEngine; - if (configurationContext != null) { - try { - // Get the actual object from the script engine, versus the proxy stored in ActionHandler. The object may have additional methods, - // where ActionHandler is a proxied interface - final Object obj = scriptRunner.getScriptEngine().get("actionHandler"); - if (obj != null) { - try { - invocable.invokeMethod(obj, "onEnabled", context); - } catch (final NoSuchMethodException nsme) { - if (getLogger().isDebugEnabled()) { - getLogger().debug("Configured script ActionHandler does not contain an onEnabled() method."); - } - } - } else { - throw new ScriptException("No ActionHandler was defined by the script."); - } - } catch (ScriptException se) { - throw new ProcessException("Error executing onEnabled(context) method", se); - } - } - } else { - throw new ProcessException("Error creating ScriptRunner"); - } - } - - - public void execute(PropertyContext context, Action action, Map facts) { - // Attempt to call a non-ActionHandler interface method (i.e. execute(context, action, facts) from PropertyContextActionHandler) - if (scriptRunner != null) { - final ScriptEngine scriptEngine = scriptRunner.getScriptEngine(); - final Invocable invocable = (Invocable) scriptEngine; - - try { - // Get the actual object from the script engine, versus the proxy stored in ActionHandler. The object may have additional methods, - // where ActionHandler is a proxied interface - final Object obj = scriptRunner.getScriptEngine().get("actionHandler"); - if (obj != null) { - try { - invocable.invokeMethod(obj, "execute", context, action, facts); - } catch (final NoSuchMethodException nsme) { - if (getLogger().isDebugEnabled()) { - getLogger().debug("Configured script ActionHandler is not a PropertyContextActionHandler and has no execute(context, action, facts) method, falling back to" - + "execute(action, facts)."); - } - execute(action, facts); - } - } else { - throw new ScriptException("No ActionHandler was defined by the script."); - } - } catch (ScriptException se) { - throw new ProcessException("Error executing onEnabled(context) method: " + se.getMessage(), se); - } - } else { - throw new ProcessException("Error creating ScriptRunner"); - } - } - - @Override - public void execute(Action action, Map facts) { - if (actionHandler.get() != null) { - actionHandler.get().execute(action, facts); - } - } -} diff --git a/nifi-nar-bundles/nifi-scripting-bundle/nifi-scripting-processors/src/main/resources/META-INF/services/org.apache.nifi.controller.ControllerService b/nifi-nar-bundles/nifi-scripting-bundle/nifi-scripting-processors/src/main/resources/META-INF/services/org.apache.nifi.controller.ControllerService index 4d5d6dad79..ef26145562 100644 --- a/nifi-nar-bundles/nifi-scripting-bundle/nifi-scripting-processors/src/main/resources/META-INF/services/org.apache.nifi.controller.ControllerService +++ b/nifi-nar-bundles/nifi-scripting-bundle/nifi-scripting-processors/src/main/resources/META-INF/services/org.apache.nifi.controller.ControllerService @@ -12,11 +12,8 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. - -org.apache.nifi.rules.handlers.script.ScriptedActionHandler org.apache.nifi.record.script.ScriptedReader org.apache.nifi.record.script.ScriptedRecordSetWriter org.apache.nifi.record.sink.script.ScriptedRecordSink -org.apache.nifi.rules.engine.script.ScriptedRulesEngine org.apache.nifi.lookup.script.ScriptedLookupService org.apache.nifi.lookup.script.SimpleScriptedLookupService \ No newline at end of file diff --git a/nifi-nar-bundles/nifi-scripting-bundle/nifi-scripting-processors/src/test/java/org/apache/nifi/rules/engine/script/ScriptedRulesEngineTest.java b/nifi-nar-bundles/nifi-scripting-bundle/nifi-scripting-processors/src/test/java/org/apache/nifi/rules/engine/script/ScriptedRulesEngineTest.java deleted file mode 100644 index 5eea79d6a5..0000000000 --- a/nifi-nar-bundles/nifi-scripting-bundle/nifi-scripting-processors/src/test/java/org/apache/nifi/rules/engine/script/ScriptedRulesEngineTest.java +++ /dev/null @@ -1,92 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.nifi.rules.engine.script; - -import org.apache.nifi.processor.AbstractProcessor; -import org.apache.nifi.processor.ProcessContext; -import org.apache.nifi.processor.ProcessSession; -import org.apache.nifi.processor.exception.ProcessException; -import org.apache.nifi.processors.script.AccessibleScriptingComponentHelper; -import org.apache.nifi.reporting.InitializationException; -import org.apache.nifi.rules.Action; -import org.apache.nifi.script.ScriptingComponentHelper; -import org.apache.nifi.script.ScriptingComponentUtils; -import org.apache.nifi.serialization.record.MockRecordWriter; -import org.apache.nifi.util.TestRunner; -import org.apache.nifi.util.TestRunners; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; - -import java.io.IOException; -import java.util.HashMap; -import java.util.List; -import java.util.Map; - -import static org.junit.jupiter.api.Assertions.assertEquals; - - -public class ScriptedRulesEngineTest { - - private Map facts = new HashMap<>(); - - @BeforeEach - public void setup() { - facts.put("predictedQueuedCount", 60); - facts.put("predictedTimeToBytesBackpressureMillis", 299999); - } - - @Test - public void testRules() throws IOException, InitializationException { - ScriptedRulesEngine task = initTask(); - List actions = task.fireRules(facts); - assertEquals(2, actions.size()); - assertEquals("LOG", actions.get(0).getType()); - assertEquals("DEBUG", actions.get(0).getAttributes().get("level")); - assertEquals("ALERT", actions.get(1).getType()); - assertEquals("Time to backpressure < 5 mins", actions.get(1).getAttributes().get("message")); - } - - private MockScriptedRulesEngine initTask() throws InitializationException { - final TestRunner runner = TestRunners.newTestRunner(new AbstractProcessor() { - @Override - public void onTrigger(final ProcessContext context, final ProcessSession session) throws ProcessException { - } - }); - - final MockRecordWriter writer = new MockRecordWriter(null, false); // No header, don"t quote values - runner.addControllerService("writer", writer); - runner.enableControllerService(writer); - - final MockScriptedRulesEngine rulesEngine = new MockScriptedRulesEngine(); - runner.addControllerService("rulesEngine", rulesEngine); - runner.setProperty(rulesEngine, "Script Engine", "Groovy"); - runner.setProperty(rulesEngine, ScriptingComponentUtils.SCRIPT_FILE, "src/test/resources/groovy/test_rules_engine.groovy"); - runner.setProperty(rulesEngine, ScriptingComponentUtils.SCRIPT_BODY, (String) null); - runner.setProperty(rulesEngine, ScriptingComponentUtils.MODULES, (String) null); - runner.enableControllerService(rulesEngine); - - return rulesEngine; - } - - public static class MockScriptedRulesEngine extends ScriptedRulesEngine implements AccessibleScriptingComponentHelper { - - @Override - public ScriptingComponentHelper getScriptingComponentHelper() { - return this.scriptingComponentHelper; - } - } -} \ No newline at end of file diff --git a/nifi-nar-bundles/nifi-scripting-bundle/nifi-scripting-processors/src/test/java/org/apache/nifi/rules/handlers/script/ScriptedActionHandlerTest.java b/nifi-nar-bundles/nifi-scripting-bundle/nifi-scripting-processors/src/test/java/org/apache/nifi/rules/handlers/script/ScriptedActionHandlerTest.java deleted file mode 100644 index efd9e1faea..0000000000 --- a/nifi-nar-bundles/nifi-scripting-bundle/nifi-scripting-processors/src/test/java/org/apache/nifi/rules/handlers/script/ScriptedActionHandlerTest.java +++ /dev/null @@ -1,222 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License") you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.nifi.rules.handlers.script; - -import org.apache.nifi.processor.AbstractProcessor; -import org.apache.nifi.processor.ProcessContext; -import org.apache.nifi.processor.ProcessSession; -import org.apache.nifi.processor.exception.ProcessException; -import org.apache.nifi.processors.script.AccessibleScriptingComponentHelper; -import org.apache.nifi.reporting.Bulletin; -import org.apache.nifi.reporting.BulletinFactory; -import org.apache.nifi.reporting.BulletinRepository; -import org.apache.nifi.reporting.InitializationException; -import org.apache.nifi.reporting.ReportingContext; -import org.apache.nifi.reporting.Severity; -import org.apache.nifi.rules.Action; -import org.apache.nifi.script.ScriptingComponentHelper; -import org.apache.nifi.script.ScriptingComponentUtils; -import org.apache.nifi.serialization.record.MockRecordWriter; -import org.apache.nifi.util.MockBulletinRepository; -import org.apache.nifi.util.TestRunner; -import org.apache.nifi.util.TestRunners; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; -import org.mockito.Mockito; - -import java.util.ArrayList; -import java.util.Arrays; -import java.util.HashMap; -import java.util.List; -import java.util.Map; - -import static org.junit.jupiter.api.Assertions.assertInstanceOf; -import static org.junit.jupiter.api.Assertions.assertTrue; -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertFalse; -import static org.mockito.ArgumentMatchers.anyString; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.when; - - -public class ScriptedActionHandlerTest { - - private TestRunner runner; - private ReportingContext reportingContext; - private MockScriptedActionHandler actionHandler; - private MockScriptedBulletinRepository mockScriptedBulletinRepository; - - private Map facts = new HashMap<>(); - private Map attrs = new HashMap<>(); - - @BeforeEach - public void setup() { - facts.put("predictedQueuedCount", 60); - facts.put("predictedTimeToBytesBackpressureMillis", 299999); - attrs.put("level", "DEBUG"); - attrs.put("message", "Time to backpressure < 5 mins"); - } - - @Test - public void testActions() throws InitializationException { - actionHandler = initTask("src/test/resources/groovy/test_action_handler.groovy"); - List actions = Arrays.asList(new Action("LOG", attrs), new Action("ALERT", attrs)); - actions.forEach((action) -> actionHandler.execute(action, facts)); - // Verify a fact was added (not the intended operation of ActionHandler, but testable) - assertEquals(42, facts.get("testFact")); - } - - @Test - public void testActionHandlerNotPropertyContextActionHandler() throws InitializationException { - actionHandler = initTask("src/test/resources/groovy/test_action_handler.groovy"); - mockScriptedBulletinRepository = new MockScriptedBulletinRepository(); - reportingContext = mock(ReportingContext.class); - when(reportingContext.getBulletinRepository()).thenReturn(mockScriptedBulletinRepository); - when(reportingContext.createBulletin(anyString(), Mockito.any(Severity.class), anyString())) - .thenAnswer(invocation -> BulletinFactory.createBulletin(invocation.getArgument(0), invocation.getArgument(1).toString(), invocation.getArgument(2))); - List actions = Arrays.asList(new Action("LOG", attrs), new Action("ALERT", attrs)); - actions.forEach(action -> actionHandler.execute(reportingContext, action, facts)); - - // Verify instead of a bulletin being added, a fact was added (not the intended operation of ActionHandler, but testable) - assertTrue(mockScriptedBulletinRepository.bulletinList.isEmpty()); - assertEquals(42, facts.get("testFact")); - } - - @Test - public void testPropertyContextActionHandler() throws InitializationException { - actionHandler = initTask("src/test/resources/groovy/test_propertycontext_action_handler.groovy"); - mockScriptedBulletinRepository = new MockScriptedBulletinRepository(); - reportingContext = mock(ReportingContext.class); - when(reportingContext.getBulletinRepository()).thenReturn(mockScriptedBulletinRepository); - when(reportingContext.createBulletin(anyString(), Mockito.any(Severity.class), anyString())) - .thenAnswer(invocation -> BulletinFactory.createBulletin(invocation.getArgument(0), invocation.getArgument(1).toString(), invocation.getArgument(2))); - List actions = Arrays.asList(new Action("LOG", attrs), new Action("ALERT", attrs)); - actions.forEach(action -> actionHandler.execute(reportingContext, action, facts)); - - // Verify instead of a bulletin being added, a fact was added (not the intended operation of ActionHandler, but testable) - List bulletinList = mockScriptedBulletinRepository.bulletinList; - assertEquals(2, bulletinList.size()); - } - - @Test - public void testValidService() throws Exception { - setupTestRunner(); - runner.assertValid(actionHandler); - assertInstanceOf(ScriptedActionHandler.class, actionHandler); - } - - @Test - public void testAlertWithBulletinLevel() throws Exception { - setupTestRunner(); - final Map attributes = new HashMap<>(); - final Map metrics = new HashMap<>(); - - final String category = "Rules Alert"; - final String message = "This should be sent as an alert!"; - final String severity = "INFO"; - attributes.put("category", category); - attributes.put("message", message); - attributes.put("severity", severity); - metrics.put("jvmHeap", "1000000"); - metrics.put("cpu", "90"); - - final String expectedOutput = "This should be sent as an alert!\n" + - "Alert Facts:\n" + - "Field: cpu, Value: 90\n" + - "Field: jvmHeap, Value: 1000000\n"; - - final Action action = new Action(); - action.setType("ALERT"); - action.setAttributes(attributes); - actionHandler.execute(reportingContext, action, metrics); - BulletinRepository bulletinRepository = reportingContext.getBulletinRepository(); - List bulletins = bulletinRepository.findBulletinsForController(); - assertFalse(bulletins.isEmpty()); - Bulletin bulletin = bulletins.get(0); - assertEquals(bulletin.getCategory(), category); - assertEquals(bulletin.getMessage(), expectedOutput); - assertEquals(bulletin.getLevel(), severity); - } - - private static class MockScriptedBulletinRepository extends MockBulletinRepository { - - List bulletinList; - - MockScriptedBulletinRepository() { - bulletinList = new ArrayList<>(); - } - - @Override - public void addBulletin(Bulletin bulletin) { - bulletinList.add(bulletin); - } - - @Override - public List findBulletinsForController() { - return bulletinList; - } - - } - - private void setupTestRunner() throws Exception { - runner = TestRunners.newTestRunner(TestProcessor.class); - MockScriptedActionHandler handler = initTask("src/test/resources/groovy/test_propertycontext_action_handler.groovy"); - mockScriptedBulletinRepository = new MockScriptedBulletinRepository(); - Map properties = new HashMap<>(); - properties.put(handler.getScriptingComponentHelper().SCRIPT_ENGINE.getName(), "Groovy"); - properties.put(ScriptingComponentUtils.SCRIPT_FILE.getName(), "src/test/resources/groovy/test_propertycontext_action_handler.groovy"); - runner.addControllerService("MockAlertHandler", handler, properties); - runner.enableControllerService(handler); - actionHandler = (MockScriptedActionHandler) runner.getProcessContext() - .getControllerServiceLookup() - .getControllerService("MockAlertHandler"); - reportingContext = mock(ReportingContext.class); - when(reportingContext.getBulletinRepository()).thenReturn(mockScriptedBulletinRepository); - when(reportingContext.createBulletin(anyString(), Mockito.any(Severity.class), anyString())) - .thenAnswer(invocation -> BulletinFactory.createBulletin(invocation.getArgument(0), invocation.getArgument(1).toString(), invocation.getArgument(2))); - } - - private MockScriptedActionHandler initTask(String scriptFile) throws InitializationException { - final TestRunner runner = TestRunners.newTestRunner(new AbstractProcessor() { - @Override - public void onTrigger(final ProcessContext context, final ProcessSession session) throws ProcessException { - } - }); - - final MockRecordWriter writer = new MockRecordWriter(null, false); // No header, don"t quote values - runner.addControllerService("writer", writer); - runner.enableControllerService(writer); - - final MockScriptedActionHandler actionHandler = new MockScriptedActionHandler(); - runner.addControllerService("actionHandler", actionHandler); - runner.setProperty(actionHandler, "Script Engine", "Groovy"); - runner.setProperty(actionHandler, ScriptingComponentUtils.SCRIPT_FILE, scriptFile); - runner.setProperty(actionHandler, ScriptingComponentUtils.SCRIPT_BODY, (String) null); - runner.setProperty(actionHandler, ScriptingComponentUtils.MODULES, (String) null); - runner.enableControllerService(actionHandler); - - return actionHandler; - } - - public static class MockScriptedActionHandler extends ScriptedActionHandler implements AccessibleScriptingComponentHelper { - - @Override - public ScriptingComponentHelper getScriptingComponentHelper() { - return this.scriptingComponentHelper; - } - } -} \ No newline at end of file diff --git a/nifi-nar-bundles/nifi-scripting-bundle/nifi-scripting-processors/src/test/java/org/apache/nifi/rules/handlers/script/TestProcessor.java b/nifi-nar-bundles/nifi-scripting-bundle/nifi-scripting-processors/src/test/java/org/apache/nifi/rules/handlers/script/TestProcessor.java deleted file mode 100644 index 62645e5210..0000000000 --- a/nifi-nar-bundles/nifi-scripting-bundle/nifi-scripting-processors/src/test/java/org/apache/nifi/rules/handlers/script/TestProcessor.java +++ /dev/null @@ -1,47 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.nifi.rules.handlers.script; - -import org.apache.nifi.components.PropertyDescriptor; -import org.apache.nifi.processor.AbstractProcessor; -import org.apache.nifi.processor.ProcessContext; -import org.apache.nifi.processor.ProcessSession; -import org.apache.nifi.processor.exception.ProcessException; - -import java.util.ArrayList; -import java.util.List; - -public class TestProcessor extends AbstractProcessor { - - private static PropertyDescriptor prop = new PropertyDescriptor.Builder() - .name("scripted-action-handler-test") - .description("Scripted Action Handler") - .identifiesControllerService(ScriptedActionHandler.class) - .required(true) - .build(); - - @Override - public void onTrigger(ProcessContext context, ProcessSession session) throws ProcessException { - } - - @Override - protected List getSupportedPropertyDescriptors() { - List properties = new ArrayList<>(); - properties.add(prop); - return properties; - } -} diff --git a/nifi-nar-bundles/nifi-scripting-bundle/nifi-scripting-processors/src/test/resources/groovy/test_action_handler.groovy b/nifi-nar-bundles/nifi-scripting-bundle/nifi-scripting-processors/src/test/resources/groovy/test_action_handler.groovy deleted file mode 100644 index d2ec220f1f..0000000000 --- a/nifi-nar-bundles/nifi-scripting-bundle/nifi-scripting-processors/src/test/resources/groovy/test_action_handler.groovy +++ /dev/null @@ -1,32 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the 'License') you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an 'AS IS' BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import org.apache.nifi.controller.AbstractControllerService -import org.apache.nifi.rules.Action -import org.apache.nifi.rules.ActionHandler - - -class GroovyActionHandler extends AbstractControllerService implements ActionHandler { - - @Override - void execute(Action action, Map facts) { - // Add a fact for verification that execute was successfully performed - facts['testFact'] = 42 - } -} - -actionHandler = new GroovyActionHandler() diff --git a/nifi-nar-bundles/nifi-scripting-bundle/nifi-scripting-processors/src/test/resources/groovy/test_propertycontext_action_handler.groovy b/nifi-nar-bundles/nifi-scripting-bundle/nifi-scripting-processors/src/test/resources/groovy/test_propertycontext_action_handler.groovy deleted file mode 100644 index 5d48355fa3..0000000000 --- a/nifi-nar-bundles/nifi-scripting-bundle/nifi-scripting-processors/src/test/resources/groovy/test_propertycontext_action_handler.groovy +++ /dev/null @@ -1,49 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the 'License') you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an 'AS IS' BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import org.apache.nifi.context.PropertyContext -import org.apache.nifi.controller.AbstractControllerService -import org.apache.nifi.reporting.BulletinRepository -import org.apache.nifi.reporting.ReportingContext -import org.apache.nifi.reporting.Severity -import org.apache.nifi.rules.Action -import org.apache.nifi.rules.PropertyContextActionHandler - - -class GroovyPropertyContextActionHandler extends AbstractControllerService implements PropertyContextActionHandler { - - @Override - void execute(Action action, Map facts) { - // Add a fact for verification that execute was successfully performed - facts['testFact'] = 42 - } - - @Override - void execute(PropertyContext propertyContext, Action action, Map facts) { - // Add a fact for verification that execute was successfully performed - if (propertyContext instanceof ReportingContext) { - ReportingContext context = (ReportingContext) propertyContext - BulletinRepository bulletinRepository = context.bulletinRepository - bulletinRepository.addBulletin(context.createBulletin('Rules Alert', Severity.INFO, 'This should be sent as an alert!\n' - + 'Alert Facts:\n' - + 'Field: cpu, Value: 90\n' - + 'Field: jvmHeap, Value: 1000000\n')) - } - } -} - -actionHandler = new GroovyPropertyContextActionHandler() diff --git a/nifi-nar-bundles/nifi-scripting-bundle/nifi-scripting-processors/src/test/resources/groovy/test_rules_engine.groovy b/nifi-nar-bundles/nifi-scripting-bundle/nifi-scripting-processors/src/test/resources/groovy/test_rules_engine.groovy deleted file mode 100644 index ffc966d9e8..0000000000 --- a/nifi-nar-bundles/nifi-scripting-bundle/nifi-scripting-processors/src/test/resources/groovy/test_rules_engine.groovy +++ /dev/null @@ -1,39 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the 'License') you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an 'AS IS' BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import org.apache.nifi.controller.AbstractControllerService -import org.apache.nifi.rules.Action -import org.apache.nifi.rules.engine.RulesEngineService - - -class GroovyRulesEngine extends AbstractControllerService implements RulesEngineService { - - - @Override - List fireRules(Map facts) { - def actions = [] as List - if (facts['predictedQueuedCount'] > 50) { - actions << new Action('LOG', ['level': 'DEBUG']) - } - if (facts['predictedTimeToBytesBackpressureMillis'] < 300000) { - actions << new Action('ALERT', ['message': 'Time to backpressure < 5 mins']) - } - actions - } -} - -rulesEngine = new GroovyRulesEngine() diff --git a/nifi-nar-bundles/nifi-sql-reporting-bundle/nifi-sql-reporting-tasks/pom.xml b/nifi-nar-bundles/nifi-sql-reporting-bundle/nifi-sql-reporting-tasks/pom.xml index e98f9a321f..f710dab1c5 100644 --- a/nifi-nar-bundles/nifi-sql-reporting-bundle/nifi-sql-reporting-tasks/pom.xml +++ b/nifi-nar-bundles/nifi-sql-reporting-bundle/nifi-sql-reporting-tasks/pom.xml @@ -96,12 +96,6 @@ 2.0.0-SNAPSHOT test - - org.apache.nifi - nifi-rules-engine-service-api - 2.0.0-SNAPSHOT - provided - diff --git a/nifi-nar-bundles/nifi-sql-reporting-bundle/nifi-sql-reporting-tasks/src/main/java/org/apache/nifi/reporting/sql/MetricsEventReportingTask.java b/nifi-nar-bundles/nifi-sql-reporting-bundle/nifi-sql-reporting-tasks/src/main/java/org/apache/nifi/reporting/sql/MetricsEventReportingTask.java deleted file mode 100644 index 1f0831e8fe..0000000000 --- a/nifi-nar-bundles/nifi-sql-reporting-bundle/nifi-sql-reporting-tasks/src/main/java/org/apache/nifi/reporting/sql/MetricsEventReportingTask.java +++ /dev/null @@ -1,119 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.nifi.reporting.sql; - -import org.apache.nifi.annotation.behavior.Stateful; -import org.apache.nifi.annotation.documentation.CapabilityDescription; -import org.apache.nifi.annotation.documentation.Tags; -import org.apache.nifi.annotation.lifecycle.OnScheduled; -import org.apache.nifi.components.PropertyDescriptor; -import org.apache.nifi.components.state.Scope; -import org.apache.nifi.controller.ConfigurationContext; -import org.apache.nifi.reporting.AbstractReportingTask; -import org.apache.nifi.reporting.ReportingContext; -import org.apache.nifi.reporting.ReportingInitializationContext; -import org.apache.nifi.reporting.sql.util.QueryMetricsUtil; -import org.apache.nifi.rules.Action; -import org.apache.nifi.rules.PropertyContextActionHandler; -import org.apache.nifi.rules.engine.RulesEngineService; -import org.apache.nifi.serialization.record.Record; -import org.apache.nifi.serialization.record.ResultSetRecordSet; - -import java.util.ArrayList; -import java.util.Collections; -import java.util.HashMap; -import java.util.List; -import java.util.Map; - -import static org.apache.nifi.reporting.sql.util.TrackedQueryTime.BULLETIN_END_TIME; -import static org.apache.nifi.reporting.sql.util.TrackedQueryTime.BULLETIN_START_TIME; -import static org.apache.nifi.reporting.sql.util.TrackedQueryTime.PROVENANCE_END_TIME; -import static org.apache.nifi.reporting.sql.util.TrackedQueryTime.PROVENANCE_START_TIME; -import static org.apache.nifi.util.db.JdbcProperties.VARIABLE_REGISTRY_ONLY_DEFAULT_PRECISION; -import static org.apache.nifi.util.db.JdbcProperties.VARIABLE_REGISTRY_ONLY_DEFAULT_SCALE; - -@Tags({"reporting", "rules", "action", "action handler", "status", "connection", "processor", "jvm", "metrics", "history", "bulletin", "sql"}) -@CapabilityDescription("Triggers rules-driven actions based on metrics values ") -@Stateful(scopes = Scope.LOCAL, description = "Stores the Reporting Task's last execution time so that on restart the task knows where it left off.") -public class MetricsEventReportingTask extends AbstractReportingTask implements QueryTimeAware { - - private List properties; - private MetricsQueryService metricsQueryService; - private volatile RulesEngineService rulesEngineService; - private volatile PropertyContextActionHandler actionHandler; - - @Override - protected List getSupportedPropertyDescriptors() { - return properties; - } - - @Override - protected void init(final ReportingInitializationContext config) { - final List properties = new ArrayList<>(); - properties.add(QueryMetricsUtil.QUERY); - properties.add(QueryMetricsUtil.RULES_ENGINE); - properties.add(QueryMetricsUtil.ACTION_HANDLER); - properties.add(VARIABLE_REGISTRY_ONLY_DEFAULT_PRECISION); - properties.add(VARIABLE_REGISTRY_ONLY_DEFAULT_SCALE); - this.properties = Collections.unmodifiableList(properties); - } - - @OnScheduled - public void setup(final ConfigurationContext context) { - actionHandler = context.getProperty(QueryMetricsUtil.ACTION_HANDLER).asControllerService(PropertyContextActionHandler.class); - rulesEngineService = context.getProperty(QueryMetricsUtil.RULES_ENGINE).asControllerService(RulesEngineService.class); - final Integer defaultPrecision = context.getProperty(VARIABLE_REGISTRY_ONLY_DEFAULT_PRECISION).evaluateAttributeExpressions().asInteger(); - final Integer defaultScale = context.getProperty(VARIABLE_REGISTRY_ONLY_DEFAULT_SCALE).evaluateAttributeExpressions().asInteger(); - metricsQueryService = new MetricsSqlQueryService(getLogger(), defaultPrecision, defaultScale); - } - - @Override - public void onTrigger(ReportingContext context) { - String sql = context.getProperty(QueryMetricsUtil.QUERY).evaluateAttributeExpressions().getValue(); - try { - sql = processStartAndEndTimes(context, sql, BULLETIN_START_TIME, BULLETIN_END_TIME); - sql = processStartAndEndTimes(context, sql, PROVENANCE_START_TIME, PROVENANCE_END_TIME); - - fireRules(context, actionHandler, rulesEngineService, sql); - } catch (Exception e) { - getLogger().error("Error opening loading rules: {}", e.getMessage(), e); - } - } - - private void fireRules(ReportingContext context, PropertyContextActionHandler actionHandler, RulesEngineService engine, String query) throws Exception { - getLogger().debug("Executing query: {}", query); - QueryResult queryResult = metricsQueryService.query(context, query); - ResultSetRecordSet recordSet = metricsQueryService.getResultSetRecordSet(queryResult); - Record record; - try { - while ((record = recordSet.next()) != null) { - final Map facts = new HashMap<>(); - for (String fieldName : record.getRawFieldNames()) { - facts.put(fieldName, record.getValue(fieldName)); - } - List actions = engine.fireRules(facts); - if (actions == null || actions.isEmpty()) { - getLogger().debug("No actions required for provided facts."); - } else { - actions.forEach(action -> actionHandler.execute(context, action, facts)); - } - } - } finally { - metricsQueryService.closeQuietly(recordSet); - } - } -} \ No newline at end of file diff --git a/nifi-nar-bundles/nifi-sql-reporting-bundle/nifi-sql-reporting-tasks/src/main/java/org/apache/nifi/reporting/sql/util/QueryMetricsUtil.java b/nifi-nar-bundles/nifi-sql-reporting-bundle/nifi-sql-reporting-tasks/src/main/java/org/apache/nifi/reporting/sql/util/QueryMetricsUtil.java index 868dae1132..f0c1c78f07 100644 --- a/nifi-nar-bundles/nifi-sql-reporting-bundle/nifi-sql-reporting-tasks/src/main/java/org/apache/nifi/reporting/sql/util/QueryMetricsUtil.java +++ b/nifi-nar-bundles/nifi-sql-reporting-bundle/nifi-sql-reporting-tasks/src/main/java/org/apache/nifi/reporting/sql/util/QueryMetricsUtil.java @@ -24,9 +24,6 @@ import org.apache.nifi.components.ValidationResult; import org.apache.nifi.components.Validator; import org.apache.nifi.expression.ExpressionLanguageScope; import org.apache.nifi.record.sink.RecordSinkService; -import org.apache.nifi.rules.PropertyContextActionHandler; -import org.apache.nifi.rules.engine.RulesEngineService; - public class QueryMetricsUtil { @@ -59,22 +56,6 @@ public class QueryMetricsUtil { .required(true) .build(); - public static final PropertyDescriptor RULES_ENGINE = new PropertyDescriptor.Builder() - .name("rules-engine-service") - .displayName("Rules Engine Service") - .description("Specifies the Controller Service to use for applying rules to metrics.") - .identifiesControllerService(RulesEngineService.class) - .required(true) - .build(); - - public static final PropertyDescriptor ACTION_HANDLER = new PropertyDescriptor.Builder() - .name("action-handler") - .displayName("Event Action Handler") - .description("Handler that will execute the defined action returned from rules engine (if Action type is supported by the handler)") - .identifiesControllerService(PropertyContextActionHandler.class) - .required(true) - .build(); - public static class SqlValidator implements Validator { @Override public ValidationResult validate(final String subject, final String input, final ValidationContext context) { diff --git a/nifi-nar-bundles/nifi-sql-reporting-bundle/nifi-sql-reporting-tasks/src/main/resources/META-INF/services/org.apache.nifi.reporting.ReportingTask b/nifi-nar-bundles/nifi-sql-reporting-bundle/nifi-sql-reporting-tasks/src/main/resources/META-INF/services/org.apache.nifi.reporting.ReportingTask index c3f5883710..2ab79a8bb2 100644 --- a/nifi-nar-bundles/nifi-sql-reporting-bundle/nifi-sql-reporting-tasks/src/main/resources/META-INF/services/org.apache.nifi.reporting.ReportingTask +++ b/nifi-nar-bundles/nifi-sql-reporting-bundle/nifi-sql-reporting-tasks/src/main/resources/META-INF/services/org.apache.nifi.reporting.ReportingTask @@ -14,4 +14,3 @@ # limitations under the License. org.apache.nifi.reporting.sql.QueryNiFiReportingTask -org.apache.nifi.reporting.sql.MetricsEventReportingTask \ No newline at end of file diff --git a/nifi-nar-bundles/nifi-sql-reporting-bundle/nifi-sql-reporting-tasks/src/main/resources/docs/org.apache.nifi.reporting.sql.MetricsEventReportingTask/additionalDetails.html b/nifi-nar-bundles/nifi-sql-reporting-bundle/nifi-sql-reporting-tasks/src/main/resources/docs/org.apache.nifi.reporting.sql.MetricsEventReportingTask/additionalDetails.html deleted file mode 100644 index f29975f236..0000000000 --- a/nifi-nar-bundles/nifi-sql-reporting-bundle/nifi-sql-reporting-tasks/src/main/resources/docs/org.apache.nifi.reporting.sql.MetricsEventReportingTask/additionalDetails.html +++ /dev/null @@ -1,63 +0,0 @@ - - - - - - Metrics Event Reporting Task - - - - - -

Summary

-

- This reporting task can be used to issue SQL queries against various NiFi metrics information, submit returned data to a rules engine (which will determine if any actions should be performed) - and execute the prescribed actions using action handlers. This task requires a RulesEngineService (which will identify any actions that should be performed) and an ActionHandler which will execute the action(s). - A distinct ActionHandler can be used to service all events or an ActionHandlerLookup can be used for dynamic handler lookup. NOTE: Optimally action handler should be associated with the expected action types - returned from the rules engine. -

-

- The reporting task can uniquely handle items from the bulletin and provenance repositories. This means that an item will only be processed once when the query is set to unique. - The query can be set to unique by defining a time window with special sql placeholders ($bulletinStartTime, $bulletinEndTime, $provenanceStartTime, $provenanceEndTime) - that the reporting task will evaluate runtime. See the SQL Query Examples section. -

-

-

SQL Query Examples

-

- Example: Select all fields from the CONNECTION_STATUS table:
-

SELECT * FROM CONNECTION_STATUS
-

-
-

- Example: Select connection IDs where time-to-backpressure (based on queue count) is less than 5 minutes:
-

SELECT connectionId FROM CONNECTION_STATUS_PREDICTIONS WHERE predictedTimeToCountBackpressureMillis < 300000
-

-
-

- Example: Get the unique bulletin categories associated with errors:
-

SELECT DISTINCT(bulletinCategory) FROM BULLETINS WHERE bulletinLevel = "ERROR"
-

-

- Example: Select all fields from the BULLETINS table with time window:
-

SELECT * from BULLETINS WHERE bulletinTimestamp > $bulletinStartTime AND bulletinTimestamp <= $bulletinEndTime
-

-

- Example: Select all fields from the PROVENANCE table with time window:
-

SELECT * from PROVENANCE where timestampMillis > $provenanceStartTime and timestampMillis <= $provenanceEndTime
-

-
- - \ No newline at end of file diff --git a/nifi-nar-bundles/nifi-sql-reporting-bundle/nifi-sql-reporting-tasks/src/test/java/org/apache/nifi/reporting/sql/TestMetricsEventReportingTask.java b/nifi-nar-bundles/nifi-sql-reporting-bundle/nifi-sql-reporting-tasks/src/test/java/org/apache/nifi/reporting/sql/TestMetricsEventReportingTask.java deleted file mode 100644 index 196871baca..0000000000 --- a/nifi-nar-bundles/nifi-sql-reporting-bundle/nifi-sql-reporting-tasks/src/test/java/org/apache/nifi/reporting/sql/TestMetricsEventReportingTask.java +++ /dev/null @@ -1,397 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.nifi.reporting.sql; - -import org.apache.nifi.attribute.expression.language.StandardPropertyValue; -import org.apache.nifi.components.PropertyDescriptor; -import org.apache.nifi.components.PropertyValue; -import org.apache.nifi.components.state.Scope; -import org.apache.nifi.context.PropertyContext; -import org.apache.nifi.controller.ConfigurationContext; -import org.apache.nifi.controller.status.ConnectionStatus; -import org.apache.nifi.controller.status.ProcessGroupStatus; -import org.apache.nifi.controller.status.ProcessorStatus; -import org.apache.nifi.controller.status.analytics.ConnectionStatusPredictions; -import org.apache.nifi.logging.ComponentLog; -import org.apache.nifi.processor.Processor; -import org.apache.nifi.provenance.MockProvenanceRepository; -import org.apache.nifi.provenance.ProvenanceEventRecord; -import org.apache.nifi.provenance.ProvenanceEventType; -import org.apache.nifi.reporting.Bulletin; -import org.apache.nifi.reporting.BulletinFactory; -import org.apache.nifi.reporting.BulletinQuery; -import org.apache.nifi.reporting.ComponentType; -import org.apache.nifi.reporting.EventAccess; -import org.apache.nifi.reporting.InitializationException; -import org.apache.nifi.reporting.ReportingContext; -import org.apache.nifi.reporting.ReportingInitializationContext; -import org.apache.nifi.reporting.sql.util.QueryMetricsUtil; -import org.apache.nifi.reporting.sql.util.TrackedQueryTime; -import org.apache.nifi.rules.MockPropertyContextActionHandler; -import org.apache.nifi.rules.PropertyContextActionHandler; -import org.apache.nifi.rules.engine.MockRulesEngineService; -import org.apache.nifi.rules.engine.RulesEngineService; -import org.apache.nifi.state.MockStateManager; -import org.apache.nifi.util.MockBulletinRepository; -import org.apache.nifi.util.MockFlowFile; -import org.apache.nifi.util.MockProcessSession; -import org.apache.nifi.util.MockPropertyValue; -import org.apache.nifi.util.SharedSessionState; -import org.apache.nifi.util.db.JdbcProperties; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; -import org.mockito.Mockito; -import org.mockito.stubbing.Answer; - -import java.io.IOException; -import java.time.Instant; -import java.util.ArrayList; -import java.util.Collection; -import java.util.Collections; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Optional; -import java.util.UUID; -import java.util.concurrent.atomic.AtomicLong; - -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.mockito.Mockito.mock; - -class TestMetricsEventReportingTask { - - private ReportingContext context; - private MockMetricsEventReportingTask reportingTask; - private MockPropertyContextActionHandler actionHandler; - private ProcessGroupStatus status; - private MockQueryBulletinRepository mockBulletinRepository; - private MockProvenanceRepository mockProvenanceRepository; - private AtomicLong currentTime; - private MockStateManager mockStateManager; - - @BeforeEach - public void setup() { - currentTime = new AtomicLong(); - status = new ProcessGroupStatus(); - actionHandler = new MockPropertyContextActionHandler(); - status.setId("1234"); - status.setFlowFilesReceived(5); - status.setBytesReceived(10000); - status.setFlowFilesSent(10); - status.setBytesRead(20000L); - status.setBytesSent(20000); - status.setQueuedCount(100); - status.setQueuedContentSize(1024L); - status.setBytesWritten(80000L); - status.setActiveThreadCount(5); - - // create a processor status with processing time - ProcessorStatus procStatus = new ProcessorStatus(); - procStatus.setId("proc"); - procStatus.setProcessingNanos(123456789); - - Collection processorStatuses = new ArrayList<>(); - processorStatuses.add(procStatus); - status.setProcessorStatus(processorStatuses); - - ConnectionStatusPredictions connectionStatusPredictions = new ConnectionStatusPredictions(); - connectionStatusPredictions.setPredictedTimeToCountBackpressureMillis(1000); - connectionStatusPredictions.setPredictedTimeToBytesBackpressureMillis(1000); - connectionStatusPredictions.setNextPredictedQueuedCount(1000000000); - connectionStatusPredictions.setNextPredictedQueuedBytes(1000000000000000L); - - ConnectionStatus root1ConnectionStatus = new ConnectionStatus(); - root1ConnectionStatus.setId("root1"); - root1ConnectionStatus.setQueuedCount(1000); - root1ConnectionStatus.setPredictions(connectionStatusPredictions); - - ConnectionStatus root2ConnectionStatus = new ConnectionStatus(); - root2ConnectionStatus.setId("root2"); - root2ConnectionStatus.setQueuedCount(500); - root2ConnectionStatus.setPredictions(connectionStatusPredictions); - - Collection rootConnectionStatuses = new ArrayList<>(); - rootConnectionStatuses.add(root1ConnectionStatus); - rootConnectionStatuses.add(root2ConnectionStatus); - status.setConnectionStatus(rootConnectionStatuses); - } - - @Test - void testConnectionStatusTable() throws InitializationException { - final Map properties = new HashMap<>(); - properties.put(QueryMetricsUtil.QUERY, "select connectionId, predictedQueuedCount, predictedTimeToBytesBackpressureMillis from CONNECTION_STATUS_PREDICTIONS"); - reportingTask = initTask(properties); - reportingTask.onTrigger(context); - List propertyContexts = actionHandler.getPropertyContexts(); - assertEquals(2, actionHandler.getRows().size()); - assertEquals(2, propertyContexts.size()); - } - - @Test - void testUniqueBulletinQueryIsInTimeWindow() throws InitializationException { - final Map properties = new HashMap<>(); - properties.put(QueryMetricsUtil.QUERY, "select bulletinCategory from BULLETINS where bulletinTimestamp > $bulletinStartTime and bulletinTimestamp <= $bulletinEndTime"); - reportingTask = initTask(properties); - currentTime.set(Instant.now().toEpochMilli()); - reportingTask.onTrigger(context); - assertEquals(1, actionHandler.getRows().size()); - - actionHandler.reset(); - final Bulletin bulletin = BulletinFactory.createBulletin(ComponentType.CONTROLLER_SERVICE.name().toLowerCase(), "WARN", "test bulletin 2", "testFlowFileUuid"); - mockBulletinRepository.addBulletin(bulletin); - currentTime.set(bulletin.getTimestamp().getTime()); - reportingTask.onTrigger(context); - assertEquals(1, actionHandler.getRows().size()); - } - - @Test - void testUniqueBulletinQueryIsOutOfTimeWindow() throws InitializationException { - final Map properties = new HashMap<>(); - properties.put(QueryMetricsUtil.QUERY, "select bulletinCategory from BULLETINS where bulletinTimestamp > $bulletinStartTime and bulletinTimestamp <= $bulletinEndTime"); - reportingTask = initTask(properties); - currentTime.set(Instant.now().toEpochMilli()); - reportingTask.onTrigger(context); - assertEquals(1, actionHandler.getRows().size()); - - actionHandler.reset(); - final Bulletin bulletin = BulletinFactory.createBulletin(ComponentType.CONTROLLER_SERVICE.name().toLowerCase(), "WARN", "test bulletin 2", "testFlowFileUuid"); - mockBulletinRepository.addBulletin(bulletin); - currentTime.set(bulletin.getTimestamp().getTime() - 1); - reportingTask.onTrigger(context); - assertEquals(0, actionHandler.getRows().size()); - } - - @Test - void testUniqueProvenanceQueryIsInTimeWindow() throws InitializationException { - final Map properties = new HashMap<>(); - properties.put(QueryMetricsUtil.QUERY, "select componentId from PROVENANCE where timestampMillis > $provenanceStartTime and timestampMillis <= $provenanceEndTime"); - reportingTask = initTask(properties); - currentTime.set(Instant.now().toEpochMilli()); - reportingTask.onTrigger(context); - assertEquals(1, actionHandler.getRows().size()); - - actionHandler.reset(); - - MockFlowFile mockFlowFile = new MockFlowFile(2L); - ProvenanceEventRecord prov2 = mockProvenanceRepository.eventBuilder() - .setEventType(ProvenanceEventType.CREATE) - .fromFlowFile(mockFlowFile) - .setComponentId("2") - .setComponentType("ReportingTask") - .setFlowFileUUID("I am FlowFile 2") - .setEventTime(Instant.now().toEpochMilli()) - .setEventDuration(100) - .setTransitUri("test://") - .setSourceSystemFlowFileIdentifier("I am FlowFile 2") - .setAlternateIdentifierUri("remote://test") - .build(); - mockProvenanceRepository.registerEvent(prov2); - - currentTime.set(prov2.getEventTime()); - reportingTask.onTrigger(context); - - assertEquals(1, actionHandler.getRows().size()); - } - - @Test - void testUniqueProvenanceQueryIsOutOfTimeWindow() throws InitializationException { - final Map properties = new HashMap<>(); - properties.put(QueryMetricsUtil.QUERY, "select componentId from PROVENANCE where timestampMillis > $provenanceStartTime and timestampMillis <= $provenanceEndTime"); - reportingTask = initTask(properties); - currentTime.set(Instant.now().toEpochMilli()); - reportingTask.onTrigger(context); - assertEquals(1, actionHandler.getRows().size()); - - actionHandler.reset(); - - MockFlowFile mockFlowFile = new MockFlowFile(2L); - ProvenanceEventRecord prov2 = mockProvenanceRepository.eventBuilder() - .setEventType(ProvenanceEventType.CREATE) - .fromFlowFile(mockFlowFile) - .setComponentId("2") - .setComponentType("ReportingTask") - .setFlowFileUUID("I am FlowFile 2") - .setEventTime(Instant.now().toEpochMilli()) - .setEventDuration(100) - .setTransitUri("test://") - .setSourceSystemFlowFileIdentifier("I am FlowFile 2") - .setAlternateIdentifierUri("remote://test") - .build(); - mockProvenanceRepository.registerEvent(prov2); - - currentTime.set(prov2.getEventTime() - 1); - reportingTask.onTrigger(context); - - assertEquals(0, actionHandler.getRows().size()); - } - - @Test - void testTimeWindowFromStateMap() throws IOException, InitializationException { - final Map properties = new HashMap<>(); - properties.put(QueryMetricsUtil.RECORD_SINK, "mock-record-sink"); - properties.put(QueryMetricsUtil.QUERY, "select * from BULLETINS, PROVENANCE where " + - "bulletinTimestamp > $bulletinStartTime and bulletinTimestamp <= $bulletinEndTime " + - "and timestampMillis > $provenanceStartTime and timestampMillis <= $provenanceEndTime"); - reportingTask = initTask(properties); - - long testBulletinStartTime = 1609538145L; - long testProvenanceStartTime = 1641074145L; - final Map stateMap = new HashMap<>(); - stateMap.put(TrackedQueryTime.BULLETIN_START_TIME.name(), String.valueOf(testBulletinStartTime)); - stateMap.put(TrackedQueryTime.PROVENANCE_START_TIME.name(), String.valueOf(testProvenanceStartTime)); - mockStateManager.setState(stateMap, Scope.LOCAL); - - final long bulletinStartTime = Long.parseLong(context.getStateManager().getState(Scope.LOCAL).get(TrackedQueryTime.BULLETIN_START_TIME.name())); - final long provenanceStartTime = Long.parseLong(context.getStateManager().getState(Scope.LOCAL).get(TrackedQueryTime.PROVENANCE_START_TIME.name())); - - assertEquals(testBulletinStartTime, bulletinStartTime); - assertEquals(testProvenanceStartTime, provenanceStartTime); - - final long currentTime = Instant.now().toEpochMilli(); - this.currentTime.set(currentTime); - - reportingTask.onTrigger(context); - - final long updatedBulletinStartTime = Long.parseLong(context.getStateManager().getState(Scope.LOCAL).get(TrackedQueryTime.BULLETIN_START_TIME.name())); - final long updatedProvenanceStartTime = Long.parseLong(context.getStateManager().getState(Scope.LOCAL).get(TrackedQueryTime.PROVENANCE_START_TIME.name())); - - assertEquals(currentTime, updatedBulletinStartTime); - assertEquals(currentTime, updatedProvenanceStartTime); - } - - private MockMetricsEventReportingTask initTask(Map customProperties) throws InitializationException { - final ComponentLog logger = Mockito.mock(ComponentLog.class); - reportingTask = new MockMetricsEventReportingTask(); - final ReportingInitializationContext initContext = Mockito.mock(ReportingInitializationContext.class); - Mockito.when(initContext.getIdentifier()).thenReturn(UUID.randomUUID().toString()); - Mockito.when(initContext.getLogger()).thenReturn(logger); - reportingTask.initialize(initContext); - Map properties = new HashMap<>(); - - for (final PropertyDescriptor descriptor : reportingTask.getSupportedPropertyDescriptors()) { - properties.put(descriptor, descriptor.getDefaultValue()); - } - properties.putAll(customProperties); - - context = Mockito.mock(ReportingContext.class); - Mockito.when(context.isAnalyticsEnabled()).thenReturn(true); - mockStateManager = new MockStateManager(reportingTask); - Mockito.when(context.getStateManager()).thenReturn(mockStateManager); - - Mockito.doAnswer((Answer) invocation -> { - final PropertyDescriptor descriptor = invocation.getArgument(0, PropertyDescriptor.class); - return new MockPropertyValue(properties.get(descriptor)); - }).when(context).getProperty(Mockito.any(PropertyDescriptor.class)); - - final EventAccess eventAccess = Mockito.mock(EventAccess.class); - Mockito.when(context.getEventAccess()).thenReturn(eventAccess); - Mockito.when(eventAccess.getControllerStatus()).thenReturn(status); - - final PropertyValue pValue = Mockito.mock(StandardPropertyValue.class); - actionHandler = new MockPropertyContextActionHandler(); - Mockito.when(pValue.asControllerService(PropertyContextActionHandler.class)).thenReturn(actionHandler); - - final PropertyValue resValue = Mockito.mock(StandardPropertyValue.class); - MockRulesEngineService rulesEngineService = new MockRulesEngineService(); - Mockito.when(resValue.asControllerService(RulesEngineService.class)).thenReturn(rulesEngineService); - - ConfigurationContext configContext = Mockito.mock(ConfigurationContext.class); - Mockito.when(configContext.getProperty(QueryMetricsUtil.RULES_ENGINE)).thenReturn(resValue); - Mockito.when(configContext.getProperty(QueryMetricsUtil.ACTION_HANDLER)).thenReturn(pValue); - Mockito.when(configContext.getProperty(JdbcProperties.VARIABLE_REGISTRY_ONLY_DEFAULT_PRECISION)).thenReturn(new MockPropertyValue("10")); - Mockito.when(configContext.getProperty(JdbcProperties.VARIABLE_REGISTRY_ONLY_DEFAULT_SCALE)).thenReturn(new MockPropertyValue("0")); - reportingTask.setup(configContext); - - setupMockProvenanceRepository(eventAccess); - setupMockBulletinRepository(); - - return reportingTask; - } - - private final class MockMetricsEventReportingTask extends MetricsEventReportingTask { - @Override - public long getCurrentTime() { - return currentTime.get(); - } - } - - private void setupMockBulletinRepository() { - mockBulletinRepository = new MockQueryBulletinRepository(); - mockBulletinRepository.addBulletin(BulletinFactory.createBulletin(ComponentType.PROCESSOR.name().toLowerCase(), "WARN", "test bulletin 1", "testFlowFileUuid")); - - Mockito.when(context.getBulletinRepository()).thenReturn(mockBulletinRepository); - } - - private void setupMockProvenanceRepository(final EventAccess eventAccess) { - - mockProvenanceRepository = new MockProvenanceRepository(); - long currentTimeMillis = System.currentTimeMillis(); - Map previousAttributes = new HashMap<>(); - previousAttributes.put("mime.type", "application/json"); - previousAttributes.put("test.value", "A"); - Map updatedAttributes = new HashMap<>(previousAttributes); - updatedAttributes.put("test.value", "B"); - - // Generate provenance events and put them in a repository - Processor processor = mock(Processor.class); - SharedSessionState sharedState = new SharedSessionState(processor, new AtomicLong(0)); - MockProcessSession processSession = new MockProcessSession(sharedState, processor); - MockFlowFile mockFlowFile = processSession.createFlowFile("Test content".getBytes()); - - ProvenanceEventRecord prov1 = mockProvenanceRepository.eventBuilder() - .setEventType(ProvenanceEventType.CREATE) - .fromFlowFile(mockFlowFile) - .setComponentId("1") - .setComponentType("ReportingTask") - .setFlowFileUUID("I am FlowFile 1") - .setEventTime(currentTimeMillis) - .setEventDuration(100) - .setTransitUri("test://") - .setSourceSystemFlowFileIdentifier("I am FlowFile 1") - .setAlternateIdentifierUri("remote://test") - .setAttributes(previousAttributes, updatedAttributes) - .build(); - - mockProvenanceRepository.registerEvent(prov1); - - Mockito.when(eventAccess.getProvenanceRepository()).thenReturn(mockProvenanceRepository); - } - - private static class MockQueryBulletinRepository extends MockBulletinRepository { - Map> bulletins = new HashMap<>(); - - @Override - public void addBulletin(Bulletin bulletin) { - bulletins.computeIfAbsent(bulletin.getCategory(), key -> new ArrayList<>()) - .add(bulletin); - } - - @Override - public List findBulletins(BulletinQuery bulletinQuery) { - return new ArrayList<>( - Optional.ofNullable(bulletins.get(bulletinQuery.getSourceType().name().toLowerCase())) - .orElse(Collections.emptyList()) - ); - } - - @Override - public List findBulletinsForController() { - return Optional.ofNullable(bulletins.get("controller")) - .orElse(Collections.emptyList()); - } - } -} \ No newline at end of file diff --git a/nifi-nar-bundles/nifi-sql-reporting-bundle/nifi-sql-reporting-tasks/src/test/java/org/apache/nifi/rules/MockPropertyContextActionHandler.java b/nifi-nar-bundles/nifi-sql-reporting-bundle/nifi-sql-reporting-tasks/src/test/java/org/apache/nifi/rules/MockPropertyContextActionHandler.java deleted file mode 100644 index dbbb50a601..0000000000 --- a/nifi-nar-bundles/nifi-sql-reporting-bundle/nifi-sql-reporting-tasks/src/test/java/org/apache/nifi/rules/MockPropertyContextActionHandler.java +++ /dev/null @@ -1,67 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.nifi.rules; - -import org.apache.nifi.components.AbstractConfigurableComponent; -import org.apache.nifi.context.PropertyContext; -import org.apache.nifi.controller.ControllerServiceInitializationContext; -import org.apache.nifi.reporting.InitializationException; - -import java.util.ArrayList; -import java.util.List; -import java.util.Map; - -public class MockPropertyContextActionHandler extends AbstractConfigurableComponent implements PropertyContextActionHandler { - private final List> rows = new ArrayList<>(); - private final List propertyContexts = new ArrayList<>(); - - - @Override - public void execute(PropertyContext context, Action action, Map facts) { - propertyContexts.add(context); - execute(action, facts); - } - - @Override - public void execute(Action action, Map facts) { - rows.add(facts); - } - - - @Override - public void initialize(ControllerServiceInitializationContext context) throws InitializationException { - - } - - public List> getRows() { - return rows; - } - - public List getPropertyContexts() { - return propertyContexts; - } - - @Override - public String getIdentifier() { - return "MockPropertyContextActionHandler"; - } - - public void reset() { - rows.clear(); - propertyContexts.clear(); - } -} \ No newline at end of file diff --git a/nifi-nar-bundles/nifi-sql-reporting-bundle/nifi-sql-reporting-tasks/src/test/java/org/apache/nifi/rules/engine/MockRulesEngineService.java b/nifi-nar-bundles/nifi-sql-reporting-bundle/nifi-sql-reporting-tasks/src/test/java/org/apache/nifi/rules/engine/MockRulesEngineService.java deleted file mode 100644 index 90e8472a06..0000000000 --- a/nifi-nar-bundles/nifi-sql-reporting-bundle/nifi-sql-reporting-tasks/src/test/java/org/apache/nifi/rules/engine/MockRulesEngineService.java +++ /dev/null @@ -1,42 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.nifi.rules.engine; - -import org.apache.nifi.components.AbstractConfigurableComponent; -import org.apache.nifi.controller.ControllerServiceInitializationContext; -import org.apache.nifi.rules.Action; -import org.mockito.Mockito; - -import java.util.Collections; -import java.util.List; -import java.util.Map; - -public class MockRulesEngineService extends AbstractConfigurableComponent implements RulesEngineService { - @Override - public List fireRules(Map facts) { - return Collections.singletonList(Mockito.mock(Action.class)); - } - - @Override - public void initialize(ControllerServiceInitializationContext context) { - } - - @Override - public String getIdentifier() { - return "MockRulesEngineService"; - } -} diff --git a/nifi-nar-bundles/nifi-standard-services/nifi-rules-engine-service-api/pom.xml b/nifi-nar-bundles/nifi-standard-services/nifi-rules-engine-service-api/pom.xml deleted file mode 100644 index 0054aaca00..0000000000 --- a/nifi-nar-bundles/nifi-standard-services/nifi-rules-engine-service-api/pom.xml +++ /dev/null @@ -1,33 +0,0 @@ - - - 4.0.0 - - nifi-standard-services - org.apache.nifi - 2.0.0-SNAPSHOT - - - nifi-rules-engine-service-api - jar - - - - org.apache.nifi - nifi-api - 2.0.0-SNAPSHOT - - - \ No newline at end of file diff --git a/nifi-nar-bundles/nifi-standard-services/nifi-rules-engine-service-api/src/main/java/org/apache/nifi/rules/Action.java b/nifi-nar-bundles/nifi-standard-services/nifi-rules-engine-service-api/src/main/java/org/apache/nifi/rules/Action.java deleted file mode 100644 index b5f0829411..0000000000 --- a/nifi-nar-bundles/nifi-standard-services/nifi-rules-engine-service-api/src/main/java/org/apache/nifi/rules/Action.java +++ /dev/null @@ -1,63 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.nifi.rules; - -import java.util.HashMap; -import java.util.Map; - -/** - * An Action that should be performed given a fact has met the condition specified in a Rule. - * The type of action is dictated by the type field and attributes are used as parameters to configure - * the Action's executor/handler - */ -public class Action implements Cloneable{ - private String type; - private Map attributes; - - public Action() { - } - - public Action(String type, Map attributes) { - this.type = type; - this.attributes = attributes; - } - - public String getType() { - return type; - } - - public void setType(String type) { - this.type = type; - } - - public Map getAttributes() { - return attributes; - } - - public void setAttributes(Map attributes) { - this.attributes = attributes; - } - - public Action clone(){ - Action action = new Action(); - action.setType(type); - Map attributeMap = new HashMap<>(attributes); - action.setAttributes(attributeMap); - return action; - } - -} diff --git a/nifi-nar-bundles/nifi-standard-services/nifi-rules-engine-service-api/src/main/java/org/apache/nifi/rules/ActionHandler.java b/nifi-nar-bundles/nifi-standard-services/nifi-rules-engine-service-api/src/main/java/org/apache/nifi/rules/ActionHandler.java deleted file mode 100644 index 7161f11f79..0000000000 --- a/nifi-nar-bundles/nifi-standard-services/nifi-rules-engine-service-api/src/main/java/org/apache/nifi/rules/ActionHandler.java +++ /dev/null @@ -1,31 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.nifi.rules; - -import java.util.Map; - -/** - * An ActionHandler executes the provided {@link Action} for a given set of facts - */ -public interface ActionHandler { - /** - * Execute the given action for the provided facts - * @param action The action that should be performed by the handler - * @param facts The facts that triggered this action - */ - void execute(Action action, Map facts); -} diff --git a/nifi-nar-bundles/nifi-standard-services/nifi-rules-engine-service-api/src/main/java/org/apache/nifi/rules/PropertyContextActionHandler.java b/nifi-nar-bundles/nifi-standard-services/nifi-rules-engine-service-api/src/main/java/org/apache/nifi/rules/PropertyContextActionHandler.java deleted file mode 100644 index c82beee239..0000000000 --- a/nifi-nar-bundles/nifi-standard-services/nifi-rules-engine-service-api/src/main/java/org/apache/nifi/rules/PropertyContextActionHandler.java +++ /dev/null @@ -1,28 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.nifi.rules; - -import org.apache.nifi.context.PropertyContext; -import org.apache.nifi.controller.ControllerService; - -import java.util.Map; - -public interface PropertyContextActionHandler extends ActionHandler, ControllerService { - - void execute(PropertyContext context, Action action, Map facts); - -} diff --git a/nifi-nar-bundles/nifi-standard-services/nifi-rules-engine-service-api/src/main/java/org/apache/nifi/rules/Rule.java b/nifi-nar-bundles/nifi-standard-services/nifi-rules-engine-service-api/src/main/java/org/apache/nifi/rules/Rule.java deleted file mode 100644 index d6f69d3294..0000000000 --- a/nifi-nar-bundles/nifi-standard-services/nifi-rules-engine-service-api/src/main/java/org/apache/nifi/rules/Rule.java +++ /dev/null @@ -1,115 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.nifi.rules; - -import java.util.ArrayList; -import java.util.List; - -/** - * A Rule that defines a condition that should be met by a set of facts in order to perform - * one or more {@link Action} - */ - -public class Rule implements Cloneable{ - private String name; - private String description; - private Integer priority; - private String condition; - private List actions; - private List facts; - - public Rule() { - } - - public Rule(String name, String description, Integer priority, String condition, List actions, List facts) { - this.name = name; - this.description = description; - this.priority = priority; - this.condition = condition; - this.actions = actions; - this.facts = facts; - } - - public String getName() { - return name; - } - - public void setName(String name) { - this.name = name; - } - - public String getDescription() { - return description; - } - - public void setDescription(String description) { - this.description = description; - } - - public Integer getPriority() { - return priority; - } - - public void setPriority(Integer priority) { - this.priority = priority; - } - - public String getCondition() { - return condition; - } - - public void setCondition(String condition) { - this.condition = condition; - } - - public List getActions() { - return actions; - } - - public void setActions(List actions) { - this.actions = actions; - } - - public List getFacts() { - return facts; - } - - public void setFacts(List facts) { - this.facts = facts; - } - - @Override - public Rule clone(){ - Rule rule = new Rule(); - rule.setName(name); - rule.setDescription(description); - rule.setPriority(priority); - rule.setCondition(condition); - - if (actions != null) { - final List actionList = new ArrayList<>(); - rule.setActions(actionList); - actions.forEach(action -> actionList.add((Action)action.clone())); - } - if (facts != null){ - final List factList = new ArrayList<>(); - rule.setFacts(factList); - factList.addAll(facts); - } - return rule; - } -} diff --git a/nifi-nar-bundles/nifi-standard-services/nifi-rules-engine-service-api/src/main/java/org/apache/nifi/rules/engine/RulesEngine.java b/nifi-nar-bundles/nifi-standard-services/nifi-rules-engine-service-api/src/main/java/org/apache/nifi/rules/engine/RulesEngine.java deleted file mode 100644 index 9c66c156b1..0000000000 --- a/nifi-nar-bundles/nifi-standard-services/nifi-rules-engine-service-api/src/main/java/org/apache/nifi/rules/engine/RulesEngine.java +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.nifi.rules.engine; - -import org.apache.nifi.rules.Action; -import org.apache.nifi.rules.Rule; - -import java.util.List; -import java.util.Map; - -/** - *

- * An instance of a RulesEngine which provides access to available rules - *

- *

- */ -public interface RulesEngine { - - - List fireRules(Map facts); - Map checkRules(Map facts); - -} diff --git a/nifi-nar-bundles/nifi-standard-services/nifi-rules-engine-service-api/src/main/java/org/apache/nifi/rules/engine/RulesEngineProvider.java b/nifi-nar-bundles/nifi-standard-services/nifi-rules-engine-service-api/src/main/java/org/apache/nifi/rules/engine/RulesEngineProvider.java deleted file mode 100644 index 7f553607e5..0000000000 --- a/nifi-nar-bundles/nifi-standard-services/nifi-rules-engine-service-api/src/main/java/org/apache/nifi/rules/engine/RulesEngineProvider.java +++ /dev/null @@ -1,38 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.nifi.rules.engine; - -import org.apache.nifi.annotation.documentation.CapabilityDescription; -import org.apache.nifi.annotation.documentation.Tags; -import org.apache.nifi.controller.ControllerService; - -/** - *

- * A Controller Service that is responsible for providing an instance of a Rules Engine. - *

- *

- */ -@Tags({"rules", "rules-engine","facts","actions"}) -@CapabilityDescription("Specifies a Controller Service which provides access to an instance of a Rules Engine.") -public interface RulesEngineProvider extends ControllerService { - - /** - * Retrieve an instance of a rules engine - * @return RulesEngine instance - */ - RulesEngine getRulesEngine(); -} diff --git a/nifi-nar-bundles/nifi-standard-services/nifi-rules-engine-service-api/src/main/java/org/apache/nifi/rules/engine/RulesEngineService.java b/nifi-nar-bundles/nifi-standard-services/nifi-rules-engine-service-api/src/main/java/org/apache/nifi/rules/engine/RulesEngineService.java deleted file mode 100644 index 45ab999f66..0000000000 --- a/nifi-nar-bundles/nifi-standard-services/nifi-rules-engine-service-api/src/main/java/org/apache/nifi/rules/engine/RulesEngineService.java +++ /dev/null @@ -1,47 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.nifi.rules.engine; - -import org.apache.nifi.annotation.documentation.CapabilityDescription; -import org.apache.nifi.annotation.documentation.Tags; -import org.apache.nifi.controller.ControllerService; -import org.apache.nifi.rules.Action; - -import java.util.List; -import java.util.Map; - -/** - *

- * A Controller Service that is responsible for executing a rules engine against provided facts. The subsequent - * actions can be executed either by the rules engine or a list of {@link Action} can be returned and interrogated/executed by - * the caller. - *

- *

- */ -@Tags({"rules", "rules-engine","facts","actions"}) -@CapabilityDescription("Specifies a Controller Service which hosts a rules engine, that can be used to determine actions that should be performed given provided facts.") -public interface RulesEngineService extends ControllerService { - - /** - * Returns the list of Actions that should be triggered as determined by the rules engine for the the given facts. - * Action in this case have not been executed by the engine. - * @param facts a Map of key and facts values, as objects, that should be evaluated by the rules engine - * @return a List of Actions that should be triggered - */ - List fireRules(Map facts); - -} diff --git a/nifi-nar-bundles/nifi-standard-services/nifi-standard-services-api-nar/pom.xml b/nifi-nar-bundles/nifi-standard-services/nifi-standard-services-api-nar/pom.xml index 24805cf3c4..66351a3d87 100644 --- a/nifi-nar-bundles/nifi-standard-services/nifi-standard-services-api-nar/pom.xml +++ b/nifi-nar-bundles/nifi-standard-services/nifi-standard-services-api-nar/pom.xml @@ -70,12 +70,6 @@ 2.0.0-SNAPSHOT compile - - org.apache.nifi - nifi-rules-engine-service-api - 2.0.0-SNAPSHOT - compile - org.apache.nifi nifi-schema-registry-service-api diff --git a/nifi-nar-bundles/nifi-standard-services/pom.xml b/nifi-nar-bundles/nifi-standard-services/pom.xml index 01599838fb..de31d8a20e 100644 --- a/nifi-nar-bundles/nifi-standard-services/pom.xml +++ b/nifi-nar-bundles/nifi-standard-services/pom.xml @@ -47,7 +47,6 @@ nifi-proxy-configuration-bundle nifi-key-service-api nifi-key-service-bundle - nifi-rules-engine-service-api nifi-record-sink-api nifi-record-sink-service-bundle nifi-hadoop-dbcp-service-bundle diff --git a/nifi-nar-bundles/nifi-tcp-bundle/nifi-tcp-nar/pom.xml b/nifi-nar-bundles/nifi-tcp-bundle/nifi-tcp-nar/pom.xml deleted file mode 100644 index c77ecd5d2f..0000000000 --- a/nifi-nar-bundles/nifi-tcp-bundle/nifi-tcp-nar/pom.xml +++ /dev/null @@ -1,39 +0,0 @@ - - - - 4.0.0 - - - org.apache.nifi - nifi-tcp-bundle - 2.0.0-SNAPSHOT - - - nifi-tcp-nar - nar - - true - true - - - - - org.apache.nifi - nifi-tcp-processors - 2.0.0-SNAPSHOT - - - diff --git a/nifi-nar-bundles/nifi-tcp-bundle/nifi-tcp-nar/src/main/resources/META-INF/LICENSE b/nifi-nar-bundles/nifi-tcp-bundle/nifi-tcp-nar/src/main/resources/META-INF/LICENSE deleted file mode 100644 index a57b09a9c4..0000000000 --- a/nifi-nar-bundles/nifi-tcp-bundle/nifi-tcp-nar/src/main/resources/META-INF/LICENSE +++ /dev/null @@ -1,212 +0,0 @@ - - Apache License - Version 2.0, January 2004 - http://www.apache.org/licenses/ - - TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION - - 1. Definitions. - - "License" shall mean the terms and conditions for use, reproduction, - and distribution as defined by Sections 1 through 9 of this document. - - "Licensor" shall mean the copyright owner or entity authorized by - the copyright owner that is granting the License. - - "Legal Entity" shall mean the union of the acting entity and all - other entities that control, are controlled by, or are under common - control with that entity. For the purposes of this definition, - "control" means (i) the power, direct or indirect, to cause the - direction or management of such entity, whether by contract or - otherwise, or (ii) ownership of fifty percent (50%) or more of the - outstanding shares, or (iii) beneficial ownership of such entity. - - "You" (or "Your") shall mean an individual or Legal Entity - exercising permissions granted by this License. - - "Source" form shall mean the preferred form for making modifications, - including but not limited to software source code, documentation - source, and configuration files. - - "Object" form shall mean any form resulting from mechanical - transformation or translation of a Source form, including but - not limited to compiled object code, generated documentation, - and conversions to other media types. - - "Work" shall mean the work of authorship, whether in Source or - Object form, made available under the License, as indicated by a - copyright notice that is included in or attached to the work - (an example is provided in the Appendix below). - - "Derivative Works" shall mean any work, whether in Source or Object - form, that is based on (or derived from) the Work and for which the - editorial revisions, annotations, elaborations, or other modifications - represent, as a whole, an original work of authorship. For the purposes - of this License, Derivative Works shall not include works that remain - separable from, or merely link (or bind by name) to the interfaces of, - the Work and Derivative Works thereof. - - "Contribution" shall mean any work of authorship, including - the original version of the Work and any modifications or additions - to that Work or Derivative Works thereof, that is intentionally - submitted to Licensor for inclusion in the Work by the copyright owner - or by an individual or Legal Entity authorized to submit on behalf of - the copyright owner. For the purposes of this definition, "submitted" - means any form of electronic, verbal, or written communication sent - to the Licensor or its representatives, including but not limited to - communication on electronic mailing lists, source code control systems, - and issue tracking systems that are managed by, or on behalf of, the - Licensor for the purpose of discussing and improving the Work, but - excluding communication that is conspicuously marked or otherwise - designated in writing by the copyright owner as "Not a Contribution." - - "Contributor" shall mean Licensor and any individual or Legal Entity - on behalf of whom a Contribution has been received by Licensor and - subsequently incorporated within the Work. - - 2. Grant of Copyright License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - copyright license to reproduce, prepare Derivative Works of, - publicly display, publicly perform, sublicense, and distribute the - Work and such Derivative Works in Source or Object form. - - 3. Grant of Patent License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - (except as stated in this section) patent license to make, have made, - use, offer to sell, sell, import, and otherwise transfer the Work, - where such license applies only to those patent claims licensable - by such Contributor that are necessarily infringed by their - Contribution(s) alone or by combination of their Contribution(s) - with the Work to which such Contribution(s) was submitted. If You - institute patent litigation against any entity (including a - cross-claim or counterclaim in a lawsuit) alleging that the Work - or a Contribution incorporated within the Work constitutes direct - or contributory patent infringement, then any patent licenses - granted to You under this License for that Work shall terminate - as of the date such litigation is filed. - - 4. Redistribution. You may reproduce and distribute copies of the - Work or Derivative Works thereof in any medium, with or without - modifications, and in Source or Object form, provided that You - meet the following conditions: - - (a) You must give any other recipients of the Work or - Derivative Works a copy of this License; and - - (b) You must cause any modified files to carry prominent notices - stating that You changed the files; and - - (c) You must retain, in the Source form of any Derivative Works - that You distribute, all copyright, patent, trademark, and - attribution notices from the Source form of the Work, - excluding those notices that do not pertain to any part of - the Derivative Works; and - - (d) If the Work includes a "NOTICE" text file as part of its - distribution, then any Derivative Works that You distribute must - include a readable copy of the attribution notices contained - within such NOTICE file, excluding those notices that do not - pertain to any part of the Derivative Works, in at least one - of the following places: within a NOTICE text file distributed - as part of the Derivative Works; within the Source form or - documentation, if provided along with the Derivative Works; or, - within a display generated by the Derivative Works, if and - wherever such third-party notices normally appear. The contents - of the NOTICE file are for informational purposes only and - do not modify the License. You may add Your own attribution - notices within Derivative Works that You distribute, alongside - or as an addendum to the NOTICE text from the Work, provided - that such additional attribution notices cannot be construed - as modifying the License. - - You may add Your own copyright statement to Your modifications and - may provide additional or different license terms and conditions - for use, reproduction, or distribution of Your modifications, or - for any such Derivative Works as a whole, provided Your use, - reproduction, and distribution of the Work otherwise complies with - the conditions stated in this License. - - 5. Submission of Contributions. Unless You explicitly state otherwise, - any Contribution intentionally submitted for inclusion in the Work - by You to the Licensor shall be under the terms and conditions of - this License, without any additional terms or conditions. - Notwithstanding the above, nothing herein shall supersede or modify - the terms of any separate license agreement you may have executed - with Licensor regarding such Contributions. - - 6. Trademarks. This License does not grant permission to use the trade - names, trademarks, service marks, or product names of the Licensor, - except as required for reasonable and customary use in describing the - origin of the Work and reproducing the content of the NOTICE file. - - 7. Disclaimer of Warranty. Unless required by applicable law or - agreed to in writing, Licensor provides the Work (and each - Contributor provides its Contributions) on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - implied, including, without limitation, any warranties or conditions - of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A - PARTICULAR PURPOSE. You are solely responsible for determining the - appropriateness of using or redistributing the Work and assume any - risks associated with Your exercise of permissions under this License. - - 8. Limitation of Liability. In no event and under no legal theory, - whether in tort (including negligence), contract, or otherwise, - unless required by applicable law (such as deliberate and grossly - negligent acts) or agreed to in writing, shall any Contributor be - liable to You for damages, including any direct, indirect, special, - incidental, or consequential damages of any character arising as a - result of this License or out of the use or inability to use the - Work (including but not limited to damages for loss of goodwill, - work stoppage, computer failure or malfunction, or any and all - other commercial damages or losses), even if such Contributor - has been advised of the possibility of such damages. - - 9. Accepting Warranty or Additional Liability. While redistributing - the Work or Derivative Works thereof, You may choose to offer, - and charge a fee for, acceptance of support, warranty, indemnity, - or other liability obligations and/or rights consistent with this - License. However, in accepting such obligations, You may act only - on Your own behalf and on Your sole responsibility, not on behalf - of any other Contributor, and only if You agree to indemnify, - defend, and hold each Contributor harmless for any liability - incurred by, or claims asserted against, such Contributor by reason - of your accepting any such warranty or additional liability. - - END OF TERMS AND CONDITIONS - - APPENDIX: How to apply the Apache License to your work. - - To apply the Apache License to your work, attach the following - boilerplate notice, with the fields enclosed by brackets "[]" - replaced with your own identifying information. (Don't include - the brackets!) The text should be enclosed in the appropriate - comment syntax for the file format. We also recommend that a - file or class name and description of purpose be included on the - same "printed page" as the copyright notice for easier - identification within third-party archives. - - Copyright [yyyy] [name of copyright owner] - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - - -APACHE NIFI SUBCOMPONENTS: - -The Apache NiFi project contains subcomponents with separate copyright -notices and license terms. Your use of the source code for the these -subcomponents is subject to the terms and conditions of the following -licenses. - - diff --git a/nifi-nar-bundles/nifi-tcp-bundle/nifi-tcp-nar/src/main/resources/META-INF/NOTICE b/nifi-nar-bundles/nifi-tcp-bundle/nifi-tcp-nar/src/main/resources/META-INF/NOTICE deleted file mode 100644 index 5f7ae250f8..0000000000 --- a/nifi-nar-bundles/nifi-tcp-bundle/nifi-tcp-nar/src/main/resources/META-INF/NOTICE +++ /dev/null @@ -1,47 +0,0 @@ -nifi-tcp-nar -Copyright 2015-2020 The Apache Software Foundation - -This product includes software developed at -The Apache Software Foundation (http://www.apache.org/). - -=========================================== -Apache Software License v2 -=========================================== - -The following binary components are provided under the Apache Software License v2 - - (ASLv2) Apache Commons IO - The following NOTICE information applies: - Apache Commons IO - Copyright 2002-2017 The Apache Software Foundation - - (ASLv2) Apache Commons Lang - The following NOTICE information applies: - Apache Commons Lang - Copyright 2001-2015 The Apache Software Foundation - - This product includes software from the Spring Framework, - under the Apache License 2.0 (see: StringUtils.containsWhitespace()) - - (ASLv2) Jackson JSON processor - The following NOTICE information applies: - # Jackson JSON processor - - Jackson is a high-performance, Free/Open Source JSON processing library. - It was originally written by Tatu Saloranta (tatu.saloranta@iki.fi), and has - been in development since 2007. - It is currently developed by a community of developers, as well as supported - commercially by FasterXML.com. - - ## Licensing - - Jackson core and extension components may licensed under different licenses. - To find the details that apply to this artifact see the accompanying LICENSE file. - For more information, including possible other licensing options, contact - FasterXML.com (http://fasterxml.com). - - ## Credits - - A list of contributors may be found from CREDITS file, which is included - in some artifacts (usually source distributions); but is always available - from the source code management (SCM) system project uses. \ No newline at end of file diff --git a/nifi-nar-bundles/nifi-tcp-bundle/nifi-tcp-processors/pom.xml b/nifi-nar-bundles/nifi-tcp-bundle/nifi-tcp-processors/pom.xml deleted file mode 100644 index 5f615f8237..0000000000 --- a/nifi-nar-bundles/nifi-tcp-bundle/nifi-tcp-processors/pom.xml +++ /dev/null @@ -1,41 +0,0 @@ - - - - 4.0.0 - - - org.apache.nifi - nifi-tcp-bundle - 2.0.0-SNAPSHOT - - - nifi-tcp-processors - jar - - - - org.apache.nifi - nifi-api - - - org.apache.nifi - nifi-utils - 2.0.0-SNAPSHOT - - - org.apache.nifi - nifi-mock - 2.0.0-SNAPSHOT - test - - - diff --git a/nifi-nar-bundles/nifi-tcp-bundle/nifi-tcp-processors/src/main/java/org/apache/nifi/processors/gettcp/AbstractSocketHandler.java b/nifi-nar-bundles/nifi-tcp-bundle/nifi-tcp-processors/src/main/java/org/apache/nifi/processors/gettcp/AbstractSocketHandler.java deleted file mode 100644 index 547ee79c64..0000000000 --- a/nifi-nar-bundles/nifi-tcp-bundle/nifi-tcp-processors/src/main/java/org/apache/nifi/processors/gettcp/AbstractSocketHandler.java +++ /dev/null @@ -1,275 +0,0 @@ -/* - * Copyright 2014 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.nifi.processors.gettcp; - -import java.io.IOException; -import java.net.InetSocketAddress; -import java.nio.ByteBuffer; -import java.nio.channels.NetworkChannel; -import java.nio.channels.SelectionKey; -import java.nio.channels.Selector; -import java.nio.channels.SocketChannel; -import java.util.HashSet; -import java.util.Iterator; -import java.util.Set; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; -import java.util.concurrent.atomic.AtomicBoolean; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -/** - * Base class to implement async TCP Client/Server components - * - */ -abstract class AbstractSocketHandler { - - final Logger logger = LoggerFactory.getLogger(this.getClass()); - - private final ByteBuffer readingBuffer; - - private final Runnable listenerTask; - - private volatile ExecutorService listenerTaskExecutor; - - final InetSocketAddress address; - - volatile NetworkChannel rootChannel; - - volatile Selector selector; - - private final AtomicBoolean isRunning; - - protected final byte endOfMessageByte; - - /** - * This constructor configures the address to bind to, the size of the buffer to use for reading, and - * the byte pattern to use for demarcating the end of a message. - * - * @param address the socket address - * @param readingBufferSize the buffer size - * @param endOfMessageByte the byte indicating EOM - */ - public AbstractSocketHandler(InetSocketAddress address, int readingBufferSize, byte endOfMessageByte) { - this.address = address; - this.listenerTask = new ListenerTask(); - this.readingBuffer = ByteBuffer.allocate(readingBufferSize); - this.isRunning = new AtomicBoolean(); - this.endOfMessageByte = endOfMessageByte; - } - - /** - * Once the handler is constructed this should be called to start the handler. Although - * this method is safe to be called by multiple threads, it should only be called once. - * - * @throws IllegalStateException if it fails to start listening on the port that is configured - * - */ - public void start() { - if (this.isRunning.compareAndSet(false, true)) { - try { - if (this.selector == null || !this.selector.isOpen()) { - this.selector = Selector.open(); - InetSocketAddress connectedAddress = this.connect(); - this.listenerTaskExecutor = Executors.newCachedThreadPool(); - this.listenerTaskExecutor.execute(this.listenerTask); - if (logger.isDebugEnabled()) { - logger.debug("Started listener for " + AbstractSocketHandler.this.getClass().getSimpleName()); - } - if (logger.isInfoEnabled()) { - logger.info("Successfully bound to " + connectedAddress); - } - } - } catch (Exception e) { - this.stop(); - throw new IllegalStateException("Failed to start " + this.getClass().getName(), e); - } - } - } - - /** - * This should be called to stop the handler from listening on the socket. Although it is recommended - * that this is called once, by a single thread, this method does protect itself from being called by more - * than one thread and more than one time. - * - */ - public void stop() { - if (this.isRunning.compareAndSet(true, false)) { - try { - if (this.selector != null && this.selector.isOpen()) { // since stop must be idempotent, we need to check if selector is open to avoid ClosedSelectorException - Set selectionKeys = new HashSet<>(this.selector.keys()); - for (SelectionKey key : selectionKeys) { - key.cancel(); - try { - key.channel().close(); - } catch (IOException e) { - logger.warn("Failure while closing channel", e); - } - } - try { - this.selector.close(); - } catch (Exception e) { - logger.warn("Failure while closinig selector", e); - } - logger.info(this.getClass().getSimpleName() + " is stopped listening on " + address); - } - } finally { - if (this.listenerTaskExecutor != null) { - this.listenerTaskExecutor.shutdown(); - } - } - } - } - - /** - * Checks if this component is running. - */ - public boolean isRunning() { - return this.isRunning.get(); - } - - /** - * This must be overridden by an implementing class and should establish the socket connection. - * - * @throws Exception if an exception occurs - */ - abstract InetSocketAddress connect() throws Exception; - - /** - * Will process the data received from the channel. - * - * @param selectionKey key for the channel the data came from - * @param buffer buffer of received data - * @throws IOException if there is a problem processing the data - */ - abstract void processData(SelectionKey selectionKey, ByteBuffer buffer) throws IOException; - - /** - * This does not perform any work at this time as all current implementations of this class - * provide the client side of the connection and thus do not accept connections. - * - * @param selectionKey the selection key - * @throws IOException if there is a problem - */ - void doAccept(SelectionKey selectionKey) throws IOException { - // noop - } - - /** - * Main listener task which will process delegate {@link SelectionKey} - * selected from the {@link Selector} to the appropriate processing method - * (e.g., accept, read, write etc.) - */ - private class ListenerTask implements Runnable { - @Override - public void run() { - try { - while (AbstractSocketHandler.this.rootChannel != null && AbstractSocketHandler.this.rootChannel.isOpen() && AbstractSocketHandler.this.selector.isOpen()) { - if (AbstractSocketHandler.this.selector.isOpen() && AbstractSocketHandler.this.selector.select(10) > 0) { - Iterator keys = AbstractSocketHandler.this.selector.selectedKeys().iterator(); - while (keys.hasNext()) { - SelectionKey selectionKey = keys.next(); - keys.remove(); - if (selectionKey.isValid()) { - if (selectionKey.isAcceptable()) { - this.accept(selectionKey); - } else if (selectionKey.isReadable()) { - this.read(selectionKey); - } else if (selectionKey.isConnectable()) { - this.connect(selectionKey); - } - } - } - } - } - } catch (Exception e) { - logger.error("Exception in socket listener loop", e); - } - - logger.debug("Exited Listener loop."); - AbstractSocketHandler.this.stop(); - } - - /** - * Accept the selectable channel - * - * @throws IOException in the event that something goes wrong accepting it - */ - private void accept(SelectionKey selectionKey) throws IOException { - AbstractSocketHandler.this.doAccept(selectionKey); - } - - /** - * This will connect the channel; if it is in a pending state then this will finish - * establishing the connection. Finally the socket handler is registered with this - * channel. - * - * @throws IOException if anything goes wrong during the connection establishment - * or registering of the handler - */ - private void connect(SelectionKey selectionKey) throws IOException { - SocketChannel clientChannel = (SocketChannel) selectionKey.channel(); - if (clientChannel.isConnectionPending()) { - clientChannel.finishConnect(); - } - clientChannel.register(AbstractSocketHandler.this.selector, SelectionKey.OP_READ); - } - - /** - * The main read loop which reads packets from the channel and sends - * them to implementations of - * {@link AbstractSocketHandler#processData(SelectionKey, ByteBuffer)}. - * So if a given implementation is a Server it is probably going to - * broadcast received message to all connected sockets (e.g., chat - * server). If such implementation is the Client, then it is most likely - * the end of the road where message is processed. - */ - private void read(SelectionKey selectionKey) throws IOException { - SocketChannel socketChannel = (SocketChannel) selectionKey.channel(); - - int count = -1; - boolean finished = false; - while (!finished && (count = socketChannel.read(AbstractSocketHandler.this.readingBuffer)) > 0){ - byte lastByte = AbstractSocketHandler.this.readingBuffer.get(AbstractSocketHandler.this.readingBuffer.position() - 1); - if (AbstractSocketHandler.this.readingBuffer.remaining() == 0 || lastByte == AbstractSocketHandler.this.endOfMessageByte) { - this.processBuffer(selectionKey); - if (lastByte == AbstractSocketHandler.this.endOfMessageByte) { - finished = true; - } - } - } - - if (count == -1) { - if (AbstractSocketHandler.this.readingBuffer.position() > 0) {// flush remainder, since nothing else is coming - this.processBuffer(selectionKey); - } - selectionKey.cancel(); - socketChannel.close(); - if (logger.isInfoEnabled()) { - logger.info("Connection closed by: " + socketChannel.socket()); - } - } - } - - private void processBuffer(SelectionKey selectionKey) throws IOException { - AbstractSocketHandler.this.readingBuffer.flip(); - byte[] message = new byte[AbstractSocketHandler.this.readingBuffer.limit()]; - AbstractSocketHandler.this.readingBuffer.get(message); - AbstractSocketHandler.this.processData(selectionKey, ByteBuffer.wrap(message)); - AbstractSocketHandler.this.readingBuffer.clear(); - } - } -} \ No newline at end of file diff --git a/nifi-nar-bundles/nifi-tcp-bundle/nifi-tcp-processors/src/main/java/org/apache/nifi/processors/gettcp/GetTCP.java b/nifi-nar-bundles/nifi-tcp-bundle/nifi-tcp-processors/src/main/java/org/apache/nifi/processors/gettcp/GetTCP.java deleted file mode 100644 index a746a21af6..0000000000 --- a/nifi-nar-bundles/nifi-tcp-bundle/nifi-tcp-processors/src/main/java/org/apache/nifi/processors/gettcp/GetTCP.java +++ /dev/null @@ -1,313 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.nifi.processors.gettcp; - -import org.apache.nifi.annotation.behavior.DynamicProperty; -import org.apache.nifi.annotation.behavior.InputRequirement; -import org.apache.nifi.annotation.behavior.SideEffectFree; -import org.apache.nifi.annotation.behavior.TriggerSerially; -import org.apache.nifi.annotation.behavior.WritesAttribute; -import org.apache.nifi.annotation.documentation.CapabilityDescription; -import org.apache.nifi.annotation.documentation.Tags; -import org.apache.nifi.annotation.lifecycle.OnScheduled; -import org.apache.nifi.annotation.lifecycle.OnStopped; -import org.apache.nifi.components.PropertyDescriptor; -import org.apache.nifi.expression.ExpressionLanguageScope; -import org.apache.nifi.flowfile.FlowFile; -import org.apache.nifi.processor.AbstractSessionFactoryProcessor; -import org.apache.nifi.processor.DataUnit; -import org.apache.nifi.processor.ProcessContext; -import org.apache.nifi.processor.ProcessSession; -import org.apache.nifi.processor.ProcessSessionFactory; -import org.apache.nifi.processor.Relationship; -import org.apache.nifi.processor.exception.ProcessException; -import org.apache.nifi.processor.io.OutputStreamCallback; -import org.apache.nifi.processor.util.StandardValidators; - -import java.io.IOException; -import java.io.OutputStream; -import java.net.InetSocketAddress; -import java.util.ArrayList; -import java.util.Collections; -import java.util.HashMap; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.concurrent.ScheduledThreadPoolExecutor; -import java.util.concurrent.TimeUnit; - -@TriggerSerially -@SideEffectFree -@Tags({"get", "fetch", "poll", "tcp", "ingest", "source", "input"}) -@InputRequirement(InputRequirement.Requirement.INPUT_FORBIDDEN) -@CapabilityDescription("Connects over TCP to the provided endpoint(s). Received data will be written as content to the FlowFile") -@DynamicProperty(name = "A FlowFile attribute to set", value = "The value to set it to", - description = "Sets a FlowFile attribute specified by the Dynamic Property's key with the value specified by the Dynamic Property's value") -@WritesAttribute(attribute = "source.endpoint", description = "The address of the source endpoint the message came from") -public class GetTCP extends AbstractSessionFactoryProcessor { - - private static String SOURCE_ENDPOINT_ATTRIBUTE = "source.endpoint"; - - public static final PropertyDescriptor ENDPOINT_LIST = new PropertyDescriptor.Builder() - .name("endpoint-list") - .displayName("Endpoint List") - .description("A comma delimited list of the endpoints to connect to. The format should be " + - ":. Only one server will be connected to at a time, the others " + - "will be used as fail overs.") - .required(true) - .addValidator(GetTCPUtils.ENDPOINT_VALIDATOR) - .build(); - - public static final PropertyDescriptor CONNECTION_ATTEMPT_COUNT = new PropertyDescriptor.Builder() - .name("connection-attempt-timeout") - .displayName("Connection Attempt Count") - .description("The number of times to try and establish a connection, before using a backup host if available." + - " This same attempt count would be used for a backup host as well.") - .required(true) - .addValidator(StandardValidators.POSITIVE_INTEGER_VALIDATOR) - .defaultValue("3") - .build(); - - public static final PropertyDescriptor RECONNECT_INTERVAL = new PropertyDescriptor.Builder() - .name("reconnect-interval") - .displayName("Reconnect interval") - .description("The number of seconds to wait before attempting to reconnect to the endpoint.") - .required(true) - .addValidator(StandardValidators.TIME_PERIOD_VALIDATOR) - .defaultValue("5 sec") - .build(); - - public static final PropertyDescriptor RECEIVE_BUFFER_SIZE = new PropertyDescriptor.Builder() - .name("receive-buffer-size") - .displayName("Receive Buffer Size") - .description("The size of the buffer to receive data in. Default 16384 (16MB).") - .required(false) - .defaultValue("16MB") - .addValidator(StandardValidators.DATA_SIZE_VALIDATOR) - .build(); - - public static final PropertyDescriptor END_OF_MESSAGE_BYTE = new PropertyDescriptor.Builder() - .name("end-of-message-byte") - .displayName("End of message delimiter byte") - .description("Byte value which denotes end of message. Must be specified as integer within " - + "the valid byte range (-128 thru 127). For example, '13' = Carriage return and '10' = New line. Default '13'.") - .required(true) - .defaultValue("13") - .addValidator(StandardValidators.createLongValidator(-128, 127, true)) - .build(); - - - public static final Relationship REL_SUCCESS = new Relationship.Builder() - .name("Success") - .description("The relationship that all sucessful messages from the endpoint will be sent to.") - .build(); - - public static final Relationship REL_PARTIAL = new Relationship.Builder() - .name("Partial") - .description("The relationship that all incomplete messages from the endpoint will be sent to. " - + "Incomplete message is the message that doesn't end with 'End of message delimiter byte'. " - + "This can happen when 'Receive Buffer Size' is smaller then the incoming message. If that happens that " - + "the subsequent message that completes the previous incomplete message will also end up in this " - + "relationship, after which subsequent 'complete' messages will go to 'success'.") - .build(); - - private final static List DESCRIPTORS; - - private final static Set RELATIONSHIPS; - - /* - * Will ensure that the list of property descriptors is build only once. - * Will also create a Set of relationships - */ - static { - List _propertyDescriptors = new ArrayList<>(); - _propertyDescriptors.add(ENDPOINT_LIST); - _propertyDescriptors.add(CONNECTION_ATTEMPT_COUNT); - _propertyDescriptors.add(RECONNECT_INTERVAL); - _propertyDescriptors.add(RECEIVE_BUFFER_SIZE); - _propertyDescriptors.add(END_OF_MESSAGE_BYTE); - - DESCRIPTORS = Collections.unmodifiableList(_propertyDescriptors); - - Set _relationships = new HashSet<>(); - _relationships.add(REL_SUCCESS); - _relationships.add(REL_PARTIAL); - RELATIONSHIPS = Collections.unmodifiableSet(_relationships); - } - - private final Map dynamicAttributes = new HashMap<>(); - - private final Map liveTcpClients = new HashMap<>(); - - private volatile NiFiDelegatingMessageHandler delegatingMessageHandler; - - private volatile ScheduledThreadPoolExecutor clientScheduler; - - private volatile String originalServerAddressList; - - private volatile int receiveBufferSize; - - private volatile int connectionAttemptCount; - - private volatile long reconnectInterval; - - private volatile byte endOfMessageByte; - - @Override - public Set getRelationships() { - return RELATIONSHIPS; - } - - @Override - public final List getSupportedPropertyDescriptors() { - return DESCRIPTORS; - } - - @OnScheduled - public void onScheduled(final ProcessContext context) throws ProcessException { - this.receiveBufferSize = context.getProperty(RECEIVE_BUFFER_SIZE).asDataSize(DataUnit.B).intValue(); - this.originalServerAddressList = context.getProperty(ENDPOINT_LIST).getValue(); - this.endOfMessageByte = ((byte) context.getProperty(END_OF_MESSAGE_BYTE).asInteger().intValue()); - this.connectionAttemptCount = context.getProperty(CONNECTION_ATTEMPT_COUNT).asInteger(); - this.reconnectInterval = context.getProperty(RECONNECT_INTERVAL).asTimePeriod(TimeUnit.MILLISECONDS); - - this.clientScheduler = new ScheduledThreadPoolExecutor(originalServerAddressList.split(",").length + 1); - this.clientScheduler.setKeepAliveTime(10, TimeUnit.SECONDS); - this.clientScheduler.allowCoreThreadTimeOut(true); - - for (final Map.Entry entry : context.getProperties().entrySet()) { - final PropertyDescriptor descriptor = entry.getKey(); - if (descriptor.isDynamic()) { - this.dynamicAttributes.put(descriptor.getName(), entry.getValue()); - } - } - } - - @Override - public void onTrigger(ProcessContext context, ProcessSessionFactory sessionFactory) throws ProcessException { - if (this.delegatingMessageHandler == null) { - this.delegatingMessageHandler = new NiFiDelegatingMessageHandler(sessionFactory); - } - this.run(context); - context.yield(); - } - - @OnStopped - public void tearDown() throws ProcessException { - for (ReceivingClient client : this.liveTcpClients.values()) { - try { - client.stop(); - } catch (Exception e) { - this.getLogger().warn("Failure while stopping client '" + client + "'", e); - } - } - this.liveTcpClients.clear(); - this.clientScheduler.shutdown(); - try { - if (!this.clientScheduler.awaitTermination(10000, TimeUnit.MILLISECONDS)) { - this.getLogger().info("Failed to stop client scheduler in 10 sec. Terminating"); - this.clientScheduler.shutdownNow(); - } - } catch (InterruptedException e) { - Thread.currentThread().interrupt(); - } - this.getLogger().info("Processor has successfully shut down"); - } - - @Override - protected PropertyDescriptor getSupportedDynamicPropertyDescriptor(final String propertyDescriptorName) { - return new PropertyDescriptor.Builder() - .required(false) - .name(propertyDescriptorName) - .addValidator(StandardValidators.NON_EMPTY_VALIDATOR) - .dynamic(true) - .expressionLanguageSupported(ExpressionLanguageScope.VARIABLE_REGISTRY) - .build(); - } - - private void run(ProcessContext context) { - String[] serverAddresses = this.originalServerAddressList.split(","); - for (String hostPortPair : serverAddresses) { - ReceivingClient client; - if (!this.liveTcpClients.containsKey(hostPortPair)) { - String[] hostAndPort = hostPortPair.split(":"); - InetSocketAddress address = new InetSocketAddress(hostAndPort[0], Integer.parseInt(hostAndPort[1])); - client = new ReceivingClient(address, this.clientScheduler, this.receiveBufferSize, this.endOfMessageByte); - client.setReconnectAttempts(this.connectionAttemptCount); - client.setDelayMillisBeforeReconnect(this.reconnectInterval); - client.setMessageHandler(this.delegatingMessageHandler); - this.liveTcpClients.put(hostPortPair, client); - this.startClient(client); - } else { - client = this.liveTcpClients.get(hostPortPair); - if (!client.isRunning()) { - client.stop(); // primarily for cleanup in the event of abnormal termination - this.startClient(client); - } - } - } - } - - private void startClient(ReceivingClient client) { - this.clientScheduler.execute(new Runnable() { - @Override - public void run() { - try { - client.start(); - } catch (Exception e) { - getLogger().warn("Failed to start listening client. Will attempt to start on another trigger cycle.", e); - } - } - }); - } - - /** - * This handles taking the message that has been received off the wire and writing it to the - * content of a flowfile. If only a partial message is received then the flowfile is sent to - * the Partial relationship. If a full message is received then it is sent to the Success relationship. - */ - private class NiFiDelegatingMessageHandler implements MessageHandler { - private final ProcessSessionFactory sessionFactory; - - NiFiDelegatingMessageHandler(ProcessSessionFactory sessionFactory) { - this.sessionFactory = sessionFactory; - } - - @Override - public void handle(InetSocketAddress sourceAddress, byte[] message, boolean partialMessage) { - ProcessSession session = this.sessionFactory.createSession(); - FlowFile flowFile = session.create(); - flowFile = session.write(flowFile, new OutputStreamCallback() { - @Override - public void process(OutputStream out) throws IOException { - out.write(message); - } - }); - flowFile = session.putAttribute(flowFile, SOURCE_ENDPOINT_ATTRIBUTE, sourceAddress.toString()); - if (!GetTCP.this.dynamicAttributes.isEmpty()) { - flowFile = session.putAllAttributes(flowFile, GetTCP.this.dynamicAttributes); - } - if (partialMessage) { - session.transfer(flowFile, REL_PARTIAL); - } else { - session.transfer(flowFile, REL_SUCCESS); - } - session.commitAsync(); - } - } -} diff --git a/nifi-nar-bundles/nifi-tcp-bundle/nifi-tcp-processors/src/main/java/org/apache/nifi/processors/gettcp/GetTCPUtils.java b/nifi-nar-bundles/nifi-tcp-bundle/nifi-tcp-processors/src/main/java/org/apache/nifi/processors/gettcp/GetTCPUtils.java deleted file mode 100644 index 845dda87d8..0000000000 --- a/nifi-nar-bundles/nifi-tcp-bundle/nifi-tcp-processors/src/main/java/org/apache/nifi/processors/gettcp/GetTCPUtils.java +++ /dev/null @@ -1,85 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.nifi.processors.gettcp; - -import java.util.regex.Matcher; -import java.util.regex.Pattern; - -import org.apache.nifi.components.ValidationContext; -import org.apache.nifi.components.ValidationResult; -import org.apache.nifi.components.Validator; -import org.apache.nifi.processor.util.StandardValidators; - -class GetTCPUtils { - - private static final Pattern validIpAddressRegex = Pattern.compile( - "^(([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])\\.){3}([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])$"); - - private static final Pattern validHostnameRegex = Pattern.compile( - "^(([a-zA-Z0-9]|[a-zA-Z0-9][a-zA-Z0-9\\-]*[a-zA-Z0-9])\\.)*([A-Za-z0-9]|[A-Za-z0-9][A-Za-z0-9\\-]*[A-Za-z0-9])$"); - - private static final Pattern looksLikeIpRegex = Pattern.compile("^(\\d+)\\.(\\d+)\\.(\\d+)\\.(\\d+)$"); - - public static final Validator ENDPOINT_VALIDATOR = new Validator() { - @Override - public ValidationResult validate(final String subject, final String value, final ValidationContext context) { - if (null == value || value.isEmpty()) { - return new ValidationResult.Builder().subject(subject).input(value).valid(false) - .explanation(subject + " cannot be empty").build(); - } - // The format should be :{,:} - // first split on , - final String[] hostPortPairs = value.split(","); - boolean validHostPortPairs = true; - String reason = ""; - String offendingSubject = subject; - - if (0 == hostPortPairs.length) { - return new ValidationResult.Builder().subject(subject).input(value).valid(false) - .explanation(offendingSubject + " cannot be empty").build(); - } - - for (int i = 0; i < hostPortPairs.length && validHostPortPairs; i++) { - String[] parts = hostPortPairs[i].split(":"); - - if (parts.length != 2) { - validHostPortPairs = false; - reason = " of malformed URL '" + hostPortPairs[i] + "'"; - } else { - Matcher validHost = validHostnameRegex.matcher(parts[0]); - Matcher validIp = validIpAddressRegex.matcher(parts[0]); - Matcher looksLikeValidIp = looksLikeIpRegex.matcher(parts[0]); - if (!validHost.find()) { - validHostPortPairs = false; - reason = " it contains invalid characters '" + parts[0] + "'"; - } else if (looksLikeValidIp.find() && !validIp.find()) { - validHostPortPairs = false; - reason = " it appears to be represented as an IP address which is out of legal range '" + parts[0] + "'"; - } - ValidationResult result = StandardValidators.PORT_VALIDATOR.validate(parts[1], parts[1], context); - if (!result.isValid()) { - validHostPortPairs = false; - reason = result.getExplanation(); - } - } - } - - return new ValidationResult.Builder().subject(offendingSubject).input(value).explanation(reason) - .valid(validHostPortPairs).build(); - } - }; -} diff --git a/nifi-nar-bundles/nifi-tcp-bundle/nifi-tcp-processors/src/main/java/org/apache/nifi/processors/gettcp/MessageHandler.java b/nifi-nar-bundles/nifi-tcp-bundle/nifi-tcp-processors/src/main/java/org/apache/nifi/processors/gettcp/MessageHandler.java deleted file mode 100644 index c6a6fea5a0..0000000000 --- a/nifi-nar-bundles/nifi-tcp-bundle/nifi-tcp-processors/src/main/java/org/apache/nifi/processors/gettcp/MessageHandler.java +++ /dev/null @@ -1,24 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.nifi.processors.gettcp; - -import java.net.InetSocketAddress; - -public interface MessageHandler { - - void handle(InetSocketAddress sourceAddress, byte[] message, boolean partialMessage); -} diff --git a/nifi-nar-bundles/nifi-tcp-bundle/nifi-tcp-processors/src/main/java/org/apache/nifi/processors/gettcp/ReceivingClient.java b/nifi-nar-bundles/nifi-tcp-bundle/nifi-tcp-processors/src/main/java/org/apache/nifi/processors/gettcp/ReceivingClient.java deleted file mode 100644 index 1927b9934e..0000000000 --- a/nifi-nar-bundles/nifi-tcp-bundle/nifi-tcp-processors/src/main/java/org/apache/nifi/processors/gettcp/ReceivingClient.java +++ /dev/null @@ -1,149 +0,0 @@ -/* - * Copyright 2014 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.nifi.processors.gettcp; - -import java.io.IOException; -import java.net.InetSocketAddress; -import java.nio.ByteBuffer; -import java.nio.channels.SelectionKey; -import java.nio.channels.SocketChannel; -import java.util.concurrent.CountDownLatch; -import java.util.concurrent.ScheduledExecutorService; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicInteger; -import java.util.concurrent.atomic.AtomicReference; - -/** - * Implementation of receiving network client. - */ -class ReceivingClient extends AbstractSocketHandler { - - private final ScheduledExecutorService connectionScheduler; - - private volatile int reconnectAttempts; - - private volatile long delayMillisBeforeReconnect; - - private volatile MessageHandler messageHandler; - - private volatile InetSocketAddress connectedAddress; - - public ReceivingClient(InetSocketAddress address, ScheduledExecutorService connectionScheduler, int readingBufferSize, byte endOfMessageByte) { - super(address, readingBufferSize, endOfMessageByte); - this.connectionScheduler = connectionScheduler; - } - - public void setReconnectAttempts(int reconnectAttempts) { - this.reconnectAttempts = reconnectAttempts; - } - - public void setDelayMillisBeforeReconnect(long delayMillisBeforeReconnect) { - this.delayMillisBeforeReconnect = delayMillisBeforeReconnect; - } - - public void setMessageHandler(MessageHandler messageHandler) { - this.messageHandler = messageHandler; - } - - /** - * Connects to the endpoint specified and if that fails, will attempt to connect to the - * secondary endpoint up to the number of reconnection attempts (inclusive) using the - * configured delay between attempts. - */ - @Override - InetSocketAddress connect() throws Exception { - CountDownLatch latch = new CountDownLatch(1); - AtomicInteger attempt = new AtomicInteger(); - AtomicReference connectionError = new AtomicReference(); - this.connectionScheduler.execute(new Runnable() { - @Override - public void run() { - try { - rootChannel = doConnect(address); - latch.countDown(); - connectedAddress = address; - } catch (Exception e) { - if (logger.isInfoEnabled()) { - logger.info("Failed to connect to primary endpoint '" + address + "'."); - } - if (attempt.incrementAndGet() <= reconnectAttempts) { - if (logger.isInfoEnabled()) { - logger.info("Will attempt to reconnect to '" + address + "'."); - } - connectionScheduler.schedule(this, delayMillisBeforeReconnect, TimeUnit.MILLISECONDS); - } else { - connectionError.set(e); - logger.error("Failed to connect to secondary endpoint."); - latch.countDown(); - } - } - } - }); - - try { - boolean finishedTask = latch.await(this.reconnectAttempts * delayMillisBeforeReconnect + 2000, TimeUnit.MILLISECONDS); - if (finishedTask){ - if (connectionError.get() != null) { - throw connectionError.get(); - } - } else { - logger.error("Exceeded wait time to connect. Possible deadlock, please report!. Interrupting."); // should never happen! - } - } catch (InterruptedException e) { - Thread.currentThread().interrupt(); - throw new IllegalStateException("Current thread is interrupted"); - } - return this.connectedAddress; - } - - private SocketChannel doConnect(InetSocketAddress addressToConnect) throws IOException { - SocketChannel channel = SocketChannel.open(); - if (channel.connect(addressToConnect)) { - channel.configureBlocking(false); - channel.register(this.selector, SelectionKey.OP_READ); - } else { - throw new IllegalStateException("Failed to connect to Server at: " + addressToConnect); - } - return channel; - } - - /** - * Process the message that has arrived off the wire. - */ - @Override - void processData(SelectionKey selectionKey, ByteBuffer messageBuffer) throws IOException { - byte[] message = new byte[messageBuffer.limit()]; - logger.debug("Received message(size=" + message.length + ")"); - messageBuffer.get(message); - byte lastByteValue = message[message.length - 1]; - boolean partialMessage = false; - if (lastByteValue != this.endOfMessageByte) { - partialMessage = true; - selectionKey.attach(1); - } else { - Integer wasLastPartial = (Integer) selectionKey.attachment(); - if (wasLastPartial != null) { - if (wasLastPartial.intValue() == 1) { - partialMessage = true; - selectionKey.attach(0); - } - } - } - if (this.messageHandler != null) { - this.messageHandler.handle(this.connectedAddress, message, partialMessage); - } - } -} \ No newline at end of file diff --git a/nifi-nar-bundles/nifi-tcp-bundle/nifi-tcp-processors/src/main/resources/META-INF/services/org.apache.nifi.processor.Processor b/nifi-nar-bundles/nifi-tcp-bundle/nifi-tcp-processors/src/main/resources/META-INF/services/org.apache.nifi.processor.Processor deleted file mode 100644 index a8368939bc..0000000000 --- a/nifi-nar-bundles/nifi-tcp-bundle/nifi-tcp-processors/src/main/resources/META-INF/services/org.apache.nifi.processor.Processor +++ /dev/null @@ -1,15 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -org.apache.nifi.processors.gettcp.GetTCP \ No newline at end of file diff --git a/nifi-nar-bundles/nifi-tcp-bundle/nifi-tcp-processors/src/test/java/org/apache/nifi/processors/gettcp/ReceivingClientTest.java b/nifi-nar-bundles/nifi-tcp-bundle/nifi-tcp-processors/src/test/java/org/apache/nifi/processors/gettcp/ReceivingClientTest.java deleted file mode 100644 index bc8ea3b98f..0000000000 --- a/nifi-nar-bundles/nifi-tcp-bundle/nifi-tcp-processors/src/test/java/org/apache/nifi/processors/gettcp/ReceivingClientTest.java +++ /dev/null @@ -1,227 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.nifi.processors.gettcp; - -import org.junit.jupiter.api.AfterEach; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.condition.EnabledIfSystemProperty; - -import java.io.IOException; -import java.io.PrintWriter; -import java.net.InetSocketAddress; -import java.net.ServerSocket; -import java.net.Socket; -import java.nio.charset.StandardCharsets; -import java.util.ArrayList; -import java.util.List; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; -import java.util.concurrent.ScheduledExecutorService; - -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertFalse; -import static org.junit.jupiter.api.Assertions.assertTrue; - -@EnabledIfSystemProperty( - named = "nifi.test.unstable", - matches = "true", - disabledReason = "Ignored for full build due to artificial delays given the multi-threaded nature of most of " + - "the tests. Please un-Ignore and run when working on changes" -) -public class ReceivingClientTest { - - private final static byte EOM = '\r'; - - private ScheduledExecutorService scheduler; - - @BeforeEach - public void before() { - this.scheduler = Executors.newScheduledThreadPool(1); - } - - @AfterEach - public void after() { - this.scheduler.shutdownNow(); - } - - @Test - public void validateSuccessfullConnectionAndCommunication() throws Exception { - int port = this.availablePort(); - String msgToSend = "Hello from validateSuccessfullConnectionAndCommunication"; - InetSocketAddress address = new InetSocketAddress(port); - Server server = new Server(address, 1024, EOM); - server.start(); - - ReceivingClient client = new ReceivingClient(address, this.scheduler, 1024, EOM); - StringBuilder stringBuilder = new StringBuilder(); - client.setMessageHandler((fromAddress, message, partialMessage) -> stringBuilder.append(new String(message, StandardCharsets.UTF_8))); - client.start(); - assertTrue(client.isRunning()); - - this.sendToSocket(address, msgToSend); - Thread.sleep(200); - assertEquals("", stringBuilder.toString()); - this.sendToSocket(address, "\r"); - Thread.sleep(200); - assertEquals(msgToSend + "\r", stringBuilder.toString()); - - client.stop(); - server.stop(); - assertFalse(client.isRunning()); - assertFalse(server.isRunning()); - } - - @Test - public void validateSuccessfullConnectionAndCommunicationWithClientBufferSmallerThenMessage() throws Exception { - int port = this.availablePort(); - String msgToSend = "Hello from validateSuccessfullConnectionAndCommunicationWithClientBufferSmallerThenMessage"; - InetSocketAddress address = new InetSocketAddress(port); - Server server = new Server(address, 1024, EOM); - server.start(); - - ReceivingClient client = new ReceivingClient(address, this.scheduler, 64, EOM); - List messages = new ArrayList<>(); - client.setMessageHandler((fromAddress, message, partialMessage) -> messages.add(new String(message, StandardCharsets.UTF_8))); - client.start(); - assertTrue(client.isRunning()); - - this.sendToSocket(address, msgToSend); - this.sendToSocket(address, "\r"); - Thread.sleep(200); - assertEquals("Hello from validateSuccessfullConnectionAndCommunicationWithClie", messages.get(0)); - assertEquals("ntBufferSmallerThenMessage\r", messages.get(1)); - - client.stop(); - server.stop(); - assertFalse(client.isRunning()); - assertFalse(server.isRunning()); - } - - @Test - public void validateMessageSendBeforeAfterClientConnectDisconnectNoEndOfMessageByte() throws Exception { - int port = this.availablePort(); - String msgToSend = "Hello from validateMessageSendBeforeAfterClientConnectDisconnectNoEndOfMessageByte"; - InetSocketAddress address = new InetSocketAddress(port); - Server server = new Server(address, 1024, EOM); - server.start(); - this.sendToSocket(address, "foo"); // validates no unexpected errors - - ReceivingClient client = new ReceivingClient(address, this.scheduler, 30, EOM); - List messages = new ArrayList<>(); - client.setMessageHandler((fromAddress, message, partialMessage) -> messages.add(new String(message, StandardCharsets.UTF_8))); - client.start(); - assertTrue(client.isRunning()); - - this.sendToSocket(address, msgToSend); - Thread.sleep(200); - assertEquals(2, messages.size()); - assertEquals("Hello from validateMessageSend", messages.get(0)); - assertEquals("BeforeAfterClientConnectDiscon", messages.get(1)); - messages.clear(); - - client.stop(); - this.sendToSocket(address, msgToSend); - Thread.sleep(200); - assertEquals(0, messages.size()); - - this.sendToSocket(address, msgToSend); - - server.stop(); - assertFalse(client.isRunning()); - assertFalse(server.isRunning()); - } - - @Test - public void validateReconnectDuringReceive() throws Exception { - int port = this.availablePort(); - String msgToSend = "Hello from validateReconnectDuringReceive\r"; - InetSocketAddress addressMain = new InetSocketAddress(port); - Server server = new Server(addressMain, 1024, EOM); - server.start(); - - ExecutorService sendingExecutor = Executors.newSingleThreadExecutor(); - - ReceivingClient client = new ReceivingClient(addressMain, this.scheduler, 1024, EOM); - client.setReconnectAttempts(10); - client.setDelayMillisBeforeReconnect(1000); - client.setMessageHandler((fromAddress, message, partialMessage) -> System.out.println(new String(message))); - client.start(); - assertTrue(client.isRunning()); - - sendingExecutor.execute(new Runnable() { - @Override - public void run() { - for (int i = 0; i < 10; i++) { - try { - sendToSocket(addressMain, msgToSend); - Thread.sleep(100); - } catch (Exception e) { - try { - Thread.sleep(1000); - } catch (Exception ex) { - // ignore - } - } - - } - } - }); - - Thread.sleep(500); - server.stop(); - - Thread.sleep(500); - - server.start(); - Thread.sleep(1000); - - client.stop(); - server.stop(); - - assertFalse(client.isRunning()); - assertFalse(server.isRunning()); - } - - private void sendToSocket(InetSocketAddress address, String message) throws Exception { - Socket socket = new Socket(address.getAddress(), address.getPort()); - PrintWriter out = new PrintWriter(socket.getOutputStream(), true); - out.write(message); - out.flush(); - socket.close(); - } - - /** - * Will determine the available port used by test server. - */ - private int availablePort() { - ServerSocket s = null; - try { - s = new ServerSocket(0); - s.setReuseAddress(true); - return s.getLocalPort(); - } catch (Exception e) { - throw new IllegalStateException("Failed to discover available port.", e); - } finally { - try { - s.close(); - } catch (IOException e) { - // ignore - } - } - } -} diff --git a/nifi-nar-bundles/nifi-tcp-bundle/nifi-tcp-processors/src/test/java/org/apache/nifi/processors/gettcp/Server.java b/nifi-nar-bundles/nifi-tcp-bundle/nifi-tcp-processors/src/test/java/org/apache/nifi/processors/gettcp/Server.java deleted file mode 100644 index e6a4ca08a2..0000000000 --- a/nifi-nar-bundles/nifi-tcp-bundle/nifi-tcp-processors/src/test/java/org/apache/nifi/processors/gettcp/Server.java +++ /dev/null @@ -1,91 +0,0 @@ -/* - * Copyright 2014 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.nifi.processors.gettcp; - -import java.io.IOException; -import java.net.InetSocketAddress; -import java.nio.ByteBuffer; -import java.nio.channels.SelectionKey; -import java.nio.channels.ServerSocketChannel; -import java.nio.channels.SocketChannel; - -/** - * - */ -public class Server extends AbstractSocketHandler { - - public static void main(String[] args) throws Exception { - InetSocketAddress address = new InetSocketAddress(9999); - Server server = new Server(address, 4096, (byte) '\n'); - server.start(); - System.in.read(); - } - - /** - * @param address the socket address - * @param readingBufferSize the buffer size - * @param endOfMessageByte the byte indicating the EOM - */ - public Server(InetSocketAddress address, int readingBufferSize, byte endOfMessageByte) { - super(address, readingBufferSize, endOfMessageByte); - } - - /** - * - */ - @Override - InetSocketAddress connect() throws IOException { - this.rootChannel = ServerSocketChannel.open(); - ServerSocketChannel channel = (ServerSocketChannel) rootChannel; - channel.configureBlocking(false); - channel.socket().bind(this.address); - channel.register(this.selector, SelectionKey.OP_ACCEPT); - return this.address; - } - - /** - * - */ - @Override - void doAccept(SelectionKey selectionKey) throws IOException { - ServerSocketChannel serverChannel = (ServerSocketChannel) selectionKey.channel(); - SocketChannel channel = serverChannel.accept(); - if (logger.isInfoEnabled()) { - logger.info("Accepted connection from: " + channel.socket()); - } - channel.configureBlocking(false); - channel.register(this.selector, SelectionKey.OP_READ); - } - - /** - * Unlike the client side the read on the server will happen using receiving - * thread. - */ - @Override - void processData(SelectionKey selectionKey, ByteBuffer readBuffer) throws IOException { - logger.info("Server received message of " + readBuffer.limit() + " bytes in size and will delegate to all registered clients."); - for (SelectionKey key : selector.keys()) { - if (key.isValid() && key.channel() instanceof SocketChannel && !selectionKey.equals(key)) { - if (logger.isDebugEnabled()) { - logger.debug("Distributing incoming message to " + key.channel()); - } - SocketChannel sch = (SocketChannel) key.channel(); - sch.write(readBuffer); - readBuffer.rewind(); - } - } - } -} \ No newline at end of file diff --git a/nifi-nar-bundles/nifi-tcp-bundle/nifi-tcp-processors/src/test/java/org/apache/nifi/processors/gettcp/TestGetTCP.java b/nifi-nar-bundles/nifi-tcp-bundle/nifi-tcp-processors/src/test/java/org/apache/nifi/processors/gettcp/TestGetTCP.java deleted file mode 100644 index 68ad79f7e4..0000000000 --- a/nifi-nar-bundles/nifi-tcp-bundle/nifi-tcp-processors/src/test/java/org/apache/nifi/processors/gettcp/TestGetTCP.java +++ /dev/null @@ -1,153 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.nifi.processors.gettcp; - -import org.apache.nifi.util.TestRunner; -import org.apache.nifi.util.TestRunners; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.condition.EnabledIfSystemProperty; - -import java.io.IOException; -import java.io.PrintWriter; -import java.net.InetSocketAddress; -import java.net.ServerSocket; -import java.net.Socket; - -public final class TestGetTCP { - private TestRunner testRunner; - private GetTCP processor; - - @BeforeEach - public void setup() { - processor = new GetTCP(); - testRunner = TestRunners.newTestRunner(processor); - } - - @Test - public void testSelectPropertiesValidation() { - testRunner.setProperty(GetTCP.ENDPOINT_LIST, "!@;;*blah:9999"); - testRunner.assertNotValid(); - testRunner.setProperty(GetTCP.ENDPOINT_LIST, "localhost:9999"); - testRunner.assertValid(); - testRunner.setProperty(GetTCP.ENDPOINT_LIST, "localhost:-1"); - testRunner.assertNotValid(); - testRunner.setProperty(GetTCP.ENDPOINT_LIST, ","); - testRunner.assertNotValid(); - testRunner.setProperty(GetTCP.ENDPOINT_LIST, ",localhost:9999"); - testRunner.assertNotValid(); - testRunner.setProperty(GetTCP.ENDPOINT_LIST, "999,localhost:123"); - testRunner.assertNotValid(); - testRunner.setProperty(GetTCP.ENDPOINT_LIST, "localhost:abc_port"); - testRunner.assertNotValid(); - testRunner.setProperty(GetTCP.ENDPOINT_LIST, "localhost:9999;localhost:1234"); - testRunner.assertNotValid(); - testRunner.setProperty(GetTCP.ENDPOINT_LIST, "localhost:9999,localhost:1234"); - testRunner.assertValid(); - testRunner.setProperty(GetTCP.END_OF_MESSAGE_BYTE, "354"); - testRunner.assertNotValid(); - testRunner.setProperty(GetTCP.END_OF_MESSAGE_BYTE, "13"); - testRunner.assertValid(); - } - - @Test - public void testDynamicProperty() { - testRunner.setProperty(GetTCP.ENDPOINT_LIST, "localhost:9999,localhost:1234"); - testRunner.setProperty("MyCustomProperty", "abc"); - testRunner.assertValid(); - } - - @EnabledIfSystemProperty( - named = "nifi.test.unstable", - matches = "true", - disabledReason = "test is brittle as depends on timing - not reliable across systems, in parallel builds, etc.." - ) - @Test - public void testSuccessInteraction() throws Exception { - int port = this.availablePort(); - Server server = setupTCPServer(port); - testRunner.setProperty(GetTCP.ENDPOINT_LIST, "localhost:" + port); - testRunner.run(1000, false); - this.sendToSocket(new InetSocketAddress(port), "Hello\r"); - Thread.sleep(200); - testRunner.assertAllFlowFilesTransferred(GetTCP.REL_SUCCESS, 1); - testRunner.clearTransferState(); - testRunner.shutdown(); - server.stop(); - } - - @EnabledIfSystemProperty( - named = "nifi.test.unstable", - matches = "true", - disabledReason = "test is brittle as depends on timing - not reliable across systems, in parallel builds, etc.." - ) - @Test - public void testPartialInteraction() throws Exception { - int port = this.availablePort(); - Server server = setupTCPServer(port); - testRunner.setProperty(GetTCP.ENDPOINT_LIST, "localhost:" + port); - testRunner.setProperty(GetTCP.RECEIVE_BUFFER_SIZE, "2B"); - testRunner.run(1000, false); - this.sendToSocket(new InetSocketAddress(port), "Hello\r"); - Thread.sleep(300); - testRunner.assertAllFlowFilesTransferred(GetTCP.REL_PARTIAL, 3); - testRunner.clearTransferState(); - - this.sendToSocket(new InetSocketAddress(port), "H\r"); - Thread.sleep(300); - testRunner.assertAllFlowFilesTransferred(GetTCP.REL_SUCCESS, 1); - testRunner.clearTransferState(); - testRunner.shutdown(); - server.stop(); - } - - private Server setupTCPServer(int port) { - InetSocketAddress address = new InetSocketAddress(port); - Server server = new Server(address, 1024, (byte) '\r'); - server.start(); - return server; - } - - private void sendToSocket(InetSocketAddress address, String message) throws Exception { - Socket socket = new Socket(address.getAddress(), address.getPort()); - PrintWriter out = new PrintWriter(socket.getOutputStream(), true); - out.write(message); - out.flush(); - socket.close(); - } - - /** - * Will determine the available port used by test server. - */ - private int availablePort() { - ServerSocket s = null; - try { - s = new ServerSocket(0); - s.setReuseAddress(true); - return s.getLocalPort(); - } catch (Exception e) { - throw new IllegalStateException("Failed to discover available port.", e); - } finally { - try { - s.close(); - } catch (IOException e) { - // ignore - } - } - } -} diff --git a/nifi-nar-bundles/nifi-tcp-bundle/nifi-tcp-processors/src/test/resources/log4j.properties b/nifi-nar-bundles/nifi-tcp-bundle/nifi-tcp-processors/src/test/resources/log4j.properties deleted file mode 100644 index 5a5aa8e567..0000000000 --- a/nifi-nar-bundles/nifi-tcp-bundle/nifi-tcp-processors/src/test/resources/log4j.properties +++ /dev/null @@ -1,25 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -log4j.rootCategory=DEBUG, stdout - -log4j.appender.stdout=org.apache.log4j.ConsoleAppender -log4j.appender.stdout.layout=org.apache.log4j.PatternLayout -log4j.appender.stdout.layout.ConversionPattern=%d{ABSOLUTE} %5p %t %c{2}:%L - %m%n - -#log4j.category.com.hortonworks.minicluster=INFO -log4j.category.org.apache.tez.runtime=WARN diff --git a/nifi-nar-bundles/nifi-tcp-bundle/pom.xml b/nifi-nar-bundles/nifi-tcp-bundle/pom.xml deleted file mode 100644 index b20756f11f..0000000000 --- a/nifi-nar-bundles/nifi-tcp-bundle/pom.xml +++ /dev/null @@ -1,33 +0,0 @@ - - - - 4.0.0 - - - org.apache.nifi - nifi-nar-bundles - 2.0.0-SNAPSHOT - - - - nifi-tcp-bundle - pom - - - nifi-tcp-processors - nifi-tcp-nar - - diff --git a/nifi-nar-bundles/pom.xml b/nifi-nar-bundles/pom.xml index abdbd7f4ab..50f90bb296 100755 --- a/nifi-nar-bundles/pom.xml +++ b/nifi-nar-bundles/pom.xml @@ -54,8 +54,6 @@ nifi-ldap-iaa-providers-bundle nifi-kerberos-iaa-providers-bundle nifi-single-user-iaa-providers-bundle - nifi-riemann-bundle - nifi-html-bundle nifi-scripting-bundle nifi-elasticsearch-bundle nifi-amqp-bundle @@ -77,25 +75,21 @@ nifi-email-bundle nifi-groovyx-bundle nifi-websocket-bundle - nifi-tcp-bundle nifi-gcp-bundle nifi-registry-bundle nifi-stateful-analysis-bundle nifi-poi-bundle nifi-cdc - nifi-cybersecurity-bundle nifi-parquet-bundle nifi-extension-utils nifi-ranger-bundle nifi-redis-bundle - nifi-metrics-reporting-bundle nifi-spark-bundle nifi-atlas-bundle nifi-jolt-record-bundle nifi-network-bundle nifi-prometheus-bundle nifi-sql-reporting-bundle - nifi-rules-action-handler-bundle nifi-hazelcast-bundle nifi-accumulo-bundle nifi-asn1-bundle