Apache
/
nifi
mirror of https://github.com/apache/nifi.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

NIFI-12112 This closes #7778. Removed Unmaintained Components and Modules 

- nifi-cybersecurity-bundle
- nifi-html-bundle
- nifi-hl7-bundle
- nifi-metrics-reporting-bundle
- nifi-riemann-bundle
- nifi-tcp-bundle
- nifi-rules-action-handler-bundle
- nifi-rules-engine-service-api

Signed-off-by: Joseph Witt <joewitt@apache.org>
This commit is contained in:
exceptionfactory 2023-09-21 21:12:54 -05:00 committed by Joseph Witt
parent 713be8b539
commit d354c88f57
No known key found for this signature in database
GPG Key ID: 9093BF854F811A1A
126 changed files with 1 additions and 12527 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

79
nifi-assembly/pom.xml
View File

@ -336,12 +336,6 @@ language governing permissions and limitations under the License. -->
<version>2.0.0-SNAPSHOT</version>
<type>nar</type>
</dependency>
<dependency>
<groupId>org.apache.nifi</groupId>
<artifactId>nifi-html-nar</artifactId>
<version>2.0.0-SNAPSHOT</version>
<type>nar</type>
</dependency>
<dependency>
<groupId>org.apache.nifi</groupId>
<artifactId>nifi-lookup-services-nar</artifactId>
@ -420,12 +414,6 @@ language governing permissions and limitations under the License. -->
<version>2.0.0-SNAPSHOT</version>
<type>nar</type>
</dependency>
<dependency>
<groupId>org.apache.nifi</groupId>
<artifactId>nifi-hl7-nar</artifactId>
<version>2.0.0-SNAPSHOT</version>
<type>nar</type>
</dependency>
<dependency>
<groupId>org.apache.nifi</groupId>
<artifactId>nifi-language-translation-nar</artifactId>
@ -494,7 +482,7 @@ language governing permissions and limitations under the License. -->
</dependency>
<dependency>
<groupId>org.apache.nifi</groupId>
<artifactId>nifi-riemann-nar</artifactId>
<artifactId>nifi-hl7-nar</artifactId>
<version>2.0.0-SNAPSHOT</version>
<type>nar</type>
</dependency>
@ -552,12 +540,6 @@ language governing permissions and limitations under the License. -->
<version>2.0.0-SNAPSHOT</version>
<type>nar</type>
</dependency>
<dependency>
<groupId>org.apache.nifi</groupId>
<artifactId>nifi-cybersecurity-nar</artifactId>
<version>2.0.0-SNAPSHOT</version>
<type>nar</type>
</dependency>
<dependency>
<groupId>org.apache.nifi</groupId>
<artifactId>nifi-email-nar</artifactId>
@ -696,12 +678,6 @@ language governing permissions and limitations under the License. -->
<version>2.0.0-SNAPSHOT</version>
<type>nar</type>
</dependency>
<dependency>
<groupId>org.apache.nifi</groupId>
<artifactId>nifi-tcp-nar</artifactId>
<version>2.0.0-SNAPSHOT</version>
<type>nar</type>
</dependency>
<dependency>
<groupId>org.apache.nifi</groupId>
<artifactId>nifi-gcp-nar</artifactId>
@ -792,18 +768,6 @@ language governing permissions and limitations under the License. -->
<version>2.0.0-SNAPSHOT</version>
<type>nar</type>
</dependency>
<dependency>
<groupId>org.apache.nifi</groupId>
<artifactId>nifi-metrics-reporter-service-api-nar</artifactId>
<version>2.0.0-SNAPSHOT</version>
<type>nar</type>
</dependency>
<dependency>
<groupId>org.apache.nifi</groupId>
<artifactId>nifi-metrics-reporting-nar</artifactId>
<version>2.0.0-SNAPSHOT</version>
<type>nar</type>
</dependency>
<dependency>
<groupId>org.apache.nifi</groupId>
<artifactId>nifi-kerberos-credentials-service-nar</artifactId>
@ -1084,24 +1048,6 @@ language governing permissions and limitations under the License. -->
</dependency>
</dependencies>
</profile>
<profile>
<id>include-rules</id>
<!-- This profile handles includes of rules related artifacts. -->
<activation>
<activeByDefault>false</activeByDefault>
<property>
<name>allProfiles</name>
</property>
</activation>
<dependencies>
<dependency>
<groupId>org.apache.nifi</groupId>
<artifactId>nifi-rules-action-handler-nar</artifactId>
<version>2.0.0-SNAPSHOT</version>
<type>nar</type>
</dependency>
</dependencies>
</profile>
<profile>
<id>include-sql-reporting</id>
<!-- This profile handles the inclusion of nifi-sql-reporting artifacts. -->
@ -1118,12 +1064,6 @@ language governing permissions and limitations under the License. -->
<version>2.0.0-SNAPSHOT</version>
<type>nar</type>
</dependency>
<dependency>
<groupId>org.apache.nifi</groupId>
<artifactId>nifi-rules-action-handler-nar</artifactId>
<version>2.0.0-SNAPSHOT</version>
<type>nar</type>
</dependency>
</dependencies>
</profile>
<profile>
@ -1440,23 +1380,6 @@ language governing permissions and limitations under the License. -->
</dependency>
</dependencies>
</profile>
<profile>
<id>include-riemann</id>
<activation>
<activeByDefault>false</activeByDefault>
<property>
<name>allProfiles</name>
</property>
</activation>
<dependencies>
<dependency>
<groupId>org.apache.nifi</groupId>
<artifactId>nifi-riemann-nar</artifactId>
<version>2.0.0-SNAPSHOT</version>
<type>nar</type>
</dependency>
</dependencies>
</profile>
<profile>
<id>include-snowflake</id>
<activation>

3
nifi-docs/src/main/asciidoc/developer-guide.adoc
View File

@ -2730,10 +2730,7 @@ deprecationLogger.warn(
| ASN.1 Support | include-asn1 | Adds support for ASN.1
| Contribution Check | contrib-check | Runs various quality checks that are required to be accepted before a contribution can be accepted into the core NiFi code base.
| Graph Database Bundle | include-graph | Adds support for various common graph database scenarios. Support is currently for https://neo4j.com/developer/cypher[Cypher] and https://tinkerpop.apache.org/gremlin.html[Gremlin]-compatible databases such as Neo4J and JanusGraph. Includes controller services that provide driver functionality and a suite of processors for ingestion and querying.
| GRPC Bundle | include-grpc | **This profile is active in official builds and should be active** Provides support for the GRPC protocol.
| Media Bundle | include-media | The media bundle provides functionality based on https://tika.apache.org[Apache Tika] for extracting content and metadata from various types of binary formats supported by Apache Tika (ex. PDF, Microsoft Office).
| Riemann Bundle | include-riemann | Adds support for Riemann database components
| Rules Engine Bundle | include-rules | Adds support for creating scripted rules engines that can be integrated into existing flows. These rules engines can provide flexibility to people who need more complex flow logic or are more comfortable with flow decision-making using custom code.
| Snowflake Bundle | include-snowflake | Adds support for integration with the https://www.snowflake.com[Snowflake platform].
| SQL Reporting Bundle | include-sql-reporting | Adds reporting tasks that are designed to use SQL to update a RDBMS with metrics and other related data from Apache NiFi.
|==================================================================================================================================================

39
nifi-nar-bundles/nifi-cybersecurity-bundle/nifi-cybersecurity-nar/pom.xml
View File

@ -1,39 +0,0 @@
<?xml version="1.0" encoding="UTF-8"?>
<!--
Licensed to the Apache Software Foundation (ASF) under one or more
contributor license agreements. See the NOTICE file distributed with
this work for additional information regarding copyright ownership.
The ASF licenses this file to You under the Apache License, Version 2.0
(the "License"); you may not use this file except in compliance with
the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
-->
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>org.apache.nifi</groupId>
<artifactId>nifi-cybersecurity-bundle</artifactId>
<version>2.0.0-SNAPSHOT</version>
</parent>
<artifactId>nifi-cybersecurity-nar</artifactId>
<packaging>nar</packaging>
<properties>
<maven.javadoc.skip>true</maven.javadoc.skip>
<source.skip>true</source.skip>
</properties>
<dependencies>
<dependency>
<groupId>org.apache.nifi</groupId>
<artifactId>nifi-cybersecurity-processors</artifactId>
<version>2.0.0-SNAPSHOT</version>
</dependency>
</dependencies>
</project>

257
nifi-nar-bundles/nifi-cybersecurity-bundle/nifi-cybersecurity-nar/src/main/resources/META-INF/LICENSE
View File

@ -1,257 +0,0 @@
Apache License
Version 2.0, January 2004
http://www.apache.org/licenses/
TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
1. Definitions.
"License" shall mean the terms and conditions for use, reproduction,
and distribution as defined by Sections 1 through 9 of this document.
"Licensor" shall mean the copyright owner or entity authorized by
the copyright owner that is granting the License.
"Legal Entity" shall mean the union of the acting entity and all
other entities that control, are controlled by, or are under common
control with that entity. For the purposes of this definition,
"control" means (i) the power, direct or indirect, to cause the
direction or management of such entity, whether by contract or
otherwise, or (ii) ownership of fifty percent (50%) or more of the
outstanding shares, or (iii) beneficial ownership of such entity.
"You" (or "Your") shall mean an individual or Legal Entity
exercising permissions granted by this License.
"Source" form shall mean the preferred form for making modifications,
including but not limited to software source code, documentation
source, and configuration files.
"Object" form shall mean any form resulting from mechanical
transformation or translation of a Source form, including but
not limited to compiled object code, generated documentation,
and conversions to other media types.
"Work" shall mean the work of authorship, whether in Source or
Object form, made available under the License, as indicated by a
copyright notice that is included in or attached to the work
(an example is provided in the Appendix below).
"Derivative Works" shall mean any work, whether in Source or Object
form, that is based on (or derived from) the Work and for which the
editorial revisions, annotations, elaborations, or other modifications
represent, as a whole, an original work of authorship. For the purposes
of this License, Derivative Works shall not include works that remain
separable from, or merely link (or bind by name) to the interfaces of,
the Work and Derivative Works thereof.
"Contribution" shall mean any work of authorship, including
the original version of the Work and any modifications or additions
to that Work or Derivative Works thereof, that is intentionally
submitted to Licensor for inclusion in the Work by the copyright owner
or by an individual or Legal Entity authorized to submit on behalf of
the copyright owner. For the purposes of this definition, "submitted"
means any form of electronic, verbal, or written communication sent
to the Licensor or its representatives, including but not limited to
communication on electronic mailing lists, source code control systems,
and issue tracking systems that are managed by, or on behalf of, the
Licensor for the purpose of discussing and improving the Work, but
excluding communication that is conspicuously marked or otherwise
designated in writing by the copyright owner as "Not a Contribution."
"Contributor" shall mean Licensor and any individual or Legal Entity
on behalf of whom a Contribution has been received by Licensor and
subsequently incorporated within the Work.
2. Grant of Copyright License. Subject to the terms and conditions of
this License, each Contributor hereby grants to You a perpetual,
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
copyright license to reproduce, prepare Derivative Works of,
publicly display, publicly perform, sublicense, and distribute the
Work and such Derivative Works in Source or Object form.
3. Grant of Patent License. Subject to the terms and conditions of
this License, each Contributor hereby grants to You a perpetual,
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
(except as stated in this section) patent license to make, have made,
use, offer to sell, sell, import, and otherwise transfer the Work,
where such license applies only to those patent claims licensable
by such Contributor that are necessarily infringed by their
Contribution(s) alone or by combination of their Contribution(s)
with the Work to which such Contribution(s) was submitted. If You
institute patent litigation against any entity (including a
cross-claim or counterclaim in a lawsuit) alleging that the Work
or a Contribution incorporated within the Work constitutes direct
or contributory patent infringement, then any patent licenses
granted to You under this License for that Work shall terminate
as of the date such litigation is filed.
4. Redistribution. You may reproduce and distribute copies of the
Work or Derivative Works thereof in any medium, with or without
modifications, and in Source or Object form, provided that You
meet the following conditions:
(a) You must give any other recipients of the Work or
Derivative Works a copy of this License; and
(b) You must cause any modified files to carry prominent notices
stating that You changed the files; and
(c) You must retain, in the Source form of any Derivative Works
that You distribute, all copyright, patent, trademark, and
attribution notices from the Source form of the Work,
excluding those notices that do not pertain to any part of
the Derivative Works; and
(d) If the Work includes a "NOTICE" text file as part of its
distribution, then any Derivative Works that You distribute must
include a readable copy of the attribution notices contained
within such NOTICE file, excluding those notices that do not
pertain to any part of the Derivative Works, in at least one
of the following places: within a NOTICE text file distributed
as part of the Derivative Works; within the Source form or
documentation, if provided along with the Derivative Works; or,
within a display generated by the Derivative Works, if and
wherever such third-party notices normally appear. The contents
of the NOTICE file are for informational purposes only and
do not modify the License. You may add Your own attribution
notices within Derivative Works that You distribute, alongside
or as an addendum to the NOTICE text from the Work, provided
that such additional attribution notices cannot be construed
as modifying the License.
You may add Your own copyright statement to Your modifications and
may provide additional or different license terms and conditions
for use, reproduction, or distribution of Your modifications, or
for any such Derivative Works as a whole, provided Your use,
reproduction, and distribution of the Work otherwise complies with
the conditions stated in this License.
5. Submission of Contributions. Unless You explicitly state otherwise,
any Contribution intentionally submitted for inclusion in the Work
by You to the Licensor shall be under the terms and conditions of
this License, without any additional terms or conditions.
Notwithstanding the above, nothing herein shall supersede or modify
the terms of any separate license agreement you may have executed
with Licensor regarding such Contributions.
6. Trademarks. This License does not grant permission to use the trade
names, trademarks, service marks, or product names of the Licensor,
except as required for reasonable and customary use in describing the
origin of the Work and reproducing the content of the NOTICE file.
7. Disclaimer of Warranty. Unless required by applicable law or
agreed to in writing, Licensor provides the Work (and each
Contributor provides its Contributions) on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
implied, including, without limitation, any warranties or conditions
of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
PARTICULAR PURPOSE. You are solely responsible for determining the
appropriateness of using or redistributing the Work and assume any
risks associated with Your exercise of permissions under this License.
8. Limitation of Liability. In no event and under no legal theory,
whether in tort (including negligence), contract, or otherwise,
unless required by applicable law (such as deliberate and grossly
negligent acts) or agreed to in writing, shall any Contributor be
liable to You for damages, including any direct, indirect, special,
incidental, or consequential damages of any character arising as a
result of this License or out of the use or inability to use the
Work (including but not limited to damages for loss of goodwill,
work stoppage, computer failure or malfunction, or any and all
other commercial damages or losses), even if such Contributor
has been advised of the possibility of such damages.
9. Accepting Warranty or Additional Liability. While redistributing
the Work or Derivative Works thereof, You may choose to offer,
and charge a fee for, acceptance of support, warranty, indemnity,
or other liability obligations and/or rights consistent with this
License. However, in accepting such obligations, You may act only
on Your own behalf and on Your sole responsibility, not on behalf
of any other Contributor, and only if You agree to indemnify,
defend, and hold each Contributor harmless for any liability
incurred by, or claims asserted against, such Contributor by reason
of your accepting any such warranty or additional liability.
END OF TERMS AND CONDITIONS
APPENDIX: How to apply the Apache License to your work.
To apply the Apache License to your work, attach the following
boilerplate notice, with the fields enclosed by brackets "[]"
replaced with your own identifying information. (Don't include
the brackets!) The text should be enclosed in the appropriate
comment syntax for the file format. We also recommend that a
file or class name and description of purpose be included on the
same "printed page" as the copyright notice for easier
identification within third-party archives.
Copyright [yyyy] [name of copyright owner]
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
APACHE NIFI SUBCOMPONENTS:
The Apache NiFi project contains subcomponents with separate copyright
notices and license terms. Your use of the source code for the these
subcomponents is subject to the terms and conditions of the following
licenses.
The binary distribution of this project bundles "java-spamsum"
which is available under an MIT style license.
Copyright 2015 Thibault Debatty.
Permission is hereby granted, free of charge, to any person obtaining
a copy of this software and associated documentation files (the
"Software"), to deal in the Software without restriction, including
without limitation the rights to use, copy, modify, merge, publish,
distribute, sublicense, and/or sell copies of the Software, and to
permit persons to whom the Software is furnished to do so, subject to
the following conditions:
The above copyright notice and this permission notice shall be
included in all copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
The binary distribution of this project bundles "java-string-similarity"
which is available under an MIT style license.
Copyright 2015 Thibault Debatty.
Permission is hereby granted, free of charge, to any person obtaining
a copy of this software and associated documentation files (the
"Software"), to deal in the Software without restriction, including
without limitation the rights to use, copy, modify, merge, publish,
distribute, sublicense, and/or sell copies of the Software, and to
permit persons to whom the Software is furnished to do so, subject to
the following conditions:
The above copyright notice and this permission notice shall be
included in all copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

34
nifi-nar-bundles/nifi-cybersecurity-bundle/nifi-cybersecurity-nar/src/main/resources/META-INF/NOTICE
View File

@ -1,34 +0,0 @@
nifi-cybersecurity-nar
Copyright 2015-2020 The Apache Software Foundation
===========================================
Apache Software License v2
===========================================
The following binary components are provided under the Apache Software License v2
(ASLv2) tlsh
The following NOTICE information applies:
Java port of Trend Locality Sensitive Hash (TLSH)
Copyright 2000-2016 Idealista, S.A
This product includes software developed at
Idealista, S.A (http://www.idealista.com/)
Based on the algorithms described in Trend Micro's TLSH official
repository, available in:
https://github.com/trendmicro/tlsh
Refer to the following publication for more information:
Jonathan Oliver, Chun Cheng and Yanggui Chen, “TLSH - A Locality
Sensitive Hash” - 4th Cybercrime and Trustworthy Computing Workshop,
Sydney, November 2013
https://drive.google.com/file/d/0B6FS3SVQ1i0GTXk5eDl3Y29QWlk/edit?usp=sharing
This software is inspired in the previous Java port developed by TripleCheck:
https://github.com/triplecheck/TLSH

76
nifi-nar-bundles/nifi-cybersecurity-bundle/nifi-cybersecurity-processors/pom.xml
View File

@ -1,76 +0,0 @@
<?xml version="1.0" encoding="UTF-8"?>
<!--
Licensed to the Apache Software Foundation (ASF) under one or more
contributor license agreements. See the NOTICE file distributed with
this work for additional information regarding copyright ownership.
The ASF licenses this file to You under the Apache License, Version 2.0
(the "License"); you may not use this file except in compliance with
the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
-->
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>org.apache.nifi</groupId>
<artifactId>nifi-cybersecurity-bundle</artifactId>
<version>2.0.0-SNAPSHOT</version>
</parent>
<artifactId>nifi-cybersecurity-processors</artifactId>
<packaging>jar</packaging>
<dependencies>
<dependency>
<groupId>org.apache.nifi</groupId>
<artifactId>nifi-api</artifactId>
</dependency>
<dependency>
<groupId>org.apache.nifi</groupId>
<artifactId>nifi-utils</artifactId>
<version>2.0.0-SNAPSHOT</version>
</dependency>
<dependency>
<groupId>info.debatty</groupId>
<artifactId>java-spamsum</artifactId>
<version>0.2</version>
</dependency>
<dependency>
<groupId>com.idealista</groupId>
<artifactId>tlsh</artifactId>
<version>1.0.0</version>
</dependency>
<dependency>
<groupId>org.apache.nifi</groupId>
<artifactId>nifi-mock</artifactId>
<version>2.0.0-SNAPSHOT</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.apache.nifi</groupId>
<artifactId>nifi-properties</artifactId>
<scope>compile</scope>
</dependency>
</dependencies>
<build>
<plugins>
<plugin>
<groupId>org.apache.rat</groupId>
<artifactId>apache-rat-plugin</artifactId>
<configuration>
<excludes combine.children="append">
<exclude>src/test/resources/blank_ssdeep.list</exclude>
<exclude>src/test/resources/empty.list</exclude>
<exclude>src/test/resources/ssdeep.list</exclude>
<exclude>src/test/resources/tlsh.list</exclude>
</excludes>
</configuration>
</plugin>
</plugins>
</build>
</project>

88
nifi-nar-bundles/nifi-cybersecurity-bundle/nifi-cybersecurity-processors/src/main/java/org/apache/nifi/processors/cybersecurity/AbstractFuzzyHashProcessor.java
View File

@ -1,88 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.nifi.processors.cybersecurity;
import com.idealista.tlsh.TLSH;
import info.debatty.java.spamsum.SpamSum;
import org.apache.nifi.components.AllowableValue;
import org.apache.nifi.components.PropertyDescriptor;
import org.apache.nifi.flowfile.FlowFile;
import org.apache.nifi.processor.AbstractProcessor;
import org.apache.nifi.processor.Relationship;
import org.apache.nifi.processor.util.StandardValidators;
import java.util.List;
import java.util.Set;
abstract class AbstractFuzzyHashProcessor extends AbstractProcessor {
final protected static String ssdeep = "ssdeep";
final protected static String tlsh = "tlsh";
public static final AllowableValue allowableValueSSDEEP = new AllowableValue(
ssdeep,
ssdeep,
"Uses ssdeep / SpamSum 'context triggered piecewise hash'.");
public static final AllowableValue allowableValueTLSH = new AllowableValue(
tlsh,
tlsh,
"Uses TLSH (Trend 'Locality Sensitive Hash'). Note: FlowFile Content must be at least 512 characters long");
public static final PropertyDescriptor ATTRIBUTE_NAME = new PropertyDescriptor.Builder()
.name("ATTRIBUTE_NAME")
.displayName("Hash Attribute Name")
.description("The name of the FlowFile Attribute that should hold the Fuzzy Hash Value")
.required(true)
.addValidator(StandardValidators.NON_EMPTY_VALIDATOR)
.defaultValue("fuzzyhash.value")
.build();
public static final PropertyDescriptor HASH_ALGORITHM = new PropertyDescriptor.Builder()
.name("HASH_ALGORITHM")
.displayName("Hashing Algorithm")
.description("The hashing algorithm utilised")
.allowableValues(allowableValueSSDEEP, allowableValueTLSH)
.required(true)
.addValidator(StandardValidators.NON_EMPTY_VALIDATOR)
.build();
protected List<PropertyDescriptor> descriptors;
protected Set<Relationship> relationships;
protected boolean checkMinimumAlgorithmRequirements(String algorithm, FlowFile flowFile) {
// Check if content matches minimum length requirement
if (algorithm.equals(tlsh) && flowFile.getSize() < 512 ) {
return false;
} else {
return true;
}
}
protected String generateHash(String algorithm, String content) {
switch (algorithm) {
case tlsh:
return new TLSH(content).hash();
case ssdeep:
return new SpamSum().HashString(content);
default:
return null;
}
}
}

266
nifi-nar-bundles/nifi-cybersecurity-bundle/nifi-cybersecurity-processors/src/main/java/org/apache/nifi/processors/cybersecurity/CompareFuzzyHash.java
View File

@ -1,266 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.nifi.processors.cybersecurity;
import org.apache.nifi.annotation.behavior.InputRequirement;
import org.apache.nifi.annotation.behavior.SideEffectFree;
import org.apache.nifi.annotation.behavior.SupportsBatching;
import org.apache.nifi.annotation.behavior.WritesAttribute;
import org.apache.nifi.annotation.behavior.WritesAttributes;
import org.apache.nifi.annotation.documentation.CapabilityDescription;
import org.apache.nifi.annotation.documentation.SeeAlso;
import org.apache.nifi.annotation.documentation.Tags;
import org.apache.nifi.components.AllowableValue;
import org.apache.nifi.components.PropertyDescriptor;
import org.apache.nifi.components.resource.ResourceCardinality;
import org.apache.nifi.components.resource.ResourceType;
import org.apache.nifi.flowfile.FlowFile;
import org.apache.nifi.logging.ComponentLog;
import org.apache.nifi.processor.ProcessContext;
import org.apache.nifi.processor.ProcessSession;
import org.apache.nifi.processor.ProcessorInitializationContext;
import org.apache.nifi.processor.Relationship;
import org.apache.nifi.processor.exception.ProcessException;
import org.apache.nifi.processor.util.StandardValidators;
import org.apache.nifi.processors.cybersecurity.matchers.FuzzyHashMatcher;
import org.apache.nifi.processors.cybersecurity.matchers.SSDeepHashMatcher;
import org.apache.nifi.processors.cybersecurity.matchers.TLSHHashMatcher;
import org.apache.nifi.util.StringUtils;
import java.io.BufferedReader;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.Set;
@SideEffectFree
@SupportsBatching
@InputRequirement(InputRequirement.Requirement.INPUT_REQUIRED)
@SeeAlso({FuzzyHashContent.class})
@Tags({"hashing", "fuzzy-hashing", "cyber-security"})
@CapabilityDescription("Compares an attribute containing a Fuzzy Hash against a file containing a list of fuzzy hashes, " +
"appending an attribute to the FlowFile in case of a successful match.")
@WritesAttributes({
@WritesAttribute(attribute = "XXXX.N.match", description = "The match that resembles the attribute specified " +
"by the <Hash Attribute Name> property. Note that: 'XXX' gets replaced with the <Hash Attribute Name>"),
@WritesAttribute(attribute = "XXXX.N.similarity", description = "The similarity score between this flowfile" +
"and its match of the same number N. Note that: 'XXX' gets replaced with the <Hash Attribute Name>")})
public class CompareFuzzyHash extends AbstractFuzzyHashProcessor {
public static final AllowableValue singleMatch = new AllowableValue(
"single",
"single",
"Send FlowFile to matched after the first match above threshold");
public static final AllowableValue multiMatch = new AllowableValue(
"multi-match",
"multi-match",
"Iterate full list of hashes before deciding to send FlowFile to matched or unmatched");
public static final PropertyDescriptor HASH_LIST_FILE = new PropertyDescriptor.Builder()
.name("HASH_LIST_FILE")
.displayName("Hash List Source File")
.description("Path to the file containing hashes to be validated against")
.required(true)
.identifiesExternalResource(ResourceCardinality.SINGLE, ResourceType.FILE)
.build();
// Note we add a PropertyDescriptor HASH_ALGORITHM and ATTRIBUTE_NAME from parent class
public static final PropertyDescriptor MATCH_THRESHOLD = new PropertyDescriptor.Builder()
// Note that while both TLSH and SSDeep seems to return int, we treat them as double in code.
// The rationale behind being the expectation that other algorithms thatmay return double values
// may be added to the processor later on.
.name("MATCH_THRESHOLD")
.displayName("Match Threshold")
.description("The similarity score must exceed or be equal to in order for" +
"match to be considered true. Refer to Additional Information for differences between TLSH " +
"and SSDEEP scores and how they relate to this property.")
.required(true)
.addValidator(StandardValidators.NUMBER_VALIDATOR)
.build();
public static final PropertyDescriptor MATCHING_MODE = new PropertyDescriptor.Builder()
.name("MATCHING_MODE")
.displayName("Matching Mode")
.description("Defines if the Processor should try to match as many entries as possible (" + multiMatch.getDisplayName() +
") or if it should stop after the first match (" + singleMatch.getDisplayName() + ")")
.required(true)
.allowableValues(singleMatch,multiMatch)
.defaultValue(singleMatch.getValue())
.build();
public static final Relationship REL_FOUND = new Relationship.Builder()
.name("found")
.description("Any FlowFile that is successfully matched to an existing hash will be sent to this Relationship.")
.build();
public static final Relationship REL_NOT_FOUND = new Relationship.Builder()
.name("not-found")
.description("Any FlowFile that cannot be matched to an existing hash will be sent to this Relationship.")
.build();
public static final Relationship REL_FAILURE = new Relationship.Builder()
.name("failure")
.description("Any FlowFile that cannot be matched, e.g. (lacks the attribute) will be sent to this Relationship.")
.build();
@Override
protected void init(final ProcessorInitializationContext context) {
final List<PropertyDescriptor> descriptors = new ArrayList<PropertyDescriptor>();
descriptors.add(HASH_LIST_FILE);
// As mentioned above, add the PropertyDescriptor HASH_ALGORITHM and ATTRIBUTE_NAME from parent class
descriptors.add(HASH_ALGORITHM);
descriptors.add(ATTRIBUTE_NAME);
descriptors.add(MATCH_THRESHOLD);
descriptors.add(MATCHING_MODE);
this.descriptors = Collections.unmodifiableList(descriptors);
final Set<Relationship> relationships = new HashSet<Relationship>();
relationships.add(REL_FOUND);
relationships.add(REL_NOT_FOUND);
relationships.add(REL_FAILURE);
this.relationships = Collections.unmodifiableSet(relationships);
}
@Override
public Set<Relationship> getRelationships() {
return this.relationships;
}
@Override
public final List<PropertyDescriptor> getSupportedPropertyDescriptors() {
return descriptors;
}
@Override
public void onTrigger(ProcessContext context, ProcessSession session) throws ProcessException {
FlowFile flowFile = session.get();
if (flowFile == null) {
return;
}
final ComponentLog logger = getLogger();
String algorithm = context.getProperty(HASH_ALGORITHM).getValue();
final String attributeName = context.getProperty(ATTRIBUTE_NAME).getValue();
String inputHash = flowFile.getAttribute(attributeName);
if (inputHash == null) {
getLogger().info("FlowFile {} lacks the required '{}' attribute, routing to failure.",
new Object[]{flowFile, attributeName});
session.transfer(flowFile, REL_FAILURE);
return;
}
final FuzzyHashMatcher fuzzyHashMatcher;
switch (algorithm) {
case tlsh:
fuzzyHashMatcher = new TLSHHashMatcher(getLogger());
break;
case ssdeep:
fuzzyHashMatcher = new SSDeepHashMatcher(getLogger());
break;
default:
getLogger().error("Seems like the processor is configured to use unsupported algorithm '{}' ? Yielding.",
new Object[]{algorithm});
context.yield();
return;
}
if (!fuzzyHashMatcher.isValidHash(inputHash)) {
// and if that is the case we log
logger.error("Invalid hash provided for {}. Sending to failure", flowFile);
// and send to failure
session.transfer(flowFile, REL_FAILURE);
return;
}
double similarity = 0;
double matchThreshold = context.getProperty(MATCH_THRESHOLD).asDouble();
try {
Map<String, Double> matched = new LinkedHashMap<>();
try (BufferedReader reader = fuzzyHashMatcher.getReader(context.getProperty(HASH_LIST_FILE).getValue())) {
String line = null;
while ((line = reader.readLine()) != null) {
similarity = fuzzyHashMatcher.getSimilarity(inputHash, line);
if (fuzzyHashMatcher.matchExceedsThreshold(similarity, matchThreshold)) {
String match = fuzzyHashMatcher.getMatch(line);
// A malformed file may cause a match with no filename
// Because this would simply look odd, we ignore such entry and log
if (!StringUtils.isEmpty(match)) {
matched.put(match, similarity);
} else {
logger.error("Found a match against a malformed entry '{}'. Please inspect the contents of" +
"the {} file and ensure they are properly formatted",
new Object[]{line, HASH_LIST_FILE.getDisplayName()});
}
}
// Check if single match is desired and if a match has been made
if (Objects.equals(context.getProperty(MATCHING_MODE).getValue(), singleMatch.getValue()) && (matched.size() > 0)) {
// and save time by breaking the outer loop
break;
}
}
}
// Then by iterating over the hashmap of matches
if (matched.size() > 0) {
// no matter if the break was called or not, Continue processing
// First by creating a new map to hold attributes
final Map<String, String> attributes = new HashMap<>();
int x = 0;
for (Map.Entry<String, Double> entry : matched.entrySet()) {
// defining attributes accordingly
attributes.put(attributeName + "." + x + ".match", entry.getKey());
attributes.put(attributeName + "." + x + ".similarity", String.valueOf(entry.getValue()));
x++;
}
// Finally, append the attributes to the flowfile and sent to match
flowFile = session.putAllAttributes(flowFile, attributes);
session.transfer(flowFile, REL_FOUND);
} else {
// Otherwise send it to non-match
session.transfer(flowFile, REL_NOT_FOUND);
}
} catch (final IOException e) {
logger.error("Error while reading the hash input source for {}", flowFile, e);
session.transfer(flowFile, REL_FAILURE);
}
}
}

175
nifi-nar-bundles/nifi-cybersecurity-bundle/nifi-cybersecurity-processors/src/main/java/org/apache/nifi/processors/cybersecurity/FuzzyHashContent.java
View File

@ -1,175 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.nifi.processors.cybersecurity;
import com.idealista.tlsh.exceptions.InsufficientComplexityException;
import org.apache.nifi.annotation.behavior.InputRequirement;
import org.apache.nifi.annotation.behavior.SideEffectFree;
import org.apache.nifi.annotation.behavior.SupportsBatching;
import org.apache.nifi.annotation.behavior.WritesAttribute;
import org.apache.nifi.annotation.behavior.WritesAttributes;
import org.apache.nifi.annotation.documentation.CapabilityDescription;
import org.apache.nifi.annotation.documentation.SeeAlso;
import org.apache.nifi.annotation.documentation.Tags;
import org.apache.nifi.annotation.lifecycle.OnScheduled;
import org.apache.nifi.components.PropertyDescriptor;
import org.apache.nifi.flowfile.FlowFile;
import org.apache.nifi.logging.ComponentLog;
import org.apache.nifi.processor.exception.ProcessException;
import org.apache.nifi.processor.ProcessContext;
import org.apache.nifi.processor.ProcessSession;
import org.apache.nifi.processor.ProcessorInitializationContext;
import org.apache.nifi.processor.Relationship;
import org.apache.nifi.processor.io.InputStreamCallback;
import org.apache.nifi.processor.util.StandardValidators;
import org.apache.nifi.util.StringUtils;
import org.apache.nifi.stream.io.StreamUtils;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
import java.util.concurrent.atomic.AtomicReference;
@SideEffectFree
@SupportsBatching
@InputRequirement(InputRequirement.Requirement.INPUT_REQUIRED)
@Tags({"hashing", "fuzzy-hashing", "cyber-security"})
@CapabilityDescription("Calculates a fuzzy/locality-sensitive hash value for the Content of a FlowFile and puts that " +
"hash value on the FlowFile as an attribute whose name is determined by the <Hash Attribute Name> property." +
"Note: this processor only offers non-cryptographic hash algorithms. And it should be not be " +
"seen as a replacement to the HashContent processor." +
"Note: The underlying library loads the entirety of the streamed content into and performs result " +
"evaluations in memory. Accordingly, it is important to consider the anticipated profile of content being " +
"evaluated by this processor and the hardware supporting it especially when working against large files.")
@SeeAlso(value = {CompareFuzzyHash.class})
@WritesAttributes({@WritesAttribute(attribute = "<Hash Attribute Name>", description = "This Processor adds an attribute whose value is the result of Hashing the "
+ "existing FlowFile content. The name of this attribute is specified by the <Hash Attribute Name> property")})
public class FuzzyHashContent extends AbstractFuzzyHashProcessor {
public static final PropertyDescriptor HASH_ALGORITHM = new PropertyDescriptor.Builder()
.name("HASH_ALGORITHM")
.displayName("Hashing Algorithm")
.description("The hashing algorithm utilised")
.allowableValues(allowableValueSSDEEP, allowableValueTLSH)
.required(true)
.addValidator(StandardValidators.NON_EMPTY_VALIDATOR)
.build();
public static final Relationship REL_SUCCESS = new Relationship.Builder()
.name("success")
.description("Any FlowFile that is successfully hashed will be sent to this Relationship.")
.build();
public static final Relationship REL_FAILURE = new Relationship.Builder()
.name("failure")
.description("Any FlowFile that is successfully hashed will be sent to this Relationship.")
.build();
private List<PropertyDescriptor> descriptors;
private Set<Relationship> relationships;
@Override
protected void init(final ProcessorInitializationContext context) {
final List<PropertyDescriptor> descriptors = new ArrayList<PropertyDescriptor>();
descriptors.add(ATTRIBUTE_NAME);
descriptors.add(HASH_ALGORITHM);
this.descriptors = Collections.unmodifiableList(descriptors);
final Set<Relationship> relationships = new HashSet<Relationship>();
relationships.add(REL_SUCCESS);
relationships.add(REL_FAILURE);
this.relationships = Collections.unmodifiableSet(relationships);
}
@Override
public Set<Relationship> getRelationships() {
return this.relationships;
}
@Override
public final List<PropertyDescriptor> getSupportedPropertyDescriptors() {
return descriptors;
}
@OnScheduled
public void onScheduled(final ProcessContext context) {
}
@Override
public void onTrigger(final ProcessContext context, final ProcessSession session) throws ProcessException {
FlowFile flowFile = session.get();
if (flowFile == null) {
return;
}
final ComponentLog logger = getLogger();
String algorithm = context.getProperty(HASH_ALGORITHM).getValue();
// Check if content matches minimum length requirement
if (checkMinimumAlgorithmRequirements(algorithm, flowFile) == false) {
logger.error("The content of '{}' is smaller than the minimum required by {}, routing to failure",
new Object[]{flowFile, algorithm});
session.transfer(flowFile, REL_FAILURE);
return;
}
final AtomicReference<String> hashValueHolder = new AtomicReference<>(null);
try {
session.read(flowFile, new InputStreamCallback() {
@Override
public void process(final InputStream in) throws IOException {
try (ByteArrayOutputStream holder = new ByteArrayOutputStream()) {
StreamUtils.copy(in,holder);
String hashValue = generateHash(algorithm, holder.toString());
if (StringUtils.isBlank(hashValue) == false) {
hashValueHolder.set(hashValue);
}
}
}
});
final String attributeName = context.getProperty(ATTRIBUTE_NAME).getValue();
flowFile = session.putAttribute(flowFile, attributeName, hashValueHolder.get());
logger.info("Successfully added attribute '{}' to {} with a value of {}; routing to success", new Object[]{attributeName, flowFile, hashValueHolder.get()});
session.getProvenanceReporter().modifyAttributes(flowFile);
session.transfer(flowFile, REL_SUCCESS);
} catch (final InsufficientComplexityException | ProcessException e) {
logger.error("Failed to process {} due to {}; routing to failure", new Object[]{flowFile, e});
session.transfer(flowFile, REL_FAILURE);
}
}
}

35
nifi-nar-bundles/nifi-cybersecurity-bundle/nifi-cybersecurity-processors/src/main/java/org/apache/nifi/processors/cybersecurity/matchers/FuzzyHashMatcher.java
View File

@ -1,35 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.nifi.processors.cybersecurity.matchers;
import java.io.BufferedReader;
import java.io.IOException;
public interface FuzzyHashMatcher {
BufferedReader getReader(String source) throws IOException;
boolean matchExceedsThreshold(double similarity, double matchThreshold) ;
double getSimilarity(String inputHash, String existingHash);
boolean isValidHash(String inputHash);
String getHash(String line);
String getMatch(String line);
}

126
nifi-nar-bundles/nifi-cybersecurity-bundle/nifi-cybersecurity-processors/src/main/java/org/apache/nifi/processors/cybersecurity/matchers/SSDeepHashMatcher.java
View File

@ -1,126 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.nifi.processors.cybersecurity.matchers;
import info.debatty.java.spamsum.SpamSum;
import org.apache.nifi.logging.ComponentLog;
import java.io.BufferedReader;
import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
import java.io.InputStreamReader;
import java.util.Scanner;
public class SSDeepHashMatcher implements FuzzyHashMatcher {
ComponentLog logger;
public SSDeepHashMatcher() {
}
public SSDeepHashMatcher(ComponentLog logger) {
this.logger = logger;
}
@Override
public BufferedReader getReader(String source) throws IOException {
File file = new File(source);
FileInputStream fileInputStream = new FileInputStream(file);
BufferedReader reader = new BufferedReader(new InputStreamReader(fileInputStream));
// If SSdeep skip the first line (as the usual format used by other tools add a header line
// to a file list
reader.readLine();
return reader;
}
@Override
public boolean matchExceedsThreshold(double similarity, double matchThreshold) {
if (similarity >= matchThreshold) {
return true;
} else {
return false;
}
}
@Override
public double getSimilarity(String inputHash, String existingHash) {
String[] hashToCompare = existingHash.split(",", 2);
if (hashToCompare.length > 0) {
return new SpamSum().match(inputHash, hashToCompare[0]);
} else {
return Double.NaN;
}
}
@Override
public boolean isValidHash(String inputHash) {
// format looks like
// blocksize:hash:hash
String [] fields = inputHash.split(":", 3);
if (fields.length == 3) {
Scanner sc = new Scanner(fields[0]);
boolean isNumber = sc.hasNextInt();
if (isNumber == false && logger != null) {
if (logger.isDebugEnabled()) {
logger.debug("Field should be numeric but got '{}'. Will tell processor to ignore.",
new Object[] {fields[0]});
}
}
boolean hashOneIsNotEmpty = !fields[1].isEmpty();
boolean hashTwoIsNotEmpty = !fields[2].isEmpty();
if (isNumber && hashOneIsNotEmpty && hashTwoIsNotEmpty) {
return true;
}
}
return false;
}
@Override
public String getHash(String line) {
if (isValidHash(line)) {
return line.split(",", 2)[0];
} else {
return null;
}
}
@Override
public String getMatch(String line) {
if (isValidHash(line)) {
String[] match = line.split(",", 2);
// Because the file can be malformed and contain an unammed match,
// if match has a filename...
if (match.length == 2) {
// Return it.
return match[1];
}
}
// Or return null
return null;
}
}

117
nifi-nar-bundles/nifi-cybersecurity-bundle/nifi-cybersecurity-processors/src/main/java/org/apache/nifi/processors/cybersecurity/matchers/TLSHHashMatcher.java
View File

@ -1,117 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.nifi.processors.cybersecurity.matchers;
import com.idealista.tlsh.digests.Digest;
import com.idealista.tlsh.digests.DigestBuilder;
import org.apache.nifi.logging.ComponentLog;
import java.io.BufferedReader;
import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
import java.io.InputStreamReader;
import static org.apache.nifi.processors.cybersecurity.CompareFuzzyHash.HASH_LIST_FILE;
public class TLSHHashMatcher implements FuzzyHashMatcher {
ComponentLog logger;
public TLSHHashMatcher(ComponentLog logger) {
this.logger = logger;
}
@Override
public BufferedReader getReader(String source) throws IOException {
File file = new File(source);
FileInputStream fileInputStream = new FileInputStream(file);
BufferedReader reader = new BufferedReader(new InputStreamReader(fileInputStream));
return reader;
}
@Override
public boolean matchExceedsThreshold(double similarity, double matchThreshold) {
if (similarity <= matchThreshold) {
return true;
} else {
return false;
}
}
@Override
public double getSimilarity(String inputHash, String existingHash) {
String[] hashToCompare = existingHash.split("\t", 2);
// This will return null in case it fails validation
if (isValidHash(inputHash) && isValidHash(hashToCompare[0])) {
Digest inputDigest = new DigestBuilder().withHash(inputHash).build();
Digest existingHashDigest = new DigestBuilder().withHash(hashToCompare[0]).build();
return inputDigest.calculateDifference(existingHashDigest, true);
} else {
return Double.NaN;
}
}
@Override
public boolean isValidHash(String stringFromHashList) {
String[] hashToCompare = stringFromHashList.split("\t", 2);
// This will return null in case it fails validation
if (hashToCompare.length > 0) {
// Because DigestBuilder raises all sort of exceptions, so in order to keep the onTrigger loop a
// bit cleaner, we capture them here and return NaN to the loop above, otherwise simply return the
// similarity score.
try {
Digest digest = new DigestBuilder().withHash(hashToCompare[0]).build();
return true;
} catch (ArrayIndexOutOfBoundsException | StringIndexOutOfBoundsException | NumberFormatException e) {
logger.error("Got {} while processing the string '{}'. This usually means the file " +
"defined by '{}' property contains invalid entries.",
new Object[]{e.getCause(), hashToCompare[0], HASH_LIST_FILE.getDisplayName()});
}
}
return false;
}
@Override
public String getHash(String line) {
if (isValidHash(line)) {
return line.split("\t", 2)[0];
} else {
return null;
}
}
@Override
public String getMatch(String line) {
if (isValidHash(line)) {
String[] match = line.split("\t", 2);
// Because the file can be malformed and contain an unammed match,
// if match has a filename...
if (match.length == 2) {
// Return it.
return match[1];
}
}
// Or return null
return null;
}
}

16
nifi-nar-bundles/nifi-cybersecurity-bundle/nifi-cybersecurity-processors/src/main/resources/META-INF/services/org.apache.nifi.processor.Processor
View File

@ -1,16 +0,0 @@
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
org.apache.nifi.processors.cybersecurity.FuzzyHashContent
org.apache.nifi.processors.cybersecurity.CompareFuzzyHash

45
nifi-nar-bundles/nifi-cybersecurity-bundle/nifi-cybersecurity-processors/src/main/resources/docs/org/apache/nifi/processors/cybersecurity/CompareFuzzyHash/additionalDetails.html
View File

@ -1,45 +0,0 @@
<!DOCTYPE html>
<html lang="en">
<!--
Licensed to the Apache Software Foundation (ASF) under one or more
contributor license agreements. See the NOTICE file distributed with
this work for additional information regarding copyright ownership.
The ASF licenses this file to You under the Apache License, Version 2.0
(the "License"); you may not use this file except in compliance with
the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
-->
<head>
<meta charset="utf-8" />
<title>CompareFuzzyHash</title>
<link rel="stylesheet" href="../../../../../css/component-usage.css"
type="text/css" />
</head>
<body>
<!-- Processor Documentation ================================================== -->
<h2>Description:</h2>
<p>This Processor compares a an attribute containing a Fuzzy Hash (TLSH or SSDeep) value and compares it against a list
of hashes of the same family (i.e. TLSH is compared with a list of TLSH hashes), routing them to match or non-match
depending on a user configured threshold for similarity.
</p>
<p>It is important to note that:</p>
<ul>
<li>TLSH similarity increases as product of its comparison function decreases (i.e. 0 indicates nearly identical files)</li>
<li>SSDeep similarity directly relates to the product of its comparison function (e.g. 99 indicates nearly identical files</li>
</ul>
<p>Based on the above, this processor when referring to "exceed the score" may be referring to:
<ul>
<li>a value equal or lower than the configured threshold (in case of TLSH)</li>
<li>a value equal or higher than the configured threshold (in case of SSDeep)</li>
</ul>
</body>
</html>

381
nifi-nar-bundles/nifi-cybersecurity-bundle/nifi-cybersecurity-processors/src/test/java/org/apache/nifi/processors/cybersecurity/TestCompareFuzzyHash.java
View File

@ -1,381 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.nifi.processors.cybersecurity;
import org.apache.nifi.processors.cybersecurity.matchers.FuzzyHashMatcher;
import org.apache.nifi.processors.cybersecurity.matchers.SSDeepHashMatcher;
import org.apache.nifi.util.MockFlowFile;
import org.apache.nifi.util.TestRunner;
import org.apache.nifi.util.TestRunners;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.Test;
import java.util.Arrays;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import static org.junit.jupiter.api.Assertions.assertTrue;
public class TestCompareFuzzyHash {
String ssdeepInput = "48:c1xs8Z/m6H0eRH31S8p8bHENANkPrNy4tkPytwPyh2jTytxPythPytNdPytDgYyF:OuO/mg3HFSRHEb44RNMi6uHU2hcq3";
String tlshInput = "EB519EA4A8F95171A2A409C1DEEB9872AF55C137E00A5289F1CCD0CE4F6CCD784BB4B7";
final CompareFuzzyHash proc = new CompareFuzzyHash();
final private TestRunner runner = TestRunners.newTestRunner(proc);
@AfterEach
public void stop() {
runner.shutdown();
}
@Test
public void testSsdeepCompareFuzzyHash() {
double matchingSimilarity = 80;
runner.setProperty(CompareFuzzyHash.HASH_ALGORITHM, CompareFuzzyHash.allowableValueSSDEEP.getValue());
runner.setProperty(CompareFuzzyHash.ATTRIBUTE_NAME, "fuzzyhash.value");
runner.setProperty(CompareFuzzyHash.HASH_LIST_FILE, "src/test/resources/ssdeep.list");
runner.setProperty(CompareFuzzyHash.MATCH_THRESHOLD, String.valueOf(matchingSimilarity));
runner.setProperty(CompareFuzzyHash.MATCHING_MODE, CompareFuzzyHash.singleMatch.getValue());
Map<String, String> attributes = new HashMap<>();
attributes.put("fuzzyhash.value", ssdeepInput);
runner.enqueue("bogus".getBytes(), attributes);
runner.run();
runner.assertQueueEmpty();
runner.assertAllFlowFilesTransferred(CompareFuzzyHash.REL_FOUND, 1);
final MockFlowFile outFile = runner.getFlowFilesForRelationship(CompareFuzzyHash.REL_FOUND).get(0);
outFile.assertAttributeEquals(
"fuzzyhash.value.0.match",
"\"nifi/nifi-nar-bundles/nifi-beats-bundle/nifi-beats-processors/pom.xml\""
);
double similarity = Double.valueOf(outFile.getAttribute("fuzzyhash.value.0.similarity"));
assertTrue(similarity >= matchingSimilarity);
outFile.assertAttributeNotExists("fuzzyhash.value.1.match");
}
@Test
public void testSsdeepCompareFuzzyHashMultipleMatches() {
double matchingSimilarity = 80;
runner.setProperty(CompareFuzzyHash.HASH_ALGORITHM, CompareFuzzyHash.allowableValueSSDEEP.getValue());
runner.setProperty(CompareFuzzyHash.ATTRIBUTE_NAME, "fuzzyhash.value");
runner.setProperty(CompareFuzzyHash.HASH_LIST_FILE, "src/test/resources/ssdeep.list");
runner.setProperty(CompareFuzzyHash.MATCH_THRESHOLD, String.valueOf(matchingSimilarity));
runner.setProperty(CompareFuzzyHash.MATCHING_MODE, CompareFuzzyHash.multiMatch.getValue());
Map<String, String> attributes = new HashMap<>();
attributes.put("fuzzyhash.value", ssdeepInput );
runner.enqueue("bogus".getBytes(), attributes);
runner.run();
runner.assertQueueEmpty();
runner.assertAllFlowFilesTransferred(CompareFuzzyHash.REL_FOUND, 1);
final MockFlowFile outFile = runner.getFlowFilesForRelationship(CompareFuzzyHash.REL_FOUND).get(0);
outFile.assertAttributeEquals("fuzzyhash.value.0.match",
"\"nifi/nifi-nar-bundles/nifi-beats-bundle/nifi-beats-processors/pom.xml\""
);
double similarity = Double.valueOf(outFile.getAttribute("fuzzyhash.value.0.similarity"));
assertTrue(similarity >= matchingSimilarity);
outFile.assertAttributeEquals("fuzzyhash.value.1.match",
"\"nifi/nifi-nar-bundles/nifi-lumberjack-bundle/nifi-lumberjack-processors/pom.xml\""
);
similarity = Double.valueOf(outFile.getAttribute("fuzzyhash.value.1.similarity"));
assertTrue(similarity >= matchingSimilarity);
}
@Test
public void testSsdeepCompareFuzzyHashWithBlankHashList() {
double matchingSimilarity = 80;
runner.setProperty(CompareFuzzyHash.HASH_ALGORITHM, CompareFuzzyHash.allowableValueSSDEEP.getValue());
runner.setProperty(CompareFuzzyHash.ATTRIBUTE_NAME, "fuzzyhash.value");
runner.setProperty(CompareFuzzyHash.HASH_LIST_FILE, "src/test/resources/blank_ssdeep.list");
runner.setProperty(CompareFuzzyHash.MATCH_THRESHOLD, String.valueOf(matchingSimilarity));
Map<String, String> attributes = new HashMap<>();
attributes.put("fuzzyhash.value", "6:hERjIfhRrlB63J0FDw1NBQmEH68xwMSELN:hZrlB62IwMS");
runner.enqueue("bogus".getBytes(), attributes);
runner.run();
runner.assertQueueEmpty();
runner.assertAllFlowFilesTransferred(CompareFuzzyHash.REL_NOT_FOUND, 1);
final MockFlowFile outFile = runner.getFlowFilesForRelationship(CompareFuzzyHash.REL_NOT_FOUND).get(0);
}
@Test
public void testSsdeepCompareFuzzyHashWithInvalidHashList() {
// This is different from "BlankHashList series of tests in that the file lacks headers and as such is totally
// invalid
double matchingSimilarity = 80;
runner.setProperty(CompareFuzzyHash.HASH_ALGORITHM, CompareFuzzyHash.allowableValueSSDEEP.getValue());
runner.setProperty(CompareFuzzyHash.ATTRIBUTE_NAME, "fuzzyhash.value");
runner.setProperty(CompareFuzzyHash.HASH_LIST_FILE, "src/test/resources/empty.list");
runner.setProperty(CompareFuzzyHash.MATCH_THRESHOLD, String.valueOf(matchingSimilarity));
Map<String, String> attributes = new HashMap<>();
attributes.put("fuzzyhash.value", "6:hERjIfhRrlB63J0FDw1NBQmEH68xwMSELN:hZrlB62IwMS");
runner.enqueue("bogus".getBytes(), attributes);
runner.run();
runner.assertQueueEmpty();
runner.assertAllFlowFilesTransferred(CompareFuzzyHash.REL_NOT_FOUND, 1);
final MockFlowFile outFile = runner.getFlowFilesForRelationship(CompareFuzzyHash.REL_NOT_FOUND).get(0);
outFile.assertAttributeNotExists("fuzzyhash.value.0.match");
}
@Test
public void testSsdeepCompareFuzzyHashWithInvalidHash() {
double matchingSimilarity = 80;
runner.setProperty(CompareFuzzyHash.HASH_ALGORITHM, CompareFuzzyHash.allowableValueSSDEEP.getValue());
runner.setProperty(CompareFuzzyHash.ATTRIBUTE_NAME, "fuzzyhash.value");
runner.setProperty(CompareFuzzyHash.HASH_LIST_FILE, "src/test/resources/ssdeep.list");
runner.setProperty(CompareFuzzyHash.MATCH_THRESHOLD, String.valueOf(matchingSimilarity));
runner.setProperty(CompareFuzzyHash.MATCHING_MODE, CompareFuzzyHash.singleMatch.getValue());
Map<String, String> attributes = new HashMap<>();
attributes.put("fuzzyhash.value", "Test test test chocolate!");
runner.enqueue("bogus".getBytes(), attributes);
runner.run();
runner.assertQueueEmpty();
runner.assertAllFlowFilesTransferred(CompareFuzzyHash.REL_FAILURE, 1);
final MockFlowFile outFile = runner.getFlowFilesForRelationship(CompareFuzzyHash.REL_FAILURE).get(0);
outFile.assertAttributeNotExists("fuzzyhash.value.0.match");
}
@Test
public void testTLSHCompareFuzzyHash() {
double matchingSimilarity = 200;
runner.setProperty(CompareFuzzyHash.HASH_ALGORITHM, CompareFuzzyHash.allowableValueTLSH.getValue());
runner.setProperty(CompareFuzzyHash.ATTRIBUTE_NAME, "fuzzyhash.value");
runner.setProperty(CompareFuzzyHash.HASH_LIST_FILE, "src/test/resources/tlsh.list");
runner.setProperty(CompareFuzzyHash.MATCH_THRESHOLD, String.valueOf(matchingSimilarity));
runner.setProperty(CompareFuzzyHash.MATCHING_MODE, CompareFuzzyHash.singleMatch.getValue());
Map<String, String> attributes = new HashMap<>();
attributes.put("fuzzyhash.value", tlshInput);
runner.enqueue("bogus".getBytes(), attributes);
runner.run();
runner.assertQueueEmpty();
runner.assertAllFlowFilesTransferred(CompareFuzzyHash.REL_FOUND, 1);
final MockFlowFile outFile = runner.getFlowFilesForRelationship(CompareFuzzyHash.REL_FOUND).get(0);
outFile.assertAttributeEquals(
"fuzzyhash.value.0.match",
"nifi-nar-bundles/nifi-lumberjack-bundle/nifi-lumberjack-processors/pom.xml"
);
double similarity = Double.valueOf(outFile.getAttribute("fuzzyhash.value.0.similarity"));
assertTrue(similarity <= matchingSimilarity);
outFile.assertAttributeNotExists("fuzzyhash.value.1.match");
}
@Test
public void testTLSHCompareFuzzyHashMultipleMatches() {
double matchingSimilarity = 200;
runner.setProperty(CompareFuzzyHash.HASH_ALGORITHM, CompareFuzzyHash.allowableValueTLSH.getValue());
runner.setProperty(CompareFuzzyHash.ATTRIBUTE_NAME, "fuzzyhash.value");
runner.setProperty(CompareFuzzyHash.HASH_LIST_FILE, "src/test/resources/tlsh.list");
runner.setProperty(CompareFuzzyHash.MATCH_THRESHOLD, String.valueOf(matchingSimilarity));
runner.setProperty(CompareFuzzyHash.MATCHING_MODE, CompareFuzzyHash.multiMatch.getValue());
Map<String, String> attributes = new HashMap<>();
attributes.put("fuzzyhash.value", tlshInput);
runner.enqueue("bogus".getBytes(), attributes);
runner.run();
runner.assertQueueEmpty();
runner.assertAllFlowFilesTransferred(CompareFuzzyHash.REL_FOUND, 1);
final MockFlowFile outFile = runner.getFlowFilesForRelationship(CompareFuzzyHash.REL_FOUND).get(0);
outFile.assertAttributeEquals(
"fuzzyhash.value.0.match",
"nifi-nar-bundles/nifi-lumberjack-bundle/nifi-lumberjack-processors/pom.xml"
);
double similarity = Double.valueOf(outFile.getAttribute("fuzzyhash.value.0.similarity"));
assertTrue(similarity <= matchingSimilarity);
outFile.assertAttributeEquals(
"fuzzyhash.value.1.match",
"nifi-nar-bundles/nifi-beats-bundle/nifi-beats-processors/pom.xml"
);
similarity = Double.valueOf(outFile.getAttribute("fuzzyhash.value.1.similarity"));
assertTrue(similarity <= matchingSimilarity);
}
@Test
public void testTLSHCompareFuzzyHashWithBlankFile() {
// This is different from "BlankHashList series of tests in that the file lacks headers and as such is totally
// invalid
double matchingSimilarity = 200;
runner.setProperty(CompareFuzzyHash.HASH_ALGORITHM, CompareFuzzyHash.allowableValueTLSH.getValue());
runner.setProperty(CompareFuzzyHash.ATTRIBUTE_NAME, "fuzzyhash.value");
runner.setProperty(CompareFuzzyHash.HASH_LIST_FILE, "src/test/resources/empty.list");
runner.setProperty(CompareFuzzyHash.MATCH_THRESHOLD, String.valueOf(matchingSimilarity));
Map<String, String> attributes = new HashMap<>();
attributes.put("fuzzyhash.value", "E2F0818B7AE7173906A72221570E30979B11C0FC47B518A1E89D257E2343CEC02381ED");
runner.enqueue("bogus".getBytes(), attributes);
runner.run();
runner.assertQueueEmpty();
runner.assertAllFlowFilesTransferred(CompareFuzzyHash.REL_NOT_FOUND, 1);
final MockFlowFile outFile = runner.getFlowFilesForRelationship(CompareFuzzyHash.REL_NOT_FOUND).get(0);
outFile.assertAttributeNotExists("fuzzyhash.value.0.match");
}
@Test
public void testTLSHCompareFuzzyHashWithEmptyHashList() {
double matchingSimilarity = 200;
runner.setProperty(CompareFuzzyHash.HASH_ALGORITHM, CompareFuzzyHash.allowableValueTLSH.getValue());
runner.setProperty(CompareFuzzyHash.ATTRIBUTE_NAME, "fuzzyhash.value");
runner.setProperty(CompareFuzzyHash.HASH_LIST_FILE, "src/test/resources/empty.list");
runner.setProperty(CompareFuzzyHash.MATCH_THRESHOLD, String.valueOf(matchingSimilarity));
Map<String, String> attributes = new HashMap<>();
attributes.put("fuzzyhash.value", "E2F0818B7AE7173906A72221570E30979B11C0FC47B518A1E89D257E2343CEC02381ED");
runner.enqueue("bogus".getBytes(), attributes);
runner.run();
runner.assertQueueEmpty();
runner.assertAllFlowFilesTransferred(CompareFuzzyHash.REL_NOT_FOUND, 1);
final MockFlowFile outFile = runner.getFlowFilesForRelationship(CompareFuzzyHash.REL_NOT_FOUND).get(0);
outFile.assertAttributeNotExists("fuzzyhash.value.0.match");
}
@Test
public void testTLSHCompareFuzzyHashWithInvalidHash() {
double matchingSimilarity = 200;
runner.setProperty(CompareFuzzyHash.HASH_ALGORITHM, CompareFuzzyHash.allowableValueTLSH.getValue());
runner.setProperty(CompareFuzzyHash.ATTRIBUTE_NAME, "fuzzyhash.value");
runner.setProperty(CompareFuzzyHash.HASH_LIST_FILE, "src/test/resources/empty.list");
runner.setProperty(CompareFuzzyHash.MATCH_THRESHOLD, String.valueOf(matchingSimilarity));
Map<String, String> attributes = new HashMap<>();
attributes.put("fuzzyhash.value", "Test test test chocolate");
runner.enqueue("bogus".getBytes(), attributes);
runner.run();
runner.assertQueueEmpty();
runner.assertAllFlowFilesTransferred(CompareFuzzyHash.REL_FAILURE, 1);
final MockFlowFile outFile = runner.getFlowFilesForRelationship(CompareFuzzyHash.REL_FAILURE).get(0);
outFile.assertAttributeNotExists("fuzzyhash.value.0.match");
}
@Test
public void testMissingAttribute() {
double matchingSimilarity = 200;
runner.setProperty(CompareFuzzyHash.HASH_ALGORITHM, CompareFuzzyHash.allowableValueTLSH.getValue());
runner.setProperty(CompareFuzzyHash.ATTRIBUTE_NAME, "fuzzyhash.value");
runner.setProperty(CompareFuzzyHash.HASH_LIST_FILE, "src/test/resources/empty.list");
runner.setProperty(CompareFuzzyHash.MATCH_THRESHOLD, String.valueOf(matchingSimilarity));
runner.setProperty(CompareFuzzyHash.MATCHING_MODE, CompareFuzzyHash.multiMatch.getValue());
runner.enqueue("bogus".getBytes());
runner.run();
runner.assertQueueEmpty();
runner.assertAllFlowFilesTransferred(CompareFuzzyHash.REL_FAILURE, 1);
final MockFlowFile outFile = runner.getFlowFilesForRelationship(CompareFuzzyHash.REL_FAILURE).get(0);
outFile.assertAttributeNotExists("fuzzyhash.value.0.match");
}
@Test
public void testAttributeIsEmptyString() {
double matchingSimilarity = 200;
runner.setProperty(CompareFuzzyHash.HASH_ALGORITHM, CompareFuzzyHash.allowableValueTLSH.getValue());
runner.setProperty(CompareFuzzyHash.ATTRIBUTE_NAME, "fuzzyhash.value");
runner.setProperty(CompareFuzzyHash.HASH_LIST_FILE, "src/test/resources/empty.list");
runner.setProperty(CompareFuzzyHash.MATCH_THRESHOLD, String.valueOf(matchingSimilarity));
runner.setProperty(CompareFuzzyHash.MATCHING_MODE, CompareFuzzyHash.multiMatch.getValue());
Map<String, String> attributes = new HashMap<>();
attributes.put("fuzzyhash.value", "");
runner.enqueue("bogus".getBytes(), attributes);
runner.run();
runner.assertQueueEmpty();
runner.assertAllFlowFilesTransferred(CompareFuzzyHash.REL_FAILURE, 1);
final MockFlowFile outFile = runner.getFlowFilesForRelationship(CompareFuzzyHash.REL_FAILURE).get(0);
outFile.assertAttributeNotExists("fuzzyhash.value.0.match");
}
@Test
public void testlooksLikeSpamSum() {
FuzzyHashMatcher matcher = new SSDeepHashMatcher();
List<String> invalidPayloads = Arrays.asList(
"4AD:c1xs8Z/m6H0eRH31S8p8bHENANkPrNy4tkPytwPyh2jTytxPythPytNdPytDgYyF:OuO/mg3HFSRHEb44RNMi6uHU2hcq3", // invalidFirstField
":c1xs8Z/m6H0eRH31S8p8bHENANkPrNy4tkPytwPyh2jTytxPythPytNdPytDgYyF:OuO/mg3HFSRHEb44RNMi6uHU2hcq3", // emptyFirstField
"48::OuO/mg3HFSRHEb44RNMi6uHU2hcq3", // emptySecondField
"48:c1xs8Z/m6H0eRH31S8p8bHENANkPrNy4tkPytwPyh2jTytxPythPytNdPytDgYyF:", // emptyThirdField
"48:c1xs8Z/m6H0eRH31S8p8bHENANkPrNy4tkPytwPyh2jTytxPythPytNdPytDgYyF", // withoutThirdField
"c1xs8Z/m6H0eRH31S8p8bHENANkPrNy4tkPytwPyh2jTytxPythPytNdPytDgYyF" // Just a simple string
);
for (String item : invalidPayloads) {
assertTrue(!matcher.isValidHash(item), "item '" + item + "' should have failed validation");
}
// Now test with a valid string
assertTrue(matcher.isValidHash(ssdeepInput));
}
}

97
nifi-nar-bundles/nifi-cybersecurity-bundle/nifi-cybersecurity-processors/src/test/java/org/apache/nifi/processors/cybersecurity/TestFuzzyHashContent.java
View File

@ -1,97 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.nifi.processors.cybersecurity;
import org.apache.nifi.util.MockFlowFile;
import org.apache.nifi.util.TestRunner;
import org.apache.nifi.util.TestRunners;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import static org.junit.jupiter.api.Assertions.assertEquals;
public class TestFuzzyHashContent {
private TestRunner runner;
@BeforeEach
public void init() {
runner = TestRunners.newTestRunner(FuzzyHashContent.class);
}
@AfterEach
public void stop() {
runner.shutdown();
}
@Test
public void testSsdeepFuzzyHashContent() {
runner.setProperty(FuzzyHashContent.HASH_ALGORITHM, FuzzyHashContent.allowableValueSSDEEP.getValue());
runner.enqueue("This is the a short and meaningless sample taste of 'test test test chocolate' " +
"an odd test string that is used within some of the NiFi test units. Once day the author of " +
"such strings may decide to tell the history behind this sentence and its true meaning...\n");
runner.run();
runner.assertQueueEmpty();
runner.assertAllFlowFilesTransferred(FuzzyHashContent.REL_SUCCESS, 1);
final MockFlowFile outFile = runner.getFlowFilesForRelationship(FuzzyHashContent.REL_SUCCESS).get(0);
assertEquals("6:hERjIfhRrlB63J0FDw1NBQmEH68xwMSELN:hZrlB62IwMS",outFile.getAttribute("fuzzyhash.value") );
}
@Test
public void testTLSHFuzzyHashInvalidContent() {
runner.setProperty(FuzzyHashContent.HASH_ALGORITHM, FuzzyHashContent.allowableValueTLSH.getValue());
runner.enqueue("This is the a short and meaningless sample taste of 'test test test chocolate' " +
"an odd test string that is used within some of the NiFi test units. Once day the author of " +
"such strings may decide to tell the history behind this sentence and its true meaning...\n");
runner.run();
runner.assertQueueEmpty();
runner.assertAllFlowFilesTransferred(FuzzyHashContent.REL_FAILURE, 1);
final MockFlowFile outFile = runner.getFlowFilesForRelationship(FuzzyHashContent.REL_FAILURE).get(0);
outFile.assertAttributeNotExists("fuzzyhash.value");
}
@Test
public void testTLSHFuzzyHashValidContent() {
runner.setProperty(FuzzyHashContent.HASH_ALGORITHM, FuzzyHashContent.allowableValueTLSH.getValue());
runner.enqueue("This is the a short and meaningless sample taste of 'test test test chocolate' " +
"an odd test string that is used within some of the NiFi test units. Once day the author of " +
"such strings may decide to tell the history behind this sentence and its true meaning...\n" +
"What is certain however, is that this section of the test unit requires at least 512 " +
"characters of data to produce the expect results.\n " +
"And yet... despite all the senseless verbosity, I still have to continue writing these somewhat " +
"meaningless words that do nothing but to remind all of us of Ipsum Lorem..." );
runner.run();
runner.assertQueueEmpty();
runner.assertAllFlowFilesTransferred(FuzzyHashContent.REL_SUCCESS, 1);
final MockFlowFile outFile = runner.getFlowFilesForRelationship(FuzzyHashContent.REL_SUCCESS).get(0);
assertEquals("E2F0818B7AE7173906A72221570E30979B11C0FC47B518A1E89D257E2343CEC02381ED",
outFile.getAttribute("fuzzyhash.value"));
}
}

1
nifi-nar-bundles/nifi-cybersecurity-bundle/nifi-cybersecurity-processors/src/test/resources/blank_ssdeep.list
View File

@ -1 +0,0 @@
ssdeep,1.0--blocksize:hash:hash,filename

0
nifi-nar-bundles/nifi-cybersecurity-bundle/nifi-cybersecurity-processors/src/test/resources/empty.list
View File

11
nifi-nar-bundles/nifi-cybersecurity-bundle/nifi-cybersecurity-processors/src/test/resources/ssdeep.list
View File

@ -1,11 +0,0 @@
ssdeep,1.0--blocksize:hash:hash,filename
96:KQhaGCVZGhr83h3bc0ok3892m12wzgnH5w2pw+sxNEI58:FIVkH4x73h39LH+2w+sxaD,"config.h"
96:EQOJvOl4ab3hhiNFXc4wwcweomr0cNJDBoqXjmAHKX8dEt001nfEhVIuX0dDcs:3mzpAsZpprbshfu3oujjdENdp21,"doc\README"
48:c1xs8Z/m6H0eRH31S8p8bHENRNkPSNy4tkPytwPytyYytxPythPytNdPytDgYyse:OuO/mg3HFSRHE+H4RNc6uHU2hqoMkh
96,MD9fHjsEuddrg31904l8bgx5ROg2MQZHZqpAlycowOsexbHDbk:MJwz/l2PqGqqbr2yk6pVgrwPV,"Whatever.txt-INVALID-DUE-TO-COMMA-AFTER-96"
48:c1xs8Z/m6H0eRH31S8p8bHENANkPrNy4tkPytwPyh2jTytxPythPytNdPytDgYyF:OuO/mg3HFSRHEb44RNMi6uHU2hcq3,"nifi/nifi-nar-bundles/nifi-beats-bundle/nifi-beats-processors/pom.xml"
6:hERjIfhRrlB63J0FDw1NBQmEH68xwMSELN:hZrlB62IwMS,"c:\this_is_valid_but_should_not_match"
96:MD9fHjsEuddrg31904l8bgx5ROg2MQZHZqpAlycowOsexbHDbk:MJwz/l2PqGqqbr2yk6pVgrwPV,"INSTALL"
48:c1xs8Z/m6H0eRH31S8p8bHENRNkPSNy4tkPytwPytyYytxPythPytNdPytDgYyse:OuO/mg3HFSRHE+H4RNc6uHU2hqoMkh,"nifi/nifi-nar-bundles/nifi-lumberjack-bundle/nifi-lumberjack-processors/pom.xml"
48:c1xs8Z/m6H0eRH31S8p8bHENRNkPSNy4tkPytwPytyYytxPythPytNdPytDgYyse:OuO/mg3HFSRHE+H4RNc6uHU2hqoMkh,

9
nifi-nar-bundles/nifi-cybersecurity-bundle/nifi-cybersecurity-processors/src/test/resources/tlsh.list
View File

@ -1,9 +0,0 @@
A4518DA4A8F9517162A409C1DEEA9872AF55C137E00A62C9F0CDD0CE4F6CCD784BB4B7 nifi-nar-bundles/nifi-lumberjack-bundle/nifi-lumberjack-processors/pom.xml
THERE SEEMS TO BE SOMETHING MISSING
6FF02BEF718027B0160B4391212923ED7F1A463D563B1549B86CF62973B197AD2731F Synthetic shorter-INVALID
EB519EA4A8F95171A2A409C1DEEB9872AF55C137E00A5289F1CCD0CE4F6CCD784BB4B7
E2F0818B7AE7173906A72221570E30979B11C0FC47B518A1E89D257E2343CEC02381ED /this/is/also/valid/but/should/not/match
EB519EA4A8F95171A2A409C1DEEB9872AF55C137E00A5289F1CCD0CE4F6CCD784BB4B7 nifi-nar-bundles/nifi-beats-bundle/nifi-beats-processors/pom.xml
EB519EA4A8F95171A2A409C1DEEB9872AF55C137E00A5289F1CCD0CE4F6CCD784BB4B7
6FF02BEF718027B0160B4391212923ED7F1A463D563B1549B86CF62973B197AD2731F8 /this/is/valid/but/should/not/match

32
nifi-nar-bundles/nifi-cybersecurity-bundle/pom.xml
View File

@ -1,32 +0,0 @@
<?xml version="1.0" encoding="UTF-8"?>
<!--
Licensed to the Apache Software Foundation (ASF) under one or more
contributor license agreements. See the NOTICE file distributed with
this work for additional information regarding copyright ownership.
The ASF licenses this file to You under the Apache License, Version 2.0
(the "License"); you may not use this file except in compliance with
the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
-->
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>org.apache.nifi</groupId>
<artifactId>nifi-nar-bundles</artifactId>
<version>2.0.0-SNAPSHOT</version>
</parent>
<artifactId>nifi-cybersecurity-bundle</artifactId>
<packaging>pom</packaging>
<modules>
<module>nifi-cybersecurity-processors</module>
<module>nifi-cybersecurity-nar</module>
</modules>
</project>

44
nifi-nar-bundles/nifi-html-bundle/nifi-html-nar/pom.xml
View File

@ -1,44 +0,0 @@
<?xml version="1.0" encoding="UTF-8"?>
<!--
Licensed to the Apache Software Foundation (ASF) under one or more
contributor license agreements. See the NOTICE file distributed with
this work for additional information regarding copyright ownership.
The ASF licenses this file to You under the Apache License, Version 2.0
(the "License"); you may not use this file except in compliance with
the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
-->
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>org.apache.nifi</groupId>
<artifactId>nifi-html-bundle</artifactId>
<version>2.0.0-SNAPSHOT</version>
</parent>
<artifactId>nifi-html-nar</artifactId>
<packaging>nar</packaging>
<properties>
<maven.javadoc.skip>true</maven.javadoc.skip>
<source.skip>true</source.skip>
</properties>
<dependencies>
<dependency>
<groupId>org.apache.nifi</groupId>
<artifactId>nifi-standard-services-api-nar</artifactId>
<version>2.0.0-SNAPSHOT</version>
<type>nar</type>
</dependency>
<dependency>
<groupId>org.apache.nifi</groupId>
<artifactId>nifi-html-processors</artifactId>
</dependency>
</dependencies>
</project>

240
nifi-nar-bundles/nifi-html-bundle/nifi-html-nar/src/main/resources/META-INF/LICENSE
View File

@ -1,240 +0,0 @@
Apache License
Version 2.0, January 2004
http://www.apache.org/licenses/
TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
1. Definitions.
"License" shall mean the terms and conditions for use, reproduction,
and distribution as defined by Sections 1 through 9 of this document.
"Licensor" shall mean the copyright owner or entity authorized by
the copyright owner that is granting the License.
"Legal Entity" shall mean the union of the acting entity and all
other entities that control, are controlled by, or are under common
control with that entity. For the purposes of this definition,
"control" means (i) the power, direct or indirect, to cause the
direction or management of such entity, whether by contract or
otherwise, or (ii) ownership of fifty percent (50%) or more of the
outstanding shares, or (iii) beneficial ownership of such entity.
"You" (or "Your") shall mean an individual or Legal Entity
exercising permissions granted by this License.
"Source" form shall mean the preferred form for making modifications,
including but not limited to software source code, documentation
source, and configuration files.
"Object" form shall mean any form resulting from mechanical
transformation or translation of a Source form, including but
not limited to compiled object code, generated documentation,
and conversions to other media types.
"Work" shall mean the work of authorship, whether in Source or
Object form, made available under the License, as indicated by a
copyright notice that is included in or attached to the work
(an example is provided in the Appendix below).
"Derivative Works" shall mean any work, whether in Source or Object
form, that is based on (or derived from) the Work and for which the
editorial revisions, annotations, elaborations, or other modifications
represent, as a whole, an original work of authorship. For the purposes
of this License, Derivative Works shall not include works that remain
separable from, or merely link (or bind by name) to the interfaces of,
the Work and Derivative Works thereof.
"Contribution" shall mean any work of authorship, including
the original version of the Work and any modifications or additions
to that Work or Derivative Works thereof, that is intentionally
submitted to Licensor for inclusion in the Work by the copyright owner
or by an individual or Legal Entity authorized to submit on behalf of
the copyright owner. For the purposes of this definition, "submitted"
means any form of electronic, verbal, or written communication sent
to the Licensor or its representatives, including but not limited to
communication on electronic mailing lists, source code control systems,
and issue tracking systems that are managed by, or on behalf of, the
Licensor for the purpose of discussing and improving the Work, but
excluding communication that is conspicuously marked or otherwise
designated in writing by the copyright owner as "Not a Contribution."
"Contributor" shall mean Licensor and any individual or Legal Entity
on behalf of whom a Contribution has been received by Licensor and
subsequently incorporated within the Work.
2. Grant of Copyright License. Subject to the terms and conditions of
this License, each Contributor hereby grants to You a perpetual,
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
copyright license to reproduce, prepare Derivative Works of,
publicly display, publicly perform, sublicense, and distribute the
Work and such Derivative Works in Source or Object form.
3. Grant of Patent License. Subject to the terms and conditions of
this License, each Contributor hereby grants to You a perpetual,
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
(except as stated in this section) patent license to make, have made,
use, offer to sell, sell, import, and otherwise transfer the Work,
where such license applies only to those patent claims licensable
by such Contributor that are necessarily infringed by their
Contribution(s) alone or by combination of their Contribution(s)
with the Work to which such Contribution(s) was submitted. If You
institute patent litigation against any entity (including a
cross-claim or counterclaim in a lawsuit) alleging that the Work
or a Contribution incorporated within the Work constitutes direct
or contributory patent infringement, then any patent licenses
granted to You under this License for that Work shall terminate
as of the date such litigation is filed.
4. Redistribution. You may reproduce and distribute copies of the
Work or Derivative Works thereof in any medium, with or without
modifications, and in Source or Object form, provided that You
meet the following conditions:
(a) You must give any other recipients of the Work or
Derivative Works a copy of this License; and
(b) You must cause any modified files to carry prominent notices
stating that You changed the files; and
(c) You must retain, in the Source form of any Derivative Works
that You distribute, all copyright, patent, trademark, and
attribution notices from the Source form of the Work,
excluding those notices that do not pertain to any part of
the Derivative Works; and
(d) If the Work includes a "NOTICE" text file as part of its
distribution, then any Derivative Works that You distribute must
include a readable copy of the attribution notices contained
within such NOTICE file, excluding those notices that do not
pertain to any part of the Derivative Works, in at least one
of the following places: within a NOTICE text file distributed
as part of the Derivative Works; within the Source form or
documentation, if provided along with the Derivative Works; or,
within a display generated by the Derivative Works, if and
wherever such third-party notices normally appear. The contents
of the NOTICE file are for informational purposes only and
do not modify the License. You may add Your own attribution
notices within Derivative Works that You distribute, alongside
or as an addendum to the NOTICE text from the Work, provided
that such additional attribution notices cannot be construed
as modifying the License.
You may add Your own copyright statement to Your modifications and
may provide additional or different license terms and conditions
for use, reproduction, or distribution of Your modifications, or
for any such Derivative Works as a whole, provided Your use,
reproduction, and distribution of the Work otherwise complies with
the conditions stated in this License.
5. Submission of Contributions. Unless You explicitly state otherwise,
any Contribution intentionally submitted for inclusion in the Work
by You to the Licensor shall be under the terms and conditions of
this License, without any additional terms or conditions.
Notwithstanding the above, nothing herein shall supersede or modify
the terms of any separate license agreement you may have executed
with Licensor regarding such Contributions.
6. Trademarks. This License does not grant permission to use the trade
names, trademarks, service marks, or product names of the Licensor,
except as required for reasonable and customary use in describing the
origin of the Work and reproducing the content of the NOTICE file.
7. Disclaimer of Warranty. Unless required by applicable law or
agreed to in writing, Licensor provides the Work (and each
Contributor provides its Contributions) on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
implied, including, without limitation, any warranties or conditions
of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
PARTICULAR PURPOSE. You are solely responsible for determining the
appropriateness of using or redistributing the Work and assume any
risks associated with Your exercise of permissions under this License.
8. Limitation of Liability. In no event and under no legal theory,
whether in tort (including negligence), contract, or otherwise,
unless required by applicable law (such as deliberate and grossly
negligent acts) or agreed to in writing, shall any Contributor be
liable to You for damages, including any direct, indirect, special,
incidental, or consequential damages of any character arising as a
result of this License or out of the use or inability to use the
Work (including but not limited to damages for loss of goodwill,
work stoppage, computer failure or malfunction, or any and all
other commercial damages or losses), even if such Contributor
has been advised of the possibility of such damages.
9. Accepting Warranty or Additional Liability. While redistributing
the Work or Derivative Works thereof, You may choose to offer,
and charge a fee for, acceptance of support, warranty, indemnity,
or other liability obligations and/or rights consistent with this
License. However, in accepting such obligations, You may act only
on Your own behalf and on Your sole responsibility, not on behalf
of any other Contributor, and only if You agree to indemnify,
defend, and hold each Contributor harmless for any liability
incurred by, or claims asserted against, such Contributor by reason
of your accepting any such warranty or additional liability.
END OF TERMS AND CONDITIONS
APPENDIX: How to apply the Apache License to your work.
To apply the Apache License to your work, attach the following
boilerplate notice, with the fields enclosed by brackets "[]"
replaced with your own identifying information. (Don't include
the brackets!) The text should be enclosed in the appropriate
comment syntax for the file format. We also recommend that a
file or class name and description of purpose be included on the
same "printed page" as the copyright notice for easier
identification within third-party archives.
Copyright [yyyy] [name of copyright owner]
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
APACHE NIFI SUBCOMPONENTS:
The Apache NiFi project contains subcomponents with separate copyright
notices and license terms. Your use of the source code for the these
subcomponents is subject to the terms and conditions of the following
licenses.
This product bundles 'jsoup' which is available under the MIT License.
For details see http://jsoup.org/
jsoup License
The jsoup code-base (include source and compiled packages) are distributed under the open source MIT license as described below.
The MIT License
Copyright © 2009 - 2013 Jonathan Hedley (jonathan@hedley.net)
Permission is hereby granted, free of charge, to any person
obtaining a copy of this software and associated documentation
files (the "Software"), to deal in the Software without
restriction, including without limitation the rights to use,
copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the
Software is furnished to do so, subject to the following
conditions:
The above copyright notice and this permission notice shall be
included in all copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
OTHER DEALINGS IN THE SOFTWARE.

19
nifi-nar-bundles/nifi-html-bundle/nifi-html-nar/src/main/resources/META-INF/NOTICE
View File

@ -1,19 +0,0 @@
nifi-html-nar
Copyright 2015-2020 The Apache Software Foundation
This product includes software developed at
The Apache Software Foundation (http://www.apache.org/).
******************
Apache Software License v2
******************
The following binary components are provided under the Apache Software License v2
(ASLv2) Apache Commons Lang
The following NOTICE information applies:
Apache Commons Lang
Copyright 2001-2014 The Apache Software Foundation
This product includes software from the Spring Framework,
under the Apache License 2.0 (see: StringUtils.containsWhitespace())

67
nifi-nar-bundles/nifi-html-bundle/nifi-html-processors/pom.xml
View File

@ -1,67 +0,0 @@
<?xml version="1.0" encoding="UTF-8"?>
<!--
Licensed to the Apache Software Foundation (ASF) under one or more
contributor license agreements. See the NOTICE file distributed with
this work for additional information regarding copyright ownership.
The ASF licenses this file to You under the Apache License, Version 2.0
(the "License"); you may not use this file except in compliance with
the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
-->
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>org.apache.nifi</groupId>
<artifactId>nifi-html-bundle</artifactId>
<version>2.0.0-SNAPSHOT</version>
</parent>
<artifactId>nifi-html-processors</artifactId>
<description>Support for parsing HTML documents</description>
<dependencies>
<dependency>
<groupId>org.jsoup</groupId>
<artifactId>jsoup</artifactId>
</dependency>
<dependency>
<groupId>org.apache.commons</groupId>
<artifactId>commons-lang3</artifactId>
</dependency>
<dependency>
<groupId>org.apache.nifi</groupId>
<artifactId>nifi-api</artifactId>
</dependency>
<dependency>
<groupId>org.apache.nifi</groupId>
<artifactId>nifi-utils</artifactId>
<version>2.0.0-SNAPSHOT</version>
</dependency>
<dependency>
<groupId>org.apache.nifi</groupId>
<artifactId>nifi-mock</artifactId>
<version>2.0.0-SNAPSHOT</version>
<scope>test</scope>
</dependency>
</dependencies>
<build>
<plugins>
<plugin>
<groupId>org.apache.rat</groupId>
<artifactId>apache-rat-plugin</artifactId>
<configuration>
<excludes combine.children="append">
<exclude>src/test/resources/Weather.html</exclude>
</excludes>
</configuration>
</plugin>
</plugins>
</build>
</project>

143
nifi-nar-bundles/nifi-html-bundle/nifi-html-processors/src/main/java/org/apache/nifi/AbstractHTMLProcessor.java
View File

@ -1,143 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.nifi;
import org.apache.nifi.components.PropertyDescriptor;
import org.apache.nifi.components.ValidationContext;
import org.apache.nifi.components.ValidationResult;
import org.apache.nifi.components.Validator;
import org.apache.nifi.expression.ExpressionLanguageScope;
import org.apache.nifi.flowfile.FlowFile;
import org.apache.nifi.processor.AbstractProcessor;
import org.apache.nifi.processor.ProcessContext;
import org.apache.nifi.processor.ProcessSession;
import org.apache.nifi.processor.Relationship;
import org.apache.nifi.processor.io.InputStreamCallback;
import org.apache.nifi.processor.util.StandardValidators;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.select.Selector;
import java.io.IOException;
import java.io.InputStream;
import java.util.concurrent.atomic.AtomicReference;
public abstract class AbstractHTMLProcessor extends AbstractProcessor {
protected static final String ELEMENT_HTML = "HTML";
protected static final String ELEMENT_TEXT = "Text";
protected static final String ELEMENT_DATA = "Data";
protected static final String ELEMENT_ATTRIBUTE = "Attribute";
protected static final Validator CSS_SELECTOR_VALIDATOR = new Validator() {
@Override
public ValidationResult validate(final String subject, final String value, final ValidationContext context) {
if (context.isExpressionLanguageSupported(subject) && context.isExpressionLanguagePresent(value)) {
return new ValidationResult.Builder().subject(subject).input(value).explanation("Expression Language Present").valid(true).build();
}
String reason = null;
try {
Document doc = Jsoup.parse("<html></html>");
doc.select(value);
} catch (final Selector.SelectorParseException e) {
reason = "\"" + value + "\" is an invalid CSS selector";
}
return new ValidationResult.Builder().subject(subject).input(value).explanation(reason).valid(reason == null).build();
}
};
public static final PropertyDescriptor URL = new PropertyDescriptor
.Builder().name("URL")
.description("Base URL for the HTML page being parsed." +
" This URL will be used to resolve an absolute URL" +
" when an attribute value is extracted from a HTML element.")
.required(true)
.addValidator(StandardValidators.NON_EMPTY_VALIDATOR)
.expressionLanguageSupported(ExpressionLanguageScope.FLOWFILE_ATTRIBUTES)
.build();
public static final PropertyDescriptor CSS_SELECTOR = new PropertyDescriptor
.Builder().name("CSS Selector")
.description("CSS selector syntax string used to extract the desired HTML element(s).")
.required(true)
.addValidator(CSS_SELECTOR_VALIDATOR)
.expressionLanguageSupported(ExpressionLanguageScope.FLOWFILE_ATTRIBUTES)
.build();
public static final PropertyDescriptor HTML_CHARSET = new PropertyDescriptor
.Builder().name("HTML Character Encoding")
.description("Character encoding of the input HTML")
.defaultValue("UTF-8")
.required(true)
.addValidator(StandardValidators.CHARACTER_SET_VALIDATOR)
.build();
public static final Relationship REL_ORIGINAL = new Relationship.Builder()
.name("original")
.description("The original HTML input")
.build();
public static final Relationship REL_SUCCESS = new Relationship.Builder()
.name("success")
.description("Successfully parsed HTML element")
.build();
public static final Relationship REL_INVALID_HTML = new Relationship.Builder()
.name("invalid html")
.description("The input HTML syntax is invalid")
.build();
public static final Relationship REL_NOT_FOUND = new Relationship.Builder()
.name("element not found")
.description("Element could not be found in the HTML document. The original HTML input will remain " +
"in the FlowFile content unchanged. Relationship '" + REL_ORIGINAL + "' will not be invoked " +
"in this scenario.")
.build();
/**
* Parses the Jsoup HTML document from the FlowFile input content.
*
* @param inputFlowFile Input FlowFile containing the HTML
* @param context ProcessContext
* @param session ProcessSession
*
* @return Jsoup Document
*/
protected Document parseHTMLDocumentFromFlowfile(final FlowFile inputFlowFile, final ProcessContext context, final ProcessSession session) {
final AtomicReference<Document> doc = new AtomicReference<>();
session.read(inputFlowFile, new InputStreamCallback() {
@Override
public void process(InputStream inputStream) throws IOException {
final String baseUrl = getBaseUrl(inputFlowFile, context);
if (baseUrl == null || baseUrl.isEmpty()) {
throw new RuntimeException("Base URL was empty.");
}
doc.set(Jsoup.parse(inputStream,
context.getProperty(HTML_CHARSET).getValue(),
baseUrl));
}
});
return doc.get();
}
protected String getBaseUrl(final FlowFile inputFlowFile, final ProcessContext context) {
return "http://localhost/";
}
}

247
nifi-nar-bundles/nifi-html-bundle/nifi-html-processors/src/main/java/org/apache/nifi/GetHTMLElement.java
View File

@ -1,247 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.nifi;
import org.apache.commons.lang3.StringUtils;
import org.apache.nifi.annotation.behavior.InputRequirement;
import org.apache.nifi.annotation.behavior.SupportsBatching;
import org.apache.nifi.components.PropertyDescriptor;
import org.apache.nifi.expression.ExpressionLanguageScope;
import org.apache.nifi.flowfile.FlowFile;
import org.apache.nifi.processor.ProcessContext;
import org.apache.nifi.processor.ProcessSession;
import org.apache.nifi.processor.Relationship;
import org.apache.nifi.processor.ProcessorInitializationContext;
import org.apache.nifi.annotation.behavior.WritesAttribute;
import org.apache.nifi.annotation.behavior.WritesAttributes;
import org.apache.nifi.annotation.documentation.CapabilityDescription;
import org.apache.nifi.annotation.documentation.SeeAlso;
import org.apache.nifi.annotation.documentation.Tags;
import org.apache.nifi.processor.exception.ProcessException;
import org.apache.nifi.processor.io.StreamCallback;
import org.apache.nifi.processor.util.StandardValidators;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
import java.util.List;
import java.util.Set;
import java.util.HashSet;
import java.util.Collections;
@SupportsBatching
@Tags({"get", "html", "dom", "css", "element"})
@InputRequirement(InputRequirement.Requirement.INPUT_REQUIRED)
@CapabilityDescription("Extracts HTML element values from the incoming flowfile's content using a CSS selector." +
" The incoming HTML is first converted into a HTML Document Object Model so that HTML elements may be selected" +
" in the similar manner that CSS selectors are used to apply styles to HTML. The resulting HTML DOM is then \"queried\"" +
" using the user defined CSS selector string. The result of \"querying\" the HTML DOM may produce 0-N results." +
" If no results are found the flowfile will be transferred to the \"element not found\" relationship to indicate" +
" so to the end user. If N results are found a new flowfile will be created and emitted for each result. The query result will" +
" either be placed in the content of the new flowfile or as an attribute of the new flowfile. By default the result is written to an" +
" attribute. This can be controlled by the \"Destination\" property. Resulting query values may also have data" +
" prepended or appended to them by setting the value of property \"Prepend Element Value\" or \"Append Element Value\"." +
" Prepended and appended values are treated as string values and concatenated to the result retrieved from the" +
" HTML DOM query operation. A more thorough reference for the CSS selector syntax can be found at" +
" \"http://jsoup.org/apidocs/org/jsoup/select/Selector.html\"")
@SeeAlso({ModifyHTMLElement.class, PutHTMLElement.class})
@WritesAttributes({@WritesAttribute(attribute="HTMLElement", description="Flowfile attribute where the element result" +
" parsed from the HTML using the CSS selector syntax are placed if the destination is a flowfile attribute.")})
public class GetHTMLElement
extends AbstractHTMLProcessor {
public static final String HTML_ELEMENT_ATTRIBUTE_NAME = "HTMLElement";
public static final String DESTINATION_ATTRIBUTE = "flowfile-attribute";
public static final String DESTINATION_CONTENT = "flowfile-content";
public static final PropertyDescriptor PREPEND_ELEMENT_VALUE = new PropertyDescriptor
.Builder().name("Prepend Element Value")
.description("Prepends the specified value to the resulting Element")
.required(false)
.addValidator(StandardValidators.NON_EMPTY_VALIDATOR)
.expressionLanguageSupported(ExpressionLanguageScope.FLOWFILE_ATTRIBUTES)
.build();
public static final PropertyDescriptor APPEND_ELEMENT_VALUE = new PropertyDescriptor
.Builder().name("Append Element Value")
.description("Appends the specified value to the resulting Element")
.required(false)
.addValidator(StandardValidators.NON_EMPTY_VALIDATOR)
.expressionLanguageSupported(ExpressionLanguageScope.FLOWFILE_ATTRIBUTES)
.build();
public static final PropertyDescriptor ATTRIBUTE_KEY = new PropertyDescriptor
.Builder().name("Attribute Name")
.description(("When getting the value of a HTML element attribute this value is used as the key to determine" +
" which attribute on the selected element should be retrieved. This value is used when the \"Output Type\"" +
" is set to \"" + ELEMENT_ATTRIBUTE + "\"." +
" If this value is prefixed with 'abs:', then the extracted attribute value will be converted into" +
" an absolute URL form using the specified base URL."))
.required(false)
.addValidator(StandardValidators.NON_EMPTY_VALIDATOR)
.expressionLanguageSupported(ExpressionLanguageScope.FLOWFILE_ATTRIBUTES)
.build();
public static final PropertyDescriptor OUTPUT_TYPE = new PropertyDescriptor.Builder()
.name("Output Type")
.description("Controls the type of DOM value that is retrieved from the HTML element.")
.required(true)
.addValidator(StandardValidators.NON_EMPTY_VALIDATOR)
.allowableValues(ELEMENT_HTML, ELEMENT_TEXT, ELEMENT_ATTRIBUTE, ELEMENT_DATA)
.defaultValue(ELEMENT_HTML)
.build();
public static final PropertyDescriptor DESTINATION = new PropertyDescriptor.Builder()
.name("Destination")
.description("Control if element extracted is written as a flowfile attribute or " +
"as flowfile content.")
.required(true)
.allowableValues(DESTINATION_ATTRIBUTE, DESTINATION_CONTENT)
.defaultValue(DESTINATION_ATTRIBUTE)
.build();
private List<PropertyDescriptor> descriptors;
private Set<Relationship> relationships;
@Override
protected void init(final ProcessorInitializationContext context) {
final List<PropertyDescriptor> descriptors = new ArrayList<>();
descriptors.add(URL);
descriptors.add(CSS_SELECTOR);
descriptors.add(HTML_CHARSET);
descriptors.add(OUTPUT_TYPE);
descriptors.add(DESTINATION);
descriptors.add(PREPEND_ELEMENT_VALUE);
descriptors.add(APPEND_ELEMENT_VALUE);
descriptors.add(ATTRIBUTE_KEY);
this.descriptors = Collections.unmodifiableList(descriptors);
final Set<Relationship> relationships = new HashSet<>();
relationships.add(REL_ORIGINAL);
relationships.add(REL_SUCCESS);
relationships.add(REL_INVALID_HTML);
relationships.add(REL_NOT_FOUND);
this.relationships = Collections.unmodifiableSet(relationships);
}
@Override
public Set<Relationship> getRelationships() {
return this.relationships;
}
@Override
public final List<PropertyDescriptor> getSupportedPropertyDescriptors() {
return descriptors;
}
@Override
public void onTrigger(final ProcessContext context, final ProcessSession session) throws ProcessException {
final FlowFile flowFile = session.get();
if ( flowFile == null ) {
return;
}
final Document doc;
final Elements eles;
try {
doc = parseHTMLDocumentFromFlowfile(flowFile, context, session);
eles = doc.select(context.getProperty(CSS_SELECTOR).evaluateAttributeExpressions(flowFile).getValue());
} catch (final Exception ex) {
getLogger().error("Failed to extract HTML from {} due to {}; routing to {}", flowFile, ex, REL_INVALID_HTML, ex);
session.transfer(flowFile, REL_INVALID_HTML);
return;
}
final String prependValue = context.getProperty(PREPEND_ELEMENT_VALUE).evaluateAttributeExpressions(flowFile).getValue();
final String appendValue = context.getProperty(APPEND_ELEMENT_VALUE).evaluateAttributeExpressions(flowFile).getValue();
final String outputType = context.getProperty(OUTPUT_TYPE).getValue();
final String attributeKey = context.getProperty(ATTRIBUTE_KEY).evaluateAttributeExpressions(flowFile).getValue();
if (eles == null || eles.isEmpty()) {
// No element found
session.transfer(flowFile, REL_NOT_FOUND);
} else {
// Create a new FlowFile for each matching element.
for (final Element ele : eles) {
final String extractedElementValue = extractElementValue(prependValue, outputType, appendValue, ele, attributeKey);
final FlowFile ff = session.create(flowFile);
FlowFile updatedFF = ff;
switch (context.getProperty(DESTINATION).getValue()) {
case DESTINATION_ATTRIBUTE:
updatedFF = session.putAttribute(ff, HTML_ELEMENT_ATTRIBUTE_NAME, extractedElementValue);
break;
case DESTINATION_CONTENT:
updatedFF = session.write(ff, new StreamCallback() {
@Override
public void process(final InputStream inputStream, final OutputStream outputStream) throws IOException {
outputStream.write(extractedElementValue.getBytes(StandardCharsets.UTF_8));
}
});
break;
}
session.transfer(updatedFF, REL_SUCCESS);
}
// Transfer the original HTML
session.transfer(flowFile, REL_ORIGINAL);
}
}
/**
* Extracts the HTML value based on the configuration values.
*
* @return value from the parsed HTML element
*/
private String extractElementValue(String prependValue, final String outputType, String appendValue, final Element ele, final String attrKey) {
if (StringUtils.isEmpty(prependValue)) {
prependValue = "";
}
if (StringUtils.isEmpty(appendValue)) {
appendValue = "";
}
switch (outputType) {
case ELEMENT_HTML:
return prependValue + ele.html() + appendValue;
case ELEMENT_TEXT:
return prependValue + ele.text() + appendValue;
case ELEMENT_DATA:
return prependValue + ele.data() + appendValue;
case ELEMENT_ATTRIBUTE:
return prependValue + ele.attr(attrKey) + appendValue;
default:
return prependValue + ele.html() + appendValue;
}
}
@Override
protected String getBaseUrl(FlowFile inputFlowFile, ProcessContext context) {
return context.getProperty(URL).evaluateAttributeExpressions(inputFlowFile).getValue();
}
}

189
nifi-nar-bundles/nifi-html-bundle/nifi-html-processors/src/main/java/org/apache/nifi/ModifyHTMLElement.java
View File

@ -1,189 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.nifi;
import org.apache.nifi.annotation.behavior.InputRequirement;
import org.apache.nifi.annotation.behavior.SupportsBatching;
import org.apache.nifi.annotation.behavior.WritesAttribute;
import org.apache.nifi.annotation.behavior.WritesAttributes;
import org.apache.nifi.annotation.documentation.CapabilityDescription;
import org.apache.nifi.annotation.documentation.SeeAlso;
import org.apache.nifi.annotation.documentation.Tags;
import org.apache.nifi.components.PropertyDescriptor;
import org.apache.nifi.expression.ExpressionLanguageScope;
import org.apache.nifi.flowfile.FlowFile;
import org.apache.nifi.processor.ProcessContext;
import org.apache.nifi.processor.ProcessSession;
import org.apache.nifi.processor.Relationship;
import org.apache.nifi.processor.ProcessorInitializationContext;
import org.apache.nifi.processor.exception.ProcessException;
import org.apache.nifi.processor.io.StreamCallback;
import org.apache.nifi.processor.util.StandardValidators;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.nio.charset.StandardCharsets;
import java.util.List;
import java.util.ArrayList;
import java.util.Set;
import java.util.HashSet;
import java.util.Collections;
@Tags({"modify", "html", "dom", "css", "element"})
@SupportsBatching
@InputRequirement(InputRequirement.Requirement.INPUT_REQUIRED)
@CapabilityDescription("Modifies the value of an existing HTML element. The desired element to be modified is located by" +
" using CSS selector syntax. The incoming HTML is first converted into a HTML Document Object Model so that HTML elements may be selected" +
" in the similar manner that CSS selectors are used to apply styles to HTML. The resulting HTML DOM is then \"queried\"" +
" using the user defined CSS selector string to find the element the user desires to modify. If the HTML element is found" +
" the element's value is updated in the DOM using the value specified \"Modified Value\" property. All DOM elements" +
" that match the CSS selector will be updated. Once all of the DOM elements have been updated the DOM is rendered" +
" to HTML and the result replaces the flowfile content with the updated HTML. A more thorough reference for the" +
" CSS selector syntax can be found at" +
" \"http://jsoup.org/apidocs/org/jsoup/select/Selector.html\"")
@SeeAlso({GetHTMLElement.class, PutHTMLElement.class})
@WritesAttributes({@WritesAttribute(attribute="NumElementsModified", description="Total number of HTML " +
"element modifications made")})
public class ModifyHTMLElement extends AbstractHTMLProcessor {
public static final String NUM_ELEMENTS_MODIFIED_ATTR = "NumElementsModified";
public static final PropertyDescriptor OUTPUT_TYPE = new PropertyDescriptor.Builder()
.name("Output Type")
.description("Controls whether the HTML element is output as " +
ELEMENT_HTML + "," + ELEMENT_TEXT + " or " + ELEMENT_DATA)
.required(true)
.allowableValues(ELEMENT_HTML, ELEMENT_TEXT, ELEMENT_ATTRIBUTE)
.defaultValue(ELEMENT_HTML)
.build();
public static final PropertyDescriptor MODIFIED_VALUE = new PropertyDescriptor
.Builder().name("Modified Value")
.description("Value to update the found HTML elements with")
.required(true)
.addValidator(StandardValidators.NON_EMPTY_VALIDATOR)
.expressionLanguageSupported(ExpressionLanguageScope.FLOWFILE_ATTRIBUTES)
.build();
public static final PropertyDescriptor ATTRIBUTE_KEY = new PropertyDescriptor
.Builder().name("Attribute Name")
.description(("When modifying the value of an element attribute this value is used as the key to determine" +
" which attribute on the selected element will be modified with the new value."))
.required(false)
.addValidator(StandardValidators.NON_EMPTY_VALIDATOR)
.expressionLanguageSupported(ExpressionLanguageScope.FLOWFILE_ATTRIBUTES)
.build();
private List<PropertyDescriptor> descriptors;
private Set<Relationship> relationships;
@Override
protected void init(final ProcessorInitializationContext context) {
final List<PropertyDescriptor> descriptors = new ArrayList<>();
descriptors.add(CSS_SELECTOR);
descriptors.add(HTML_CHARSET);
descriptors.add(OUTPUT_TYPE);
descriptors.add(MODIFIED_VALUE);
descriptors.add(ATTRIBUTE_KEY);
this.descriptors = Collections.unmodifiableList(descriptors);
final Set<Relationship> relationships = new HashSet<Relationship>();
relationships.add(REL_ORIGINAL);
relationships.add(REL_SUCCESS);
relationships.add(REL_INVALID_HTML);
relationships.add(REL_NOT_FOUND);
this.relationships = Collections.unmodifiableSet(relationships);
}
@Override
public Set<Relationship> getRelationships() {
return this.relationships;
}
@Override
public final List<PropertyDescriptor> getSupportedPropertyDescriptors() {
return descriptors;
}
/**
* This processor used to support URL property, but it has been removed
* since it's not required when altering HTML elements.
* Support URL as dynamic property so that existing data flow can stay in valid state without modification.
*/
@Override
protected PropertyDescriptor getSupportedDynamicPropertyDescriptor(final String propertyDescriptorName) {
return URL;
}
@Override
public void onTrigger(final ProcessContext context, final ProcessSession session) throws ProcessException {
final FlowFile flowFile = session.get();
if (flowFile == null) {
return;
}
final Document doc;
final Elements eles;
try {
doc = parseHTMLDocumentFromFlowfile(flowFile, context, session);
eles = doc.select(context.getProperty(CSS_SELECTOR).evaluateAttributeExpressions(flowFile).getValue());
} catch (Exception ex) {
getLogger().error("Failed to extract HTML from {} due to {}; routing to {}", flowFile, ex.toString(), REL_INVALID_HTML.getName(), ex);
session.transfer(flowFile, REL_INVALID_HTML);
return;
}
final String modifiedValue = context.getProperty(MODIFIED_VALUE).evaluateAttributeExpressions(flowFile).getValue();
if (eles == null || eles.size() == 0) {
// No element found
session.transfer(flowFile, REL_NOT_FOUND);
} else {
for (Element ele : eles) {
switch (context.getProperty(OUTPUT_TYPE).getValue()) {
case ELEMENT_HTML:
ele.html(modifiedValue);
break;
case ELEMENT_ATTRIBUTE:
ele.attr(context.getProperty(ATTRIBUTE_KEY).evaluateAttributeExpressions(flowFile).getValue(), modifiedValue);
break;
case ELEMENT_TEXT:
ele.text(modifiedValue);
break;
}
}
FlowFile ff = session.write(session.create(flowFile), new StreamCallback() {
@Override
public void process(InputStream in, OutputStream out) throws IOException {
out.write(doc.html().getBytes(StandardCharsets.UTF_8));
}
});
ff = session.putAttribute(ff, NUM_ELEMENTS_MODIFIED_ATTR, Integer.valueOf(eles.size()).toString());
session.transfer(ff, REL_SUCCESS);
// Transfer the original HTML
session.transfer(flowFile, REL_ORIGINAL);
}
}
}

175
nifi-nar-bundles/nifi-html-bundle/nifi-html-processors/src/main/java/org/apache/nifi/PutHTMLElement.java
View File

@ -1,175 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.nifi;
import org.apache.nifi.annotation.behavior.InputRequirement;
import org.apache.nifi.annotation.behavior.SupportsBatching;
import org.apache.nifi.annotation.documentation.CapabilityDescription;
import org.apache.nifi.annotation.documentation.SeeAlso;
import org.apache.nifi.annotation.documentation.Tags;
import org.apache.nifi.components.PropertyDescriptor;
import org.apache.nifi.expression.ExpressionLanguageScope;
import org.apache.nifi.flowfile.FlowFile;
import org.apache.nifi.processor.ProcessContext;
import org.apache.nifi.processor.ProcessSession;
import org.apache.nifi.processor.Relationship;
import org.apache.nifi.processor.ProcessorInitializationContext;
import org.apache.nifi.processor.exception.ProcessException;
import org.apache.nifi.processor.io.StreamCallback;
import org.apache.nifi.processor.util.StandardValidators;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
import java.util.List;
import java.util.Set;
import java.util.HashSet;
import java.util.Collections;
@Tags({"put", "html", "dom", "css", "element"})
@SupportsBatching
@InputRequirement(InputRequirement.Requirement.INPUT_REQUIRED)
@CapabilityDescription("Places a new HTML element in the existing HTML DOM. The desired position for the new HTML element is specified by" +
" using CSS selector syntax. The incoming HTML is first converted into a HTML Document Object Model so that HTML DOM location may be located" +
" in a similar manner that CSS selectors are used to apply styles to HTML. The resulting HTML DOM is then \"queried\"" +
" using the user defined CSS selector string to find the position where the user desires to add the new HTML element." +
" Once the new HTML element is added to the DOM it is rendered to HTML and the result replaces the flowfile" +
" content with the updated HTML. A more thorough reference for the CSS selector syntax can be found at" +
" \"http://jsoup.org/apidocs/org/jsoup/select/Selector.html\"")
@SeeAlso({GetHTMLElement.class, ModifyHTMLElement.class})
public class PutHTMLElement extends AbstractHTMLProcessor {
public static final String APPEND_ELEMENT = "append-html";
public static final String PREPEND_ELEMENT = "prepend-html";
public static final PropertyDescriptor PUT_LOCATION_TYPE = new PropertyDescriptor.Builder()
.name("Element Insert Location Type")
.description("Controls whether the new element is prepended or appended to the children of the " +
"Element located by the CSS selector. EX: prepended value '<b>Hi</b>' inside of " +
"Element (using CSS Selector 'p') '<p>There</p>' would result in " +
"'<p><b>Hi</b>There</p>'. Appending the value would result in '<p>There<b>Hi</b></p>'")
.required(true)
.allowableValues(APPEND_ELEMENT, PREPEND_ELEMENT)
.defaultValue(APPEND_ELEMENT)
.build();
public static final PropertyDescriptor PUT_VALUE = new PropertyDescriptor.Builder()
.name("Put Value")
.description("Value used when creating the new Element. Value should be a valid HTML element. " +
"The text should be supplied unencoded: characters like '<', '>', etc will be properly HTML " +
"encoded in the resulting output.")
.required(true)
.addValidator(StandardValidators.NON_EMPTY_VALIDATOR)
.expressionLanguageSupported(ExpressionLanguageScope.FLOWFILE_ATTRIBUTES)
.build();
private List<PropertyDescriptor> descriptors;
private Set<Relationship> relationships;
@Override
protected void init(final ProcessorInitializationContext context) {
final List<PropertyDescriptor> descriptors = new ArrayList<PropertyDescriptor>();
descriptors.add(CSS_SELECTOR);
descriptors.add(HTML_CHARSET);
descriptors.add(PUT_LOCATION_TYPE);
descriptors.add(PUT_VALUE);
this.descriptors = Collections.unmodifiableList(descriptors);
final Set<Relationship> relationships = new HashSet<Relationship>();
relationships.add(REL_ORIGINAL);
relationships.add(REL_SUCCESS);
relationships.add(REL_INVALID_HTML);
relationships.add(REL_NOT_FOUND);
this.relationships = Collections.unmodifiableSet(relationships);
}
@Override
public Set<Relationship> getRelationships() {
return this.relationships;
}
@Override
public final List<PropertyDescriptor> getSupportedPropertyDescriptors() {
return descriptors;
}
/**
* This processor used to support URL property, but it has been removed
* since it's not required when altering HTML elements.
* Support URL as dynamic property so that existing data flow can stay in valid state without modification.
*/
@Override
protected PropertyDescriptor getSupportedDynamicPropertyDescriptor(final String propertyDescriptorName) {
return URL;
}
@Override
public void onTrigger(final ProcessContext context, final ProcessSession session) throws ProcessException {
final FlowFile flowFile = session.get();
if (flowFile == null) {
return;
}
final Document doc;
final Elements eles;
try {
doc = parseHTMLDocumentFromFlowfile(flowFile, context, session);
eles = doc.select(context.getProperty(CSS_SELECTOR).evaluateAttributeExpressions(flowFile).getValue());
} catch (Exception ex) {
getLogger().error("Failed to extract HTML from {} due to {}; routing to {}", flowFile, ex.toString(), REL_INVALID_HTML.getName(), ex);
session.transfer(flowFile, REL_INVALID_HTML);
return;
}
if (eles == null || eles.isEmpty()) {
// No element found
session.transfer(flowFile, REL_NOT_FOUND);
} else {
final String putValue = context.getProperty(PUT_VALUE).evaluateAttributeExpressions(flowFile).getValue();
for (final Element ele : eles) {
switch (context.getProperty(PUT_LOCATION_TYPE).getValue()) {
case APPEND_ELEMENT:
ele.append(putValue);
break;
case PREPEND_ELEMENT:
ele.prepend(putValue);
break;
}
}
FlowFile ff = session.write(session.create(flowFile), new StreamCallback() {
@Override
public void process(final InputStream in, final OutputStream out) throws IOException {
out.write(doc.html().getBytes(StandardCharsets.UTF_8));
}
});
session.transfer(ff, REL_SUCCESS);
// Transfer the original HTML
session.transfer(flowFile, REL_ORIGINAL);
}
}
}

17
nifi-nar-bundles/nifi-html-bundle/nifi-html-processors/src/main/resources/META-INF/services/org.apache.nifi.processor.Processor
View File

@ -1,17 +0,0 @@
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
org.apache.nifi.GetHTMLElement
org.apache.nifi.ModifyHTMLElement
org.apache.nifi.PutHTMLElement

26
nifi-nar-bundles/nifi-html-bundle/nifi-html-processors/src/test/java/org/apache/nifi/AbstractHTMLTest.java
View File

@ -1,26 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.nifi;
public abstract class AbstractHTMLTest {
protected final String ATL_WEATHER_TEXT = "Atlanta Weather";
protected final String GDR_WEATHER_TEXT = "<i>Grand Rapids Weather</i>";
protected final String ATL_WEATHER_LINK = "http://w1.weather.gov/obhistory/KPDK.html";
protected final String AUTHOR_NAME = "Apache NiFi Community";
protected final String ATL_ID = "ATL";
protected final String GDR_ID = "GDR";
}

352
nifi-nar-bundles/nifi-html-bundle/nifi-html-processors/src/test/java/org/apache/nifi/TestGetHTMLElement.java
View File

@ -1,352 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.nifi;
import java.io.File;
import java.io.IOException;
import java.nio.charset.StandardCharsets;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import org.apache.nifi.util.MockFlowFile;
import org.apache.nifi.util.TestRunner;
import org.apache.nifi.util.TestRunners;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.select.Selector;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import static org.junit.jupiter.api.Assertions.assertThrows;
public class TestGetHTMLElement extends AbstractHTMLTest {
private TestRunner testRunner;
@BeforeEach
public void init() {
testRunner = TestRunners.newTestRunner(GetHTMLElement.class);
testRunner.setProperty(GetHTMLElement.URL, "http://localhost");
testRunner.setProperty(GetHTMLElement.OUTPUT_TYPE, GetHTMLElement.ELEMENT_HTML);
testRunner.setProperty(GetHTMLElement.DESTINATION, GetHTMLElement.DESTINATION_CONTENT);
testRunner.setProperty(GetHTMLElement.HTML_CHARSET, "UTF-8");
}
@Test
public void testCSSSelectorSyntaxValidator() throws IOException {
Document doc = Jsoup.parse(new File("src/test/resources/Weather.html"), StandardCharsets.UTF_8.name());
assertThrows(Selector.SelectorParseException.class, () -> doc.select("---invalidCssSelector"));
}
@Test
public void testNoElementFound() throws Exception {
testRunner.setProperty(GetHTMLElement.CSS_SELECTOR, "b"); //Bold element is not present in sample HTML
testRunner.enqueue(new File("src/test/resources/Weather.html").toPath());
testRunner.run();
testRunner.assertTransferCount(GetHTMLElement.REL_SUCCESS, 0);
testRunner.assertTransferCount(GetHTMLElement.REL_INVALID_HTML, 0);
testRunner.assertTransferCount(GetHTMLElement.REL_NOT_FOUND, 1);
}
@Test
public void testInvalidSelector() throws Exception {
testRunner.setProperty(GetHTMLElement.CSS_SELECTOR, "InvalidCSSSelectorSyntax");
testRunner.enqueue(new File("src/test/resources/Weather.html").toPath());
testRunner.run();
testRunner.assertTransferCount(GetHTMLElement.REL_SUCCESS, 0);
testRunner.assertTransferCount(GetHTMLElement.REL_INVALID_HTML, 0);
testRunner.assertTransferCount(GetHTMLElement.REL_NOT_FOUND, 1);
}
@Test
public void testSingleElementFound() throws Exception {
testRunner.setProperty(GetHTMLElement.CSS_SELECTOR, "head");
testRunner.enqueue(new File("src/test/resources/Weather.html").toPath());
testRunner.run();
testRunner.assertTransferCount(GetHTMLElement.REL_SUCCESS, 1);
testRunner.assertTransferCount(GetHTMLElement.REL_INVALID_HTML, 0);
testRunner.assertTransferCount(GetHTMLElement.REL_ORIGINAL, 1);
testRunner.assertTransferCount(GetHTMLElement.REL_NOT_FOUND, 0);
}
@Test
public void testMultipleElementFound() throws Exception {
testRunner.setProperty(GetHTMLElement.CSS_SELECTOR, "a");
testRunner.enqueue(new File("src/test/resources/Weather.html").toPath());
testRunner.run();
testRunner.assertTransferCount(GetHTMLElement.REL_SUCCESS, 3);
testRunner.assertTransferCount(GetHTMLElement.REL_INVALID_HTML, 0);
testRunner.assertTransferCount(GetHTMLElement.REL_ORIGINAL, 1);
testRunner.assertTransferCount(GetHTMLElement.REL_NOT_FOUND, 0);
}
@Test
public void testElementFoundWriteToAttribute() throws Exception {
testRunner.setProperty(GetHTMLElement.CSS_SELECTOR, "#" + ATL_ID);
testRunner.setProperty(GetHTMLElement.DESTINATION, GetHTMLElement.DESTINATION_ATTRIBUTE);
testRunner.setProperty(GetHTMLElement.OUTPUT_TYPE, GetHTMLElement.ELEMENT_ATTRIBUTE);
testRunner.setProperty(GetHTMLElement.ATTRIBUTE_KEY, "href");
testRunner.enqueue(new File("src/test/resources/Weather.html").toPath());
testRunner.run();
testRunner.assertTransferCount(GetHTMLElement.REL_SUCCESS, 1);
testRunner.assertTransferCount(GetHTMLElement.REL_INVALID_HTML, 0);
testRunner.assertTransferCount(GetHTMLElement.REL_ORIGINAL, 1);
testRunner.assertTransferCount(GetHTMLElement.REL_NOT_FOUND, 0);
List<MockFlowFile> ffs = testRunner.getFlowFilesForRelationship(GetHTMLElement.REL_SUCCESS);
ffs.get(0).assertAttributeEquals(GetHTMLElement.HTML_ELEMENT_ATTRIBUTE_NAME, ATL_WEATHER_LINK);
}
@Test
public void testElementFoundWriteToContent() throws Exception {
testRunner.setProperty(GetHTMLElement.CSS_SELECTOR, "#" + ATL_ID);
testRunner.setProperty(GetHTMLElement.DESTINATION, GetHTMLElement.DESTINATION_CONTENT);
testRunner.setProperty(GetHTMLElement.OUTPUT_TYPE, GetHTMLElement.ELEMENT_ATTRIBUTE);
testRunner.setProperty(GetHTMLElement.ATTRIBUTE_KEY, "href");
testRunner.enqueue(new File("src/test/resources/Weather.html").toPath());
testRunner.run();
testRunner.assertTransferCount(GetHTMLElement.REL_SUCCESS, 1);
testRunner.assertTransferCount(GetHTMLElement.REL_INVALID_HTML, 0);
testRunner.assertTransferCount(GetHTMLElement.REL_ORIGINAL, 1);
testRunner.assertTransferCount(GetHTMLElement.REL_NOT_FOUND, 0);
List<MockFlowFile> ffs = testRunner.getFlowFilesForRelationship(GetHTMLElement.REL_SUCCESS);
ffs.get(0).assertContentEquals(ATL_WEATHER_LINK);
}
@Test
public void testValidPrependValueToFoundElement() throws Exception {
final String PREPEND_VALUE = "TestPrepend";
testRunner.setProperty(GetHTMLElement.PREPEND_ELEMENT_VALUE, PREPEND_VALUE);
testRunner.setProperty(GetHTMLElement.CSS_SELECTOR, "#" + ATL_ID);
testRunner.setProperty(GetHTMLElement.DESTINATION, GetHTMLElement.DESTINATION_CONTENT);
testRunner.setProperty(GetHTMLElement.OUTPUT_TYPE, GetHTMLElement.ELEMENT_ATTRIBUTE);
testRunner.setProperty(GetHTMLElement.ATTRIBUTE_KEY, "href");
testRunner.enqueue(new File("src/test/resources/Weather.html").toPath());
testRunner.run();
testRunner.assertTransferCount(GetHTMLElement.REL_SUCCESS, 1);
testRunner.assertTransferCount(GetHTMLElement.REL_INVALID_HTML, 0);
testRunner.assertTransferCount(GetHTMLElement.REL_ORIGINAL, 1);
testRunner.assertTransferCount(GetHTMLElement.REL_NOT_FOUND, 0);
List<MockFlowFile> ffs = testRunner.getFlowFilesForRelationship(GetHTMLElement.REL_SUCCESS);
ffs.get(0).assertContentEquals(PREPEND_VALUE + ATL_WEATHER_LINK);
}
@Test
public void testValidPrependValueToNotFoundElement() throws Exception {
final String PREPEND_VALUE = "TestPrepend";
testRunner.setProperty(GetHTMLElement.PREPEND_ELEMENT_VALUE, PREPEND_VALUE);
testRunner.setProperty(GetHTMLElement.CSS_SELECTOR, "b");
testRunner.setProperty(GetHTMLElement.DESTINATION, GetHTMLElement.DESTINATION_CONTENT);
testRunner.setProperty(GetHTMLElement.OUTPUT_TYPE, GetHTMLElement.ELEMENT_TEXT);
testRunner.enqueue(new File("src/test/resources/Weather.html").toPath());
testRunner.run();
testRunner.assertTransferCount(GetHTMLElement.REL_SUCCESS, 0);
testRunner.assertTransferCount(GetHTMLElement.REL_INVALID_HTML, 0);
testRunner.assertTransferCount(GetHTMLElement.REL_ORIGINAL, 0);
testRunner.assertTransferCount(GetHTMLElement.REL_NOT_FOUND, 1);
}
@Test
public void testValidAppendValueToFoundElement() throws Exception {
final String APPEND_VALUE = "TestAppend";
testRunner.setProperty(GetHTMLElement.APPEND_ELEMENT_VALUE, APPEND_VALUE);
testRunner.setProperty(GetHTMLElement.CSS_SELECTOR, "#" + ATL_ID);
testRunner.setProperty(GetHTMLElement.DESTINATION, GetHTMLElement.DESTINATION_CONTENT);
testRunner.setProperty(GetHTMLElement.OUTPUT_TYPE, GetHTMLElement.ELEMENT_ATTRIBUTE);
testRunner.setProperty(GetHTMLElement.ATTRIBUTE_KEY, "href");
testRunner.enqueue(new File("src/test/resources/Weather.html").toPath());
testRunner.run();
testRunner.assertTransferCount(GetHTMLElement.REL_SUCCESS, 1);
testRunner.assertTransferCount(GetHTMLElement.REL_INVALID_HTML, 0);
testRunner.assertTransferCount(GetHTMLElement.REL_ORIGINAL, 1);
testRunner.assertTransferCount(GetHTMLElement.REL_NOT_FOUND, 0);
List<MockFlowFile> ffs = testRunner.getFlowFilesForRelationship(GetHTMLElement.REL_SUCCESS);
ffs.get(0).assertContentEquals(ATL_WEATHER_LINK + APPEND_VALUE);
}
@Test
public void testValidAppendValueToNotFoundElement() throws Exception {
final String APPEND_VALUE = "TestAppend";
testRunner.setProperty(GetHTMLElement.APPEND_ELEMENT_VALUE, APPEND_VALUE);
testRunner.setProperty(GetHTMLElement.CSS_SELECTOR, "b");
testRunner.setProperty(GetHTMLElement.DESTINATION, GetHTMLElement.DESTINATION_CONTENT);
testRunner.setProperty(GetHTMLElement.OUTPUT_TYPE, GetHTMLElement.ELEMENT_TEXT);
testRunner.enqueue(new File("src/test/resources/Weather.html").toPath());
testRunner.run();
testRunner.assertTransferCount(GetHTMLElement.REL_SUCCESS, 0);
testRunner.assertTransferCount(GetHTMLElement.REL_INVALID_HTML, 0);
testRunner.assertTransferCount(GetHTMLElement.REL_ORIGINAL, 0);
testRunner.assertTransferCount(GetHTMLElement.REL_NOT_FOUND, 1);
}
@Test
public void testExtractAttributeFromElement() throws Exception {
testRunner.setProperty(GetHTMLElement.CSS_SELECTOR, "meta[name=author]");
testRunner.setProperty(GetHTMLElement.DESTINATION, GetHTMLElement.DESTINATION_CONTENT);
testRunner.setProperty(GetHTMLElement.OUTPUT_TYPE, GetHTMLElement.ELEMENT_ATTRIBUTE);
testRunner.setProperty(GetHTMLElement.ATTRIBUTE_KEY, "Content");
testRunner.enqueue(new File("src/test/resources/Weather.html").toPath());
testRunner.run();
testRunner.assertTransferCount(GetHTMLElement.REL_SUCCESS, 1);
testRunner.assertTransferCount(GetHTMLElement.REL_INVALID_HTML, 0);
testRunner.assertTransferCount(GetHTMLElement.REL_ORIGINAL, 1);
testRunner.assertTransferCount(GetHTMLElement.REL_NOT_FOUND, 0);
List<MockFlowFile> ffs = testRunner.getFlowFilesForRelationship(GetHTMLElement.REL_SUCCESS);
ffs.get(0).assertContentEquals(AUTHOR_NAME);
}
@Test
public void testExtractAttributeFromElementRelativeUrl() throws Exception {
testRunner.setProperty(GetHTMLElement.CSS_SELECTOR, "script");
testRunner.setProperty(GetHTMLElement.DESTINATION, GetHTMLElement.DESTINATION_CONTENT);
testRunner.setProperty(GetHTMLElement.OUTPUT_TYPE, GetHTMLElement.ELEMENT_ATTRIBUTE);
testRunner.setProperty(GetHTMLElement.ATTRIBUTE_KEY, "src");
testRunner.enqueue(new File("src/test/resources/Weather.html").toPath());
testRunner.run();
testRunner.assertTransferCount(GetHTMLElement.REL_SUCCESS, 1);
testRunner.assertTransferCount(GetHTMLElement.REL_INVALID_HTML, 0);
testRunner.assertTransferCount(GetHTMLElement.REL_ORIGINAL, 1);
testRunner.assertTransferCount(GetHTMLElement.REL_NOT_FOUND, 0);
List<MockFlowFile> ffs = testRunner.getFlowFilesForRelationship(GetHTMLElement.REL_SUCCESS);
ffs.get(0).assertContentEquals("js/scripts.js");
}
@Test
public void testExtractAttributeFromElementAbsoluteUrl() throws Exception {
testRunner.setProperty(GetHTMLElement.CSS_SELECTOR, "script");
testRunner.setProperty(GetHTMLElement.DESTINATION, GetHTMLElement.DESTINATION_CONTENT);
testRunner.setProperty(GetHTMLElement.OUTPUT_TYPE, GetHTMLElement.ELEMENT_ATTRIBUTE);
testRunner.setProperty(GetHTMLElement.ATTRIBUTE_KEY, "abs:src");
testRunner.enqueue(new File("src/test/resources/Weather.html").toPath());
testRunner.run();
testRunner.assertTransferCount(GetHTMLElement.REL_SUCCESS, 1);
testRunner.assertTransferCount(GetHTMLElement.REL_INVALID_HTML, 0);
testRunner.assertTransferCount(GetHTMLElement.REL_ORIGINAL, 1);
testRunner.assertTransferCount(GetHTMLElement.REL_NOT_FOUND, 0);
List<MockFlowFile> ffs = testRunner.getFlowFilesForRelationship(GetHTMLElement.REL_SUCCESS);
ffs.get(0).assertContentEquals("http://localhost/js/scripts.js");
}
@Test
public void testExtractAttributeFromElementAbsoluteUrlWithEL() throws Exception {
testRunner.setProperty(GetHTMLElement.CSS_SELECTOR, "script");
testRunner.setProperty(GetHTMLElement.DESTINATION, GetHTMLElement.DESTINATION_CONTENT);
testRunner.setProperty(GetHTMLElement.OUTPUT_TYPE, GetHTMLElement.ELEMENT_ATTRIBUTE);
testRunner.setProperty(GetHTMLElement.ATTRIBUTE_KEY, "abs:src");
testRunner.setProperty(GetHTMLElement.URL, "${contentUrl}");
final Map<String, String> attributes = new HashMap<>();
attributes.put("contentUrl", "https://example.com/a/b/c/Weather.html");
testRunner.enqueue(new File("src/test/resources/Weather.html").toPath(), attributes);
testRunner.run();
testRunner.assertTransferCount(GetHTMLElement.REL_SUCCESS, 1);
testRunner.assertTransferCount(GetHTMLElement.REL_INVALID_HTML, 0);
testRunner.assertTransferCount(GetHTMLElement.REL_ORIGINAL, 1);
testRunner.assertTransferCount(GetHTMLElement.REL_NOT_FOUND, 0);
List<MockFlowFile> ffs = testRunner.getFlowFilesForRelationship(GetHTMLElement.REL_SUCCESS);
ffs.get(0).assertContentEquals("https://example.com/a/b/c/js/scripts.js");
}
@Test
public void testExtractAttributeFromElementAbsoluteUrlWithEmptyElResult() throws Exception {
testRunner.setProperty(GetHTMLElement.CSS_SELECTOR, "script");
testRunner.setProperty(GetHTMLElement.DESTINATION, GetHTMLElement.DESTINATION_CONTENT);
testRunner.setProperty(GetHTMLElement.OUTPUT_TYPE, GetHTMLElement.ELEMENT_ATTRIBUTE);
testRunner.setProperty(GetHTMLElement.ATTRIBUTE_KEY, "abs:src");
// Expression Language returns empty string because flow-file doesn't have contentUrl attribute.
testRunner.setProperty(GetHTMLElement.URL, "${contentUrl}");
testRunner.enqueue(new File("src/test/resources/Weather.html").toPath());
testRunner.run();
testRunner.assertTransferCount(GetHTMLElement.REL_SUCCESS, 0);
testRunner.assertTransferCount(GetHTMLElement.REL_INVALID_HTML, 1);
testRunner.assertTransferCount(GetHTMLElement.REL_ORIGINAL, 0);
testRunner.assertTransferCount(GetHTMLElement.REL_NOT_FOUND, 0);
}
@Test
public void testExtractTextFromElement() throws Exception {
testRunner.setProperty(GetHTMLElement.CSS_SELECTOR, "#" + ATL_ID);
testRunner.setProperty(GetHTMLElement.DESTINATION, GetHTMLElement.DESTINATION_CONTENT);
testRunner.setProperty(GetHTMLElement.OUTPUT_TYPE, GetHTMLElement.ELEMENT_TEXT);
testRunner.enqueue(new File("src/test/resources/Weather.html").toPath());
testRunner.run();
testRunner.assertTransferCount(GetHTMLElement.REL_SUCCESS, 1);
testRunner.assertTransferCount(GetHTMLElement.REL_INVALID_HTML, 0);
testRunner.assertTransferCount(GetHTMLElement.REL_ORIGINAL, 1);
testRunner.assertTransferCount(GetHTMLElement.REL_NOT_FOUND, 0);
List<MockFlowFile> ffs = testRunner.getFlowFilesForRelationship(GetHTMLElement.REL_SUCCESS);
ffs.get(0).assertContentEquals(ATL_WEATHER_TEXT);
}
@Test
public void testExtractHTMLFromElement() throws Exception {
testRunner.setProperty(GetHTMLElement.CSS_SELECTOR, "#" + GDR_ID);
testRunner.setProperty(GetHTMLElement.DESTINATION, GetHTMLElement.DESTINATION_CONTENT);
testRunner.setProperty(GetHTMLElement.OUTPUT_TYPE, GetHTMLElement.ELEMENT_HTML);
testRunner.enqueue(new File("src/test/resources/Weather.html").toPath());
testRunner.run();
testRunner.assertTransferCount(GetHTMLElement.REL_SUCCESS, 1);
testRunner.assertTransferCount(GetHTMLElement.REL_INVALID_HTML, 0);
testRunner.assertTransferCount(GetHTMLElement.REL_ORIGINAL, 1);
testRunner.assertTransferCount(GetHTMLElement.REL_NOT_FOUND, 0);
List<MockFlowFile> ffs = testRunner.getFlowFilesForRelationship(GetHTMLElement.REL_SUCCESS);
ffs.get(0).assertContentEquals(GDR_WEATHER_TEXT);
}
}

204
nifi-nar-bundles/nifi-html-bundle/nifi-html-processors/src/test/java/org/apache/nifi/TestModifyHTMLElement.java
View File

@ -1,204 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.nifi;
import org.apache.commons.lang3.StringUtils;
import org.apache.nifi.util.MockFlowFile;
import org.apache.nifi.util.TestRunner;
import org.apache.nifi.util.TestRunners;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import java.io.File;
import java.util.List;
import static org.junit.jupiter.api.Assertions.assertNotNull;
import static org.junit.jupiter.api.Assertions.assertTrue;
public class TestModifyHTMLElement extends AbstractHTMLTest {
private TestRunner testRunner;
@BeforeEach
public void init() {
testRunner = TestRunners.newTestRunner(ModifyHTMLElement.class);
testRunner = TestRunners.newTestRunner(ModifyHTMLElement.class);
testRunner.setProperty(ModifyHTMLElement.URL, "http://localhost");
testRunner.setProperty(ModifyHTMLElement.OUTPUT_TYPE, GetHTMLElement.ELEMENT_HTML);
testRunner.setProperty(ModifyHTMLElement.HTML_CHARSET, "UTF-8");
}
@Test
public void testModifyText() throws Exception {
final String MOD_VALUE = "Newly modified value to replace " + ATL_WEATHER_TEXT;
testRunner.setProperty(ModifyHTMLElement.CSS_SELECTOR, "#" + ATL_ID);
testRunner.setProperty(ModifyHTMLElement.OUTPUT_TYPE, ModifyHTMLElement.ELEMENT_TEXT);
testRunner.setProperty(ModifyHTMLElement.MODIFIED_VALUE, MOD_VALUE);
testRunner.enqueue(new File("src/test/resources/Weather.html").toPath());
testRunner.run();
testRunner.assertTransferCount(ModifyHTMLElement.REL_SUCCESS, 1);
testRunner.assertTransferCount(ModifyHTMLElement.REL_INVALID_HTML, 0);
testRunner.assertTransferCount(ModifyHTMLElement.REL_ORIGINAL, 1);
testRunner.assertTransferCount(ModifyHTMLElement.REL_NOT_FOUND, 0);
List<MockFlowFile> ffs = testRunner.getFlowFilesForRelationship(ModifyHTMLElement.REL_SUCCESS);
assertTrue(ffs.size() == 1);
String data = new String(testRunner.getContentAsByteArray(ffs.get(0)));
//Contents will be the entire HTML doc. So lets use Jsoup again just the grab the element we want.
Document doc = Jsoup.parse(data);
Elements eles = doc.select("#" + ATL_ID);
Element ele = eles.get(0);
assertTrue(StringUtils.equals(MOD_VALUE, ele.text()));
}
@Test
public void testModifyHTMLWithExpressionLanguage() throws Exception {
final String MOD_VALUE = "Newly modified value to replace " + ATL_WEATHER_TEXT;
testRunner.setProperty(ModifyHTMLElement.CSS_SELECTOR, "#" + ATL_ID);
testRunner.setProperty(ModifyHTMLElement.OUTPUT_TYPE, ModifyHTMLElement.ELEMENT_TEXT);
testRunner.setProperty(ModifyHTMLElement.MODIFIED_VALUE, "${\" " + MOD_VALUE + " \":trim()}");
testRunner.enqueue(new File("src/test/resources/Weather.html").toPath());
testRunner.run();
testRunner.assertTransferCount(ModifyHTMLElement.REL_SUCCESS, 1);
testRunner.assertTransferCount(ModifyHTMLElement.REL_INVALID_HTML, 0);
testRunner.assertTransferCount(ModifyHTMLElement.REL_ORIGINAL, 1);
testRunner.assertTransferCount(ModifyHTMLElement.REL_NOT_FOUND, 0);
List<MockFlowFile> ffs = testRunner.getFlowFilesForRelationship(ModifyHTMLElement.REL_SUCCESS);
assertTrue(ffs.size() == 1);
String data = new String(testRunner.getContentAsByteArray(ffs.get(0)));
//Contents will be the entire HTML doc. So lets use Jsoup again just the grab the element we want.
Document doc = Jsoup.parse(data);
Elements eles = doc.select("#" + ATL_ID);
Element ele = eles.get(0);
assertNotNull(ele.text());
}
@Test
public void testModifyHTML() throws Exception {
final String MOD_VALUE = "Newly modified HTML to replace " + GDR_WEATHER_TEXT;
testRunner.setProperty(ModifyHTMLElement.CSS_SELECTOR, "#" + GDR_ID);
testRunner.setProperty(ModifyHTMLElement.OUTPUT_TYPE, ModifyHTMLElement.ELEMENT_HTML);
testRunner.setProperty(ModifyHTMLElement.MODIFIED_VALUE, MOD_VALUE);
testRunner.enqueue(new File("src/test/resources/Weather.html").toPath());
testRunner.run();
testRunner.assertTransferCount(ModifyHTMLElement.REL_SUCCESS, 1);
testRunner.assertTransferCount(ModifyHTMLElement.REL_INVALID_HTML, 0);
testRunner.assertTransferCount(ModifyHTMLElement.REL_ORIGINAL, 1);
testRunner.assertTransferCount(ModifyHTMLElement.REL_NOT_FOUND, 0);
List<MockFlowFile> ffs = testRunner.getFlowFilesForRelationship(ModifyHTMLElement.REL_SUCCESS);
assertTrue(ffs.size() == 1);
String data = new String(testRunner.getContentAsByteArray(ffs.get(0)));
//Contents will be the entire HTML doc. So lets use Jsoup again just the grab the element we want.
Document doc = Jsoup.parse(data);
Elements eles = doc.select("#" + GDR_ID);
Element ele = eles.get(0);
assertTrue(StringUtils.equals(MOD_VALUE, ele.html()));
}
@Test
public void testModifyAttribute() throws Exception {
final String MOD_VALUE = "http://localhost/newlink";
testRunner.setProperty(ModifyHTMLElement.CSS_SELECTOR, "#" + GDR_ID);
testRunner.setProperty(ModifyHTMLElement.OUTPUT_TYPE, ModifyHTMLElement.ELEMENT_ATTRIBUTE);
testRunner.setProperty(ModifyHTMLElement.ATTRIBUTE_KEY, "href");
testRunner.setProperty(ModifyHTMLElement.MODIFIED_VALUE, MOD_VALUE);
testRunner.enqueue(new File("src/test/resources/Weather.html").toPath());
testRunner.run();
testRunner.assertTransferCount(ModifyHTMLElement.REL_SUCCESS, 1);
testRunner.assertTransferCount(ModifyHTMLElement.REL_INVALID_HTML, 0);
testRunner.assertTransferCount(ModifyHTMLElement.REL_ORIGINAL, 1);
testRunner.assertTransferCount(ModifyHTMLElement.REL_NOT_FOUND, 0);
List<MockFlowFile> ffs = testRunner.getFlowFilesForRelationship(ModifyHTMLElement.REL_SUCCESS);
assertTrue(ffs.size() == 1);
String data = new String(testRunner.getContentAsByteArray(ffs.get(0)));
//Contents will be the entire HTML doc. So lets use Jsoup again just the grab the element we want.
Document doc = Jsoup.parse(data);
Elements eles = doc.select("#" + GDR_ID);
Element ele = eles.get(0);
assertTrue(StringUtils.equals(MOD_VALUE, ele.attr("href")));
}
@Test
public void testModifyElementNotFound() throws Exception {
final String MOD_VALUE = "http://localhost/newlink";
testRunner.setProperty(ModifyHTMLElement.CSS_SELECTOR, "b");
testRunner.setProperty(ModifyHTMLElement.OUTPUT_TYPE, ModifyHTMLElement.ELEMENT_HTML);
testRunner.setProperty(ModifyHTMLElement.MODIFIED_VALUE, MOD_VALUE);
testRunner.enqueue(new File("src/test/resources/Weather.html").toPath());
testRunner.run();
testRunner.assertTransferCount(ModifyHTMLElement.REL_SUCCESS, 0);
testRunner.assertTransferCount(ModifyHTMLElement.REL_INVALID_HTML, 0);
testRunner.assertTransferCount(ModifyHTMLElement.REL_ORIGINAL, 0);
testRunner.assertTransferCount(ModifyHTMLElement.REL_NOT_FOUND, 1);
}
@Test
public void testModifyValueContainsHTMLCharacters() throws Exception {
final String MOD_VALUE = "Text that contains > and < characters";
testRunner.setProperty(ModifyHTMLElement.CSS_SELECTOR, "#" + GDR_ID);
testRunner.setProperty(ModifyHTMLElement.OUTPUT_TYPE, ModifyHTMLElement.ELEMENT_HTML);
testRunner.setProperty(ModifyHTMLElement.MODIFIED_VALUE, MOD_VALUE);
testRunner.enqueue(new File("src/test/resources/Weather.html").toPath());
testRunner.run();
testRunner.assertTransferCount(ModifyHTMLElement.REL_SUCCESS, 1);
testRunner.assertTransferCount(ModifyHTMLElement.REL_INVALID_HTML, 0);
testRunner.assertTransferCount(ModifyHTMLElement.REL_ORIGINAL, 1);
testRunner.assertTransferCount(ModifyHTMLElement.REL_NOT_FOUND, 0);
List<MockFlowFile> ffs = testRunner.getFlowFilesForRelationship(ModifyHTMLElement.REL_SUCCESS);
assertTrue(ffs.size() == 1);
String data = new String(testRunner.getContentAsByteArray(ffs.get(0)));
//Contents will be the entire HTML doc. So lets use Jsoup again just the grab the element we want.
Document doc = Jsoup.parse(data);
Elements eles = doc.select("#" + GDR_ID);
Element ele = eles.get(0);
assertTrue(StringUtils.equals(MOD_VALUE, ele.text()));
assertTrue(StringUtils.equals(MOD_VALUE.replace(">", "&gt;").replace("<", "&lt;"), ele.html()));
}
}

127
nifi-nar-bundles/nifi-html-bundle/nifi-html-processors/src/test/java/org/apache/nifi/TestPutHTMLElement.java
View File

@ -1,127 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.nifi;
import org.apache.commons.lang3.StringUtils;
import org.apache.nifi.util.MockFlowFile;
import org.apache.nifi.util.TestRunner;
import org.apache.nifi.util.TestRunners;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import java.io.File;
import java.util.List;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertTrue;
public class TestPutHTMLElement extends AbstractHTMLTest {
private TestRunner testRunner;
@BeforeEach
public void init() {
testRunner = TestRunners.newTestRunner(PutHTMLElement.class);
testRunner.setProperty(PutHTMLElement.URL, "http://localhost");
}
@Test
public void testAddNewElementToRoot() throws Exception {
final String MOD_VALUE = "<p>modified value</p>";
testRunner.setProperty(PutHTMLElement.CSS_SELECTOR, "body");
testRunner.setProperty(PutHTMLElement.PUT_LOCATION_TYPE, PutHTMLElement.PREPEND_ELEMENT);
testRunner.setProperty(PutHTMLElement.PUT_VALUE, MOD_VALUE);
testRunner.enqueue(new File("src/test/resources/Weather.html").toPath());
testRunner.run();
testRunner.assertTransferCount(PutHTMLElement.REL_SUCCESS, 1);
testRunner.assertTransferCount(PutHTMLElement.REL_INVALID_HTML, 0);
testRunner.assertTransferCount(PutHTMLElement.REL_ORIGINAL, 1);
testRunner.assertTransferCount(PutHTMLElement.REL_NOT_FOUND, 0);
List<MockFlowFile> ffs = testRunner.getFlowFilesForRelationship(PutHTMLElement.REL_SUCCESS);
assertEquals(1, ffs.size());
String data = new String(testRunner.getContentAsByteArray(ffs.get(0)));
//Contents will be the entire HTML doc. So lets use Jsoup again just the grab the element we want.
Document doc = Jsoup.parse(data);
Elements eles = doc.select("body > p");
Element ele = eles.get(0);
assertTrue(StringUtils.equals(MOD_VALUE.replace("<p>", "").replace("</p>", ""), ele.html()));
}
@Test
public void testPrependPElementToDiv() throws Exception {
final String MOD_VALUE = "<p>modified value</p>";
testRunner.setProperty(PutHTMLElement.CSS_SELECTOR, "#put");
testRunner.setProperty(PutHTMLElement.PUT_LOCATION_TYPE, PutHTMLElement.PREPEND_ELEMENT);
testRunner.setProperty(PutHTMLElement.PUT_VALUE, MOD_VALUE);
testRunner.enqueue(new File("src/test/resources/Weather.html").toPath());
testRunner.run();
testRunner.assertTransferCount(PutHTMLElement.REL_SUCCESS, 1);
testRunner.assertTransferCount(PutHTMLElement.REL_INVALID_HTML, 0);
testRunner.assertTransferCount(PutHTMLElement.REL_ORIGINAL, 1);
testRunner.assertTransferCount(PutHTMLElement.REL_NOT_FOUND, 0);
List<MockFlowFile> ffs = testRunner.getFlowFilesForRelationship(PutHTMLElement.REL_SUCCESS);
assertEquals(1, ffs.size());
String data = new String(testRunner.getContentAsByteArray(ffs.get(0)));
//Contents will be the entire HTML doc. So lets use Jsoup again just the grab the element we want.
Document doc = Jsoup.parse(data);
Elements eles = doc.select("#put");
Element ele = eles.get(0);
assertEquals("<p>modified value</p><a href=\"httpd://localhost\"></a>", ele.html());
}
@Test
public void testAppendPElementToDiv() throws Exception {
final String MOD_VALUE = "<p>modified value</p>";
testRunner.setProperty(PutHTMLElement.CSS_SELECTOR, "#put");
testRunner.setProperty(PutHTMLElement.PUT_LOCATION_TYPE, PutHTMLElement.APPEND_ELEMENT);
testRunner.setProperty(PutHTMLElement.PUT_VALUE, MOD_VALUE);
testRunner.enqueue(new File("src/test/resources/Weather.html").toPath());
testRunner.run();
testRunner.assertTransferCount(PutHTMLElement.REL_SUCCESS, 1);
testRunner.assertTransferCount(PutHTMLElement.REL_INVALID_HTML, 0);
testRunner.assertTransferCount(PutHTMLElement.REL_ORIGINAL, 1);
testRunner.assertTransferCount(PutHTMLElement.REL_NOT_FOUND, 0);
List<MockFlowFile> ffs = testRunner.getFlowFilesForRelationship(PutHTMLElement.REL_SUCCESS);
assertEquals(1, ffs.size());
String data = new String(testRunner.getContentAsByteArray(ffs.get(0)));
//Contents will be the entire HTML doc. So lets use Jsoup again just the grab the element we want.
Document doc = Jsoup.parse(data);
Elements eles = doc.select("#put");
Element ele = eles.get(0);
assertEquals("<a href=\"httpd://localhost\"></a><p>modified value</p>", StringUtils.remove(ele.html(), "\n"));
}
}

25
nifi-nar-bundles/nifi-html-bundle/nifi-html-processors/src/test/resources/Weather.html
View File

@ -1,25 +0,0 @@
<!doctype html>
<html lang="en">
<head>
<title>NiFi HTML Parsing Demo</title>
<meta charset="utf-8">
<meta name="description" content="NiFi HTML Parsing Demo">
<meta name="author" content="Apache NiFi Community">
<link rel="stylesheet" href="css/styles.css?v=1.0">
<!--[if lt IE 9]>
<script src="http://html5shiv.googlecode.com/svn/trunk/html5.js"></script>
<![endif]-->
</head>
<body>
<script src="js/scripts.js"></script>
<p>Check out this weather!
<a id="ATL" href="http://w1.weather.gov/obhistory/KPDK.html">Atlanta Weather</a>
</p>
<p>I guess it could be colder ...
<a id="GDR" href="http://w1.weather.gov/obhistory/KGRR.html"><i>Grand Rapids Weather</i></a>
</p>
<div id="put"><a href="httpd://localhost" /></div>
</body>
</html>

42
nifi-nar-bundles/nifi-html-bundle/pom.xml
View File

@ -1,42 +0,0 @@
<?xml version="1.0" encoding="UTF-8"?>
<!--
Licensed to the Apache Software Foundation (ASF) under one or more
contributor license agreements. See the NOTICE file distributed with
this work for additional information regarding copyright ownership.
The ASF licenses this file to You under the Apache License, Version 2.0
(the "License"); you may not use this file except in compliance with
the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
-->
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>org.apache.nifi</groupId>
<artifactId>nifi-nar-bundles</artifactId>
<version>2.0.0-SNAPSHOT</version>
</parent>
<artifactId>nifi-html-bundle</artifactId>
<packaging>pom</packaging>
<modules>
<module>nifi-html-processors</module>
<module>nifi-html-nar</module>
</modules>
<dependencyManagement>
<dependencies>
<dependency>
<groupId>org.apache.nifi</groupId>
<artifactId>nifi-html-processors</artifactId>
<version>2.0.0-SNAPSHOT</version>
</dependency>
</dependencies>
</dependencyManagement>
</project>

39
nifi-nar-bundles/nifi-metrics-reporting-bundle/nifi-metrics-reporter-service-api-nar/pom.xml
View File

@ -1,39 +0,0 @@
<?xml version="1.0" encoding="UTF-8"?>
<!--
Licensed to the Apache Software Foundation (ASF) under one or more
contributor license agreements. See the NOTICE file distributed with
this work for additional information regarding copyright ownership.
The ASF licenses this file to You under the Apache License, Version 2.0
(the "License"); you may not use this file except in compliance with
the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
-->
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/xsd/maven-4.0.0.xsd">
<parent>
<artifactId>nifi-metrics-reporting-bundle</artifactId>
<groupId>org.apache.nifi</groupId>
<version>2.0.0-SNAPSHOT</version>
</parent>
<packaging>nar</packaging>
<modelVersion>4.0.0</modelVersion>
<artifactId>nifi-metrics-reporter-service-api-nar</artifactId>
<dependencies>
<dependency>
<groupId>org.apache.nifi</groupId>
<artifactId>nifi-standard-services-api-nar</artifactId>
<version>2.0.0-SNAPSHOT</version>
<type>nar</type>
</dependency>
<dependency>
<groupId>org.apache.nifi</groupId>
<artifactId>nifi-metrics-reporter-service-api</artifactId>
<version>2.0.0-SNAPSHOT</version>
</dependency>
</dependencies>
</project>

30
nifi-nar-bundles/nifi-metrics-reporting-bundle/nifi-metrics-reporter-service-api-nar/src/main/resources/META-INF/NOTICE
View File

@ -1,30 +0,0 @@
nifi-metrics-reporter-service-api-nar
Copyright 2015-2020 The Apache Software Foundation
This product includes software developed at
The Apache Software Foundation (http://www.apache.org/).
******************
Apache Software License v2
******************
The following binary components are provided under the Apache Software License v2
(ASLv2) Dropwizard Metrics
The following NOTICE information applies:
Metrics
Copyright 2010-2013 Coda Hale and Yammer, Inc., 2014-2017 Dropwizard Team
This product includes software developed by Coda Hale and Yammer, Inc.
This product includes code derived from the JSR-166 project (ThreadLocalRandom, Striped64,
LongAdder), which was released with the following comments:
Written by Doug Lea with assistance from members of JCP JSR-166
Expert Group and released to the public domain, as explained at
http://creativecommons.org/publicdomain/zero/1.0/
The derived work in the nifi-metrics module is adapted from
https://github.com/dropwizard/metrics/blob/v2.2.0/metrics-core/src/main/java/com/yammer/metrics/core/VirtualMachineMetrics.java
and can be found in
nifi-commons/nifi-metrics/src/main/java/org/apache/nifi/metrics/jvm/JvmMetrics.java
nifi-commons/nifi-metrics/src/main/java/org/apache/nifi/metrics/jvm/JmxJvmMetrics.java

32
nifi-nar-bundles/nifi-metrics-reporting-bundle/nifi-metrics-reporter-service-api/pom.xml
View File

@ -1,32 +0,0 @@
<?xml version="1.0" encoding="UTF-8"?>
<!--
Licensed to the Apache Software Foundation (ASF) under one or more
contributor license agreements. See the NOTICE file distributed with
this work for additional information regarding copyright ownership.
The ASF licenses this file to You under the Apache License, Version 2.0
(the "License"); you may not use this file except in compliance with
the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
-->
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/xsd/maven-4.0.0.xsd">
<parent>
<artifactId>nifi-metrics-reporting-bundle</artifactId>
<groupId>org.apache.nifi</groupId>
<version>2.0.0-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
<artifactId>nifi-metrics-reporter-service-api</artifactId>
<dependencies>
<dependency>
<groupId>org.apache.nifi</groupId>
<artifactId>nifi-metrics</artifactId>
</dependency>
</dependencies>
</project>

40
nifi-nar-bundles/nifi-metrics-reporting-bundle/nifi-metrics-reporter-service-api/src/main/java/org/apache/nifi/metrics/reporting/reporter/service/MetricReporterService.java
View File

@ -1,40 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.nifi.metrics.reporting.reporter.service;
import com.codahale.metrics.MetricRegistry;
import com.codahale.metrics.ScheduledReporter;
import org.apache.nifi.controller.ControllerService;
import org.apache.nifi.processor.exception.ProcessException;
/**
* An interface for controller services used by MetricsReportingTask. In order to report to a new
* client, implement this interface and make sure to return the desired implementation of {@link ScheduledReporter}.
*
* @author Omer Hadari
*/
public interface MetricReporterService extends ControllerService {
/**
* Create a reporter to a metric client (i.e. graphite).
*
* @param metricRegistry registry with the metrics to report.
* @return an instance of the reporter.
* @throws ProcessException if there was an error creating the reporter.
*/
ScheduledReporter createReporter(MetricRegistry metricRegistry) throws ProcessException;
}

40
nifi-nar-bundles/nifi-metrics-reporting-bundle/nifi-metrics-reporting-nar/pom.xml
View File

@ -1,40 +0,0 @@
<?xml version="1.0" encoding="UTF-8"?>
<!--
Licensed to the Apache Software Foundation (ASF) under one or more
contributor license agreements. See the NOTICE file distributed with
this work for additional information regarding copyright ownership.
The ASF licenses this file to You under the Apache License, Version 2.0
(the "License"); you may not use this file except in compliance with
the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
-->
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/xsd/maven-4.0.0.xsd">
<parent>
<artifactId>nifi-metrics-reporting-bundle</artifactId>
<groupId>org.apache.nifi</groupId>
<version>2.0.0-SNAPSHOT</version>
</parent>
<packaging>nar</packaging>
<modelVersion>4.0.0</modelVersion>
<artifactId>nifi-metrics-reporting-nar</artifactId>
<dependencies>
<dependency>
<groupId>org.apache.nifi</groupId>
<artifactId>nifi-metrics-reporting-task</artifactId>
<version>2.0.0-SNAPSHOT</version>
</dependency>
<dependency>
<groupId>org.apache.nifi</groupId>
<artifactId>nifi-metrics-reporter-service-api-nar</artifactId>
<version>2.0.0-SNAPSHOT</version>
<type>nar</type>
</dependency>
</dependencies>
</project>

30
nifi-nar-bundles/nifi-metrics-reporting-bundle/nifi-metrics-reporting-nar/src/main/resources/META-INF/NOTICE
View File

@ -1,30 +0,0 @@
nifi-metrics-reporting-nar
Copyright 2015-2020 The Apache Software Foundation
This product includes software developed at
The Apache Software Foundation (http://www.apache.org/).
******************
Apache Software License v2
******************
The following binary components are provided under the Apache Software License v2
(ASLv2) Dropwizard Metrics
The following NOTICE information applies:
Metrics
Copyright 2010-2013 Coda Hale and Yammer, Inc., 2014-2017 Dropwizard Team
This product includes software developed by Coda Hale and Yammer, Inc.
This product includes code derived from the JSR-166 project (ThreadLocalRandom, Striped64,
LongAdder), which was released with the following comments:
Written by Doug Lea with assistance from members of JCP JSR-166
Expert Group and released to the public domain, as explained at
http://creativecommons.org/publicdomain/zero/1.0/
The derived work in the nifi-metrics module is adapted from
https://github.com/dropwizard/metrics/blob/v2.2.0/metrics-core/src/main/java/com/yammer/metrics/core/VirtualMachineMetrics.java
and can be found in
nifi-commons/nifi-metrics/src/main/java/org/apache/nifi/metrics/jvm/JvmMetrics.java
nifi-commons/nifi-metrics/src/main/java/org/apache/nifi/metrics/jvm/JmxJvmMetrics.java

54
nifi-nar-bundles/nifi-metrics-reporting-bundle/nifi-metrics-reporting-task/pom.xml
View File

@ -1,54 +0,0 @@
<?xml version="1.0" encoding="UTF-8"?>
<!--
Licensed to the Apache Software Foundation (ASF) under one or more
contributor license agreements. See the NOTICE file distributed with
this work for additional information regarding copyright ownership.
The ASF licenses this file to You under the Apache License, Version 2.0
(the "License"); you may not use this file except in compliance with
the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
-->
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/xsd/maven-4.0.0.xsd">
<parent>
<artifactId>nifi-metrics-reporting-bundle</artifactId>
<groupId>org.apache.nifi</groupId>
<version>2.0.0-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
<artifactId>nifi-metrics-reporting-task</artifactId>
<dependencies>
<dependency>
<groupId>org.apache.nifi</groupId>
<artifactId>nifi-utils</artifactId>
<version>2.0.0-SNAPSHOT</version>
</dependency>
<dependency>
<groupId>org.apache.nifi</groupId>
<artifactId>nifi-metrics-reporter-service-api</artifactId>
<version>2.0.0-SNAPSHOT</version>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>io.dropwizard.metrics</groupId>
<artifactId>metrics-graphite</artifactId>
<version>4.2.19</version>
</dependency>
<dependency>
<groupId>org.apache.nifi</groupId>
<artifactId>nifi-metrics</artifactId>
</dependency>
<dependency>
<groupId>org.apache.nifi</groupId>
<artifactId>nifi-mock</artifactId>
<version>2.0.0-SNAPSHOT</version>
<scope>test</scope>
</dependency>
</dependencies>
</project>

95
nifi-nar-bundles/nifi-metrics-reporting-bundle/nifi-metrics-reporting-task/src/main/java/org/apache/nifi/metrics/FlowMetricSet.java
View File

@ -1,95 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.nifi.metrics;
import com.codahale.metrics.Gauge;
import com.codahale.metrics.Metric;
import com.codahale.metrics.MetricSet;
import org.apache.nifi.controller.status.ProcessGroupStatus;
import org.apache.nifi.controller.status.ProcessorStatus;
import java.util.HashMap;
import java.util.Map;
import java.util.concurrent.atomic.AtomicReference;
/**
* A metric set of NiFi instance related metrics.
*
* @author Omer Hadari
*/
public class FlowMetricSet implements MetricSet {
/**
* Reference to the process status that should be reported. Should be updated when the status changes.
*/
private final AtomicReference<ProcessGroupStatus> currentStatusReference;
/**
* Create a metric set that will look at a given process status reference for deciding metrics.
*
* @param currentStatusReference a reference to the process status.
*/
public FlowMetricSet(AtomicReference<ProcessGroupStatus> currentStatusReference) {
this.currentStatusReference = currentStatusReference;
}
/**
* Create a map of {@link Gauge}s for the {@link #currentStatusReference}. This methods reports the metrics as
* found in the reference.
*
* @return map between the metric name and a {@link Gauge} to it's value.
*/
@Override
public Map<String, Metric> getMetrics() {
Map<String, Metric> metrics = new HashMap<>();
metrics.put(MetricNames.ACTIVE_THREADS, (Gauge<Integer>) () -> currentStatusReference.get().getActiveThreadCount());
metrics.put(MetricNames.BYTES_QUEUED, (Gauge<Long>) () -> currentStatusReference.get().getQueuedContentSize());
metrics.put(MetricNames.BYTES_READ, (Gauge<Long>) () -> currentStatusReference.get().getBytesRead());
metrics.put(MetricNames.BYTES_RECEIVED, (Gauge<Long>) () -> currentStatusReference.get().getBytesReceived());
metrics.put(MetricNames.BYTES_SENT, (Gauge<Long>) () -> currentStatusReference.get().getBytesSent());
metrics.put(MetricNames.BYTES_WRITTEN, (Gauge<Long>) () -> currentStatusReference.get().getBytesWritten());
metrics.put(MetricNames.FLOW_FILES_RECEIVED, (Gauge<Integer>) () -> currentStatusReference.get().getFlowFilesReceived());
metrics.put(MetricNames.FLOW_FILES_QUEUED, (Gauge<Integer>) () -> currentStatusReference.get().getQueuedCount());
metrics.put(MetricNames.FLOW_FILES_SENT, (Gauge<Integer>) () -> currentStatusReference.get().getFlowFilesSent());
metrics.put(MetricNames.TOTAL_TASK_DURATION_NANOS, (Gauge<Long>) () -> calculateProcessingNanos(currentStatusReference.get()));
return metrics;
}
/**
* Calculate the total processing time of a process group.
*
* @param status the current process group status.
* @return the total amount of nanoseconds spent in each processor in the process group.
*/
private long calculateProcessingNanos(final ProcessGroupStatus status) {
long nanos = 0L;
for (final ProcessorStatus procStats : status.getProcessorStatus()) {
nanos += procStats.getProcessingNanos();
}
for (final ProcessGroupStatus childGroupStatus : status.getProcessGroupStatus()) {
nanos += calculateProcessingNanos(childGroupStatus);
}
return nanos;
}
}

35
nifi-nar-bundles/nifi-metrics-reporting-bundle/nifi-metrics-reporting-task/src/main/java/org/apache/nifi/metrics/MetricNames.java
View File

@ -1,35 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.nifi.metrics;
/**
* The Metric names to send to Ambari.
*/
public interface MetricNames {
// NiFi Metrics
String FLOW_FILES_RECEIVED = "FlowFilesReceivedLast5Minutes";
String BYTES_RECEIVED = "BytesReceivedLast5Minutes";
String FLOW_FILES_SENT = "FlowFilesSentLast5Minutes";
String BYTES_SENT = "BytesSentLast5Minutes";
String FLOW_FILES_QUEUED = "FlowFilesQueued";
String BYTES_QUEUED = "BytesQueued";
String BYTES_READ = "BytesReadLast5Minutes";
String BYTES_WRITTEN = "BytesWrittenLast5Minutes";
String ACTIVE_THREADS = "ActiveThreads";
String TOTAL_TASK_DURATION_NANOS = "TotalTaskDurationNanoSeconds";
}

181
nifi-nar-bundles/nifi-metrics-reporting-bundle/nifi-metrics-reporting-task/src/main/java/org/apache/nifi/metrics/reporting/reporter/service/GraphiteMetricReporterService.java
View File

@ -1,181 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.nifi.metrics.reporting.reporter.service;
import com.codahale.metrics.MetricRegistry;
import com.codahale.metrics.ScheduledReporter;
import com.codahale.metrics.graphite.Graphite;
import com.codahale.metrics.graphite.GraphiteReporter;
import com.codahale.metrics.graphite.GraphiteSender;
import org.apache.nifi.annotation.documentation.CapabilityDescription;
import org.apache.nifi.annotation.documentation.Tags;
import org.apache.nifi.annotation.lifecycle.OnDisabled;
import org.apache.nifi.annotation.lifecycle.OnEnabled;
import org.apache.nifi.components.PropertyDescriptor;
import org.apache.nifi.controller.AbstractControllerService;
import org.apache.nifi.controller.ConfigurationContext;
import org.apache.nifi.expression.ExpressionLanguageScope;
import org.apache.nifi.metrics.reporting.task.MetricsReportingTask;
import org.apache.nifi.processor.util.StandardValidators;
import javax.net.SocketFactory;
import java.io.IOException;
import java.nio.charset.Charset;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
/**
* A controller service that provides metric reporters for graphite, can be used by {@link MetricsReportingTask}.
*
* @author Omer Hadari
*/
@Tags({"metrics", "reporting", "graphite"})
@CapabilityDescription("A controller service that provides metric reporters for graphite. " +
"Used by MetricsReportingTask.")
public class GraphiteMetricReporterService extends AbstractControllerService implements MetricReporterService {
/**
* Points to the hostname of the graphite listener.
*/
public static final PropertyDescriptor HOST = new PropertyDescriptor.Builder()
.name("host")
.displayName("Host")
.description("The hostname of the carbon listener")
.required(true)
.addValidator(StandardValidators.URI_VALIDATOR)
.expressionLanguageSupported(ExpressionLanguageScope.VARIABLE_REGISTRY)
.build();
/**
* Points to the port on which the graphite server listens.
*/
public static final PropertyDescriptor PORT = new PropertyDescriptor.Builder()
.name("port")
.displayName("Port")
.description("The port on which carbon listens")
.required(true)
.addValidator(StandardValidators.PORT_VALIDATOR)
.expressionLanguageSupported(ExpressionLanguageScope.VARIABLE_REGISTRY)
.build();
/**
* Points to the charset name that the graphite server expects.
*/
public static final PropertyDescriptor CHARSET = new PropertyDescriptor.Builder()
.name("charset")
.displayName("Charset")
.description("The charset used by the graphite server")
.required(true)
.defaultValue("UTF-8")
.addValidator(StandardValidators.CHARACTER_SET_VALIDATOR)
.build();
/**
* Prefix for all metric names sent by reporters - for separation of NiFi stats in graphite.
*/
protected static final PropertyDescriptor METRIC_NAME_PREFIX = new PropertyDescriptor.Builder()
.name("metric name prefix")
.displayName("Metric Name Prefix")
.description("A prefix that will be used for all metric names sent by reporters provided by this service.")
.required(true)
.defaultValue("nifi")
.addValidator(StandardValidators.NON_BLANK_VALIDATOR)
.expressionLanguageSupported(ExpressionLanguageScope.VARIABLE_REGISTRY)
.build();
/**
* List of property descriptors used by the service.
*/
private static final List<PropertyDescriptor> properties;
static {
final List<PropertyDescriptor> props = new ArrayList<>();
props.add(HOST);
props.add(PORT);
props.add(CHARSET);
props.add(METRIC_NAME_PREFIX);
properties = Collections.unmodifiableList(props);
}
/**
* Graphite sender, a connection to the server.
*/
private GraphiteSender graphiteSender;
/**
* The configured {@link #METRIC_NAME_PREFIX} value.
*/
private String metricNamePrefix;
/**
* Create the {@link #graphiteSender} according to configuration.
*
* @param context used to access properties.
*/
@OnEnabled
public void onEnabled(final ConfigurationContext context) {
String host = context.getProperty(HOST).evaluateAttributeExpressions().getValue();
int port = context.getProperty(PORT).evaluateAttributeExpressions().asInteger();
Charset charset = Charset.forName(context.getProperty(CHARSET).getValue());
graphiteSender = createSender(host, port, charset);
metricNamePrefix = context.getProperty(METRIC_NAME_PREFIX).evaluateAttributeExpressions().getValue();
}
/**
* Close the graphite sender.
*
* @throws IOException if failed to close the connection.
*/
@OnDisabled
public void shutdown() throws IOException {
try {
graphiteSender.close();
} finally {
graphiteSender = null;
}
}
/**
* Use the {@link #graphiteSender} in order to create a reporter.
*
* @param metricRegistry registry with the metrics to report.
* @return a reporter instance.
*/
@Override
public ScheduledReporter createReporter(MetricRegistry metricRegistry) {
return GraphiteReporter.forRegistry(metricRegistry).prefixedWith(metricNamePrefix).build(graphiteSender);
}
/**
* Create a sender.
*
* @param host the hostname of the server to connect to.
* @param port the port on which the server listens.
* @param charset the charset in which the server expects logs.
* @return The created sender.
*/
protected GraphiteSender createSender(String host, int port, Charset charset) {
return new Graphite(host, port, SocketFactory.getDefault(), charset);
}
@Override
protected List<PropertyDescriptor> getSupportedPropertyDescriptors() {
return properties;
}
}

152
nifi-nar-bundles/nifi-metrics-reporting-bundle/nifi-metrics-reporting-task/src/main/java/org/apache/nifi/metrics/reporting/task/MetricsReportingTask.java
View File

@ -1,152 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.nifi.metrics.reporting.task;
import com.codahale.metrics.MetricRegistry;
import com.codahale.metrics.ScheduledReporter;
import com.codahale.metrics.jvm.MemoryUsageGaugeSet;
import org.apache.nifi.annotation.documentation.CapabilityDescription;
import org.apache.nifi.annotation.documentation.Tags;
import org.apache.nifi.annotation.lifecycle.OnScheduled;
import org.apache.nifi.components.PropertyDescriptor;
import org.apache.nifi.controller.ConfigurationContext;
import org.apache.nifi.controller.status.ProcessGroupStatus;
import org.apache.nifi.expression.ExpressionLanguageScope;
import org.apache.nifi.metrics.FlowMetricSet;
import org.apache.nifi.metrics.reporting.reporter.service.MetricReporterService;
import org.apache.nifi.processor.util.StandardValidators;
import org.apache.nifi.reporting.AbstractReportingTask;
import org.apache.nifi.reporting.ReportingContext;
import org.apache.nifi.reporting.ReportingInitializationContext;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import java.util.concurrent.atomic.AtomicReference;
/**
* A reporting task for NiFi instance and JVM related metrics.
* <p>
* This task reports metrics to services according to a provided {@link ScheduledReporter}, reached by using a
* {@link MetricReporterService}. In order to report to different clients, simply use different implementations of
* the controller service.
*
* @author Omer Hadari
* @see MetricReporterService
*/
@Tags({"metrics", "reporting"})
@CapabilityDescription("This reporting task reports a set of metrics regarding the JVM and the NiFi instance" +
"to a reporter. The reporter is provided by a MetricReporterService. It can be optionally used for a specific" +
"process group if a property with the group id is provided.")
public class MetricsReportingTask extends AbstractReportingTask {
/**
* Points to the service which provides {@link ScheduledReporter} instances.
*/
protected static final PropertyDescriptor REPORTER_SERVICE = new PropertyDescriptor.Builder()
.name("metric reporter service")
.displayName("Metric Reporter Service")
.description("The service that provides a reporter for the gathered metrics")
.identifiesControllerService(MetricReporterService.class)
.required(true)
.build();
/**
* Metrics of the process group with this ID should be reported. If not specified, use the root process group.
*/
protected static final PropertyDescriptor PROCESS_GROUP_ID = new PropertyDescriptor.Builder()
.name("process group id")
.displayName("Process Group ID")
.description("The id of the process group to report. If not specified, metrics of the root process group" +
"are reported.")
.required(false)
.expressionLanguageSupported(ExpressionLanguageScope.VARIABLE_REGISTRY)
.addValidator(StandardValidators.NON_EMPTY_VALIDATOR)
.build();
/**
* Contains the metrics that should be reported.
*/
private MetricRegistry metricRegistry;
/**
* Used for actually reporting metrics.
*/
private ScheduledReporter reporter;
// Protected for testing sake. DO NOT ACCESS FOR OTHER PURPOSES.
/**
* Points to the most recent process group status seen by this task.
*/
protected AtomicReference<ProcessGroupStatus> currentStatusReference;
/**
* Register all wanted metrics to {@link #metricRegistry}.
* <p>
* {@inheritDoc}
*/
@Override
protected void init(ReportingInitializationContext config) {
metricRegistry = new MetricRegistry();
currentStatusReference = new AtomicReference<>();
metricRegistry.registerAll(new MemoryUsageGaugeSet());
metricRegistry.registerAll(new FlowMetricSet(currentStatusReference));
}
/**
* Populate {@link #reporter} using the {@link MetricReporterService}. If the reporter is active already,
* do nothing.
*
* @param context used for accessing the controller service.
*/
@OnScheduled
public void connect(ConfigurationContext context) {
if (reporter == null) {
reporter = ((MetricReporterService) context.getProperty(REPORTER_SERVICE).asControllerService())
.createReporter(metricRegistry);
}
}
/**
* Report the registered metrics.
*
* @param context used for getting the most recent {@link ProcessGroupStatus}.
*/
@Override
public void onTrigger(ReportingContext context) {
String groupId = context.getProperty(PROCESS_GROUP_ID).evaluateAttributeExpressions().getValue();
ProcessGroupStatus statusToReport = groupId == null
? context.getEventAccess().getControllerStatus()
: context.getEventAccess().getGroupStatus(groupId);
if (statusToReport != null) {
currentStatusReference.set(statusToReport);
reporter.report();
} else {
getLogger().error("Process group with provided group id could not be found.");
}
}
@Override
protected List<PropertyDescriptor> getSupportedPropertyDescriptors() {
List<PropertyDescriptor> properties = new ArrayList<>();
properties.add(REPORTER_SERVICE);
properties.add(PROCESS_GROUP_ID);
return Collections.unmodifiableList(properties);
}
}

15
nifi-nar-bundles/nifi-metrics-reporting-bundle/nifi-metrics-reporting-task/src/main/resources/META-INF/services/org.apache.nifi.controller.ControllerService
View File

@ -1,15 +0,0 @@
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
org.apache.nifi.metrics.reporting.reporter.service.GraphiteMetricReporterService

15
nifi-nar-bundles/nifi-metrics-reporting-bundle/nifi-metrics-reporting-task/src/main/resources/META-INF/services/org.apache.nifi.reporting.ReportingTask
View File

@ -1,15 +0,0 @@
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
org.apache.nifi.metrics.reporting.task.MetricsReportingTask

209
nifi-nar-bundles/nifi-metrics-reporting-bundle/nifi-metrics-reporting-task/src/test/java/org/apache/nifi/metrics/reporting/reporter/service/GraphiteMetricReporterServiceTest.java
View File

@ -1,209 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.nifi.metrics.reporting.reporter.service;
import com.codahale.metrics.Gauge;
import com.codahale.metrics.MetricRegistry;
import com.codahale.metrics.ScheduledReporter;
import com.codahale.metrics.graphite.GraphiteSender;
import org.apache.nifi.processor.Processor;
import org.apache.nifi.util.TestRunner;
import org.apache.nifi.util.TestRunners;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith;
import org.mockito.Mock;
import org.mockito.junit.jupiter.MockitoExtension;
import java.nio.charset.Charset;
import java.nio.charset.StandardCharsets;
import static org.mockito.ArgumentMatchers.anyLong;
import static org.mockito.ArgumentMatchers.eq;
import static org.mockito.Mockito.verify;
/**
* Test class for {@link GraphiteMetricReporterService}.
*
* @author Omer Hadari
*/
@ExtendWith(MockitoExtension.class)
public class GraphiteMetricReporterServiceTest {
/**
* Service identifier for registerting the tested service to the tests runner.
*/
private static final String SERVICE_IDENTIFIER = "graphite-metric-reporter-service";
/**
* Sample host name for the {@link GraphiteMetricReporterService#HOST} property.
*/
private static final String TEST_HOST = "some-host";
/**
* Sample port for the {@link GraphiteMetricReporterService#PORT} property.
*/
private static final int TEST_PORT = 12345;
/**
* Sample charset for the {@link GraphiteMetricReporterService#CHARSET} property.
*/
private static final Charset TEST_CHARSET = StandardCharsets.UTF_16LE;
/**
* Sample prefix for metric names.
*/
private static final String METRIC_NAMES_PREFIX = "test-metric-name-prefix";
/**
* Sample metric for verifying that a graphite sender with the correct configuration is used.
*/
private static final String TEST_METRIC_NAME = "test-metric";
/**
* The fixed value of {@link #TEST_METRIC_NAME}.
*/
private static final int TEST_METRIC_VALUE = 2;
/**
* Dummy processor for creating {@link #runner}.
*/
@Mock
private Processor processorDummy;
/**
* Mock sender for verifying creation with the correct configuration.
*/
@Mock
private GraphiteSender graphiteSenderMock;
/**
* Stub metric registry, that contains the test metrics.
*/
private MetricRegistry metricRegistryStub;
/**
* Test runner for activating and configuring the service.
*/
private TestRunner runner;
/**
* The test subject.
*/
private GraphiteMetricReporterService testedService;
/**
* Instantiate the runner and mocks between tests. Register metrics to the {@link #metricRegistryStub}.
*/
@BeforeEach
public void setUp() throws Exception {
runner = TestRunners.newTestRunner(processorDummy);
testedService = new GraphiteMetricReporterService();
metricRegistryStub = new MetricRegistry();
metricRegistryStub.register(TEST_METRIC_NAME, ((Gauge<Integer>) () -> TEST_METRIC_VALUE));
}
/**
* Make sure that a correctly configured service can be activated.
*/
@Test
public void testGraphiteMetricReporterSanityConfiguration() throws Exception {
runner.addControllerService(SERVICE_IDENTIFIER, testedService);
setServiceProperties(TEST_HOST, TEST_PORT, TEST_CHARSET, METRIC_NAMES_PREFIX);
runner.enableControllerService(testedService);
runner.assertValid(testedService);
}
/**
* Make sure that a correctly configured service provides a reporter for the matching configuration, and
* actually reports to the correct address.
*/
@Test
public void testCreateReporterUsesCorrectSender() throws Exception {
testedService = new TestableGraphiteMetricReporterService();
runner.addControllerService(SERVICE_IDENTIFIER, testedService);
setServiceProperties(TEST_HOST, TEST_PORT, TEST_CHARSET, METRIC_NAMES_PREFIX);
runner.enableControllerService(testedService);
ScheduledReporter createdReporter = testedService.createReporter(metricRegistryStub);
createdReporter.report();
String expectedMetricName = MetricRegistry.name(METRIC_NAMES_PREFIX, TEST_METRIC_NAME);
verify(graphiteSenderMock).send(eq(expectedMetricName), eq(String.valueOf(TEST_METRIC_VALUE)), anyLong());
}
/**
* Make sure that {@link GraphiteMetricReporterService#shutdown()} closes the connection to graphite.
*/
@Test
public void testShutdownClosesSender() throws Exception {
testedService = new TestableGraphiteMetricReporterService();
runner.addControllerService(SERVICE_IDENTIFIER, testedService);
setServiceProperties(TEST_HOST, TEST_PORT, TEST_CHARSET, METRIC_NAMES_PREFIX);
runner.enableControllerService(testedService);
runner.disableControllerService(testedService);
verify(graphiteSenderMock).close();
}
/**
* Set the test subject's properties.
*
* @param host populates {@link GraphiteMetricReporterService#HOST}.
* @param port populates {@link GraphiteMetricReporterService#PORT}.
* @param charset populates {@link GraphiteMetricReporterService#CHARSET}.
* @param metricNamesPrefix populates {@link GraphiteMetricReporterService#METRIC_NAME_PREFIX}.
*/
private void setServiceProperties(String host, int port, Charset charset, String metricNamesPrefix) {
runner.setProperty(testedService, GraphiteMetricReporterService.HOST, host);
runner.setProperty(testedService, GraphiteMetricReporterService.PORT, String.valueOf(port));
runner.setProperty(testedService, GraphiteMetricReporterService.CHARSET, charset.name());
runner.setProperty(testedService, GraphiteMetricReporterService.METRIC_NAME_PREFIX, metricNamesPrefix);
}
/**
* This class is a patch. It overrides {@link GraphiteMetricReporterService#createSender(String, int, Charset)}
* so that it is possible to verify a correct creation of graphite senders according to property values.
*/
private class TestableGraphiteMetricReporterService extends GraphiteMetricReporterService {
/**
* Overrides the actual methods in order to inject the mock {@link #graphiteSenderMock}.
* <p>
* If this method is called with the test property values, it returns the mock. Otherwise operate
* regularly.
*
* @param host the provided hostname.
* @param port the provided port.
* @param charset the provided graphite server charset.
* @return {@link #graphiteSenderMock} if all params were the constant test params, regular result otherwise.
*/
@Override
protected GraphiteSender createSender(String host, int port, Charset charset) {
if (TEST_HOST.equals(host) && TEST_PORT == port && TEST_CHARSET.equals(charset)) {
return graphiteSenderMock;
}
return super.createSender(host, port, charset);
}
}
}

256
nifi-nar-bundles/nifi-metrics-reporting-bundle/nifi-metrics-reporting-task/src/test/java/org/apache/nifi/metrics/reporting/task/MetricsReportingTaskTest.java
View File

@ -1,256 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.nifi.metrics.reporting.task;
import com.codahale.metrics.Metric;
import com.codahale.metrics.MetricRegistry;
import com.codahale.metrics.ScheduledReporter;
import com.codahale.metrics.jvm.MemoryUsageGaugeSet;
import org.apache.nifi.components.PropertyDescriptor;
import org.apache.nifi.controller.ConfigurationContext;
import org.apache.nifi.controller.ControllerService;
import org.apache.nifi.controller.status.ProcessGroupStatus;
import org.apache.nifi.metrics.FlowMetricSet;
import org.apache.nifi.metrics.reporting.reporter.service.MetricReporterService;
import org.apache.nifi.reporting.ReportingContext;
import org.apache.nifi.reporting.ReportingInitializationContext;
import org.apache.nifi.state.MockStateManager;
import org.apache.nifi.util.MockComponentLog;
import org.apache.nifi.util.MockConfigurationContext;
import org.apache.nifi.util.MockReportingContext;
import org.apache.nifi.util.MockReportingInitializationContext;
import org.apache.nifi.util.MockVariableRegistry;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith;
import org.mockito.ArgumentCaptor;
import org.mockito.Mock;
import org.mockito.junit.jupiter.MockitoExtension;
import org.mockito.junit.jupiter.MockitoSettings;
import org.mockito.quality.Strictness;
import java.util.Arrays;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertNull;
import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.Mockito.never;
import static org.mockito.Mockito.times;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
/**
* Test class for {@link MetricsReportingTask}.
*
* @author Omer Hadari
*/
@ExtendWith(MockitoExtension.class)
@MockitoSettings(strictness = Strictness.LENIENT)
public class MetricsReportingTaskTest {
/**
* Identifier for {@link #reporterServiceStub}.
*/
private static final String REPORTER_SERVICE_IDENTIFIER = "reporter-service";
/**
* Id for the group with status {@link #innerGroupStatus}.
*/
private static final String TEST_GROUP_ID = "test-process-group-id";
/**
* Id for the {@link #reportingInitContextStub}.
*/
private static final String TEST_INIT_CONTEXT_ID = "test-init-context-id";
/**
* Name for {@link #reportingInitContextStub}.
*/
private static final String TEST_INIT_CONTEXT_NAME = "test-init-context-name";
/**
* Id for the tested tested reporting task.
*/
private static final String TEST_TASK_ID = "test-task-id";
/**
* Stub context, used by {@link MetricsReportingTask#onTrigger(ReportingContext)} for reaching the status.
*/
private MockReportingContext reportingContextStub;
/**
* Stub context, used by {@link MetricsReportingTask#connect(ConfigurationContext)} for reaching the service.
*/
private MockConfigurationContext configurationContextStub;
/**
* Stub service for providing {@link #reporterMock}, used for actual reporting
*/
@Mock
private MetricReporterService reporterServiceStub;
/**
* Mock reporter, used for verifying actual reporting.
*/
@Mock
private ScheduledReporter reporterMock;
/**
* A status for the "root" process group.
*/
private ProcessGroupStatus rootGroupStatus;
/**
* Same as {@link #rootGroupStatus}, used when {@link MetricsReportingTask#PROCESS_GROUP_ID} is set.
*/
private ProcessGroupStatus innerGroupStatus;
/**
* Stub initialization context for calling {@link MetricsReportingTask#initialize(ReportingInitializationContext)}.
*/
private MockReportingInitializationContext reportingInitContextStub;
/**
* The test subject.
*/
private MetricsReportingTask testedReportingTask;
/**
* Set up the test environment and mock behaviour. This includes registering {@link #reporterServiceStub} in the
* different contexts, overriding {@link MetricsReportingTask#currentStatusReference} and instantiating the test
* subject.
*/
@BeforeEach
public void setUp() throws Exception {
Map<String, ControllerService> services = new HashMap<>();
services.put(REPORTER_SERVICE_IDENTIFIER, reporterServiceStub);
testedReportingTask = new MetricsReportingTask();
reportingContextStub = new MockReportingContext(
services, new MockStateManager(testedReportingTask), new MockVariableRegistry());
rootGroupStatus = new ProcessGroupStatus();
innerGroupStatus = new ProcessGroupStatus();
when(reporterServiceStub.createReporter(any())).thenReturn(reporterMock);
reportingContextStub.setProperty(MetricsReportingTask.REPORTER_SERVICE.getName(), REPORTER_SERVICE_IDENTIFIER);
reportingContextStub.addControllerService(reporterServiceStub, REPORTER_SERVICE_IDENTIFIER);
configurationContextStub = new MockConfigurationContext(reportingContextStub.getProperties(),
reportingContextStub.getControllerServiceLookup());
reportingInitContextStub = new MockReportingInitializationContext(
TEST_INIT_CONTEXT_ID,
TEST_INIT_CONTEXT_NAME,
new MockComponentLog(TEST_TASK_ID, testedReportingTask));
}
/**
* Make sure that in a single life cycle the correct metrics are registered, the correct {@link ProcessGroupStatus}
* is used and that metrics are actually reported.
*/
@Test
public void testValidLifeCycleReportsCorrectly() throws Exception {
reportingContextStub.getEventAccess().setProcessGroupStatus(rootGroupStatus);
testedReportingTask.initialize(reportingInitContextStub);
testedReportingTask.connect(configurationContextStub);
testedReportingTask.onTrigger(reportingContextStub);
verify(reporterMock).report();
// Verify correct metrics are registered
ArgumentCaptor<MetricRegistry> registryCaptor = ArgumentCaptor.forClass(MetricRegistry.class);
verify(reporterServiceStub).createReporter(registryCaptor.capture());
MetricRegistry usedRegistry = registryCaptor.getValue();
Map<String, Metric> usedMetrics = usedRegistry.getMetrics();
assertTrue(usedMetrics.keySet().containsAll(new MemoryUsageGaugeSet().getMetrics().keySet()));
assertTrue(usedMetrics.keySet()
.containsAll(new FlowMetricSet(testedReportingTask.currentStatusReference).getMetrics().keySet()));
// Verify the most current ProcessGroupStatus is updated
assertEquals(testedReportingTask.currentStatusReference.get(), rootGroupStatus);
}
/**
* Make sure that in a single life cycle the correct metrics are registered, the correct {@link ProcessGroupStatus}
* is used and that metrics are actually reported.
*/
@Test
public void testValidLifeCycleReportsCorrectlyProcessGroupSpecified() throws Exception {
reportingContextStub.setProperty(MetricsReportingTask.PROCESS_GROUP_ID.getName(), TEST_GROUP_ID);
reportingContextStub.getEventAccess().setProcessGroupStatus(TEST_GROUP_ID, innerGroupStatus);
testedReportingTask.initialize(reportingInitContextStub);
testedReportingTask.connect(configurationContextStub);
testedReportingTask.onTrigger(reportingContextStub);
verify(reporterMock).report();
// Verify correct metrics are registered
ArgumentCaptor<MetricRegistry> registryCaptor = ArgumentCaptor.forClass(MetricRegistry.class);
verify(reporterServiceStub).createReporter(registryCaptor.capture());
MetricRegistry usedRegistry = registryCaptor.getValue();
Map<String, Metric> usedMetrics = usedRegistry.getMetrics();
assertTrue(usedMetrics.keySet().containsAll(new MemoryUsageGaugeSet().getMetrics().keySet()));
assertTrue(usedMetrics.keySet()
.containsAll(new FlowMetricSet(testedReportingTask.currentStatusReference).getMetrics().keySet()));
// Verify the most current ProcessGroupStatus is updated
assertEquals(testedReportingTask.currentStatusReference.get(), innerGroupStatus);
}
/**
* Make sure that in a single life cycle the correct metrics are registered, the correct {@link ProcessGroupStatus}
* is used and that metrics are actually reported.
*/
@Test
public void testInvalidProcessGroupId() throws Exception {
reportingContextStub.setProperty(MetricsReportingTask.PROCESS_GROUP_ID.getName(), TEST_GROUP_ID + "-invalid");
reportingContextStub.getEventAccess().setProcessGroupStatus(TEST_GROUP_ID, innerGroupStatus);
testedReportingTask.initialize(reportingInitContextStub);
testedReportingTask.connect(configurationContextStub);
testedReportingTask.onTrigger(reportingContextStub);
verify(reporterMock, never()).report();
assertNull(testedReportingTask.currentStatusReference.get());
}
/**
* Make sure that {@link MetricsReportingTask#connect(ConfigurationContext)} does not create a new reporter
* if there is already an active reporter.
*/
@Test
public void testConnectCreatesSingleReporter() throws Exception {
testedReportingTask.initialize(reportingInitContextStub);
testedReportingTask.connect(configurationContextStub);
testedReportingTask.connect(configurationContextStub);
verify(reporterServiceStub, times(1)).createReporter(any());
}
/**
* Sanity check for registered properties.
*/
@Test
public void testGetSupportedPropertyDescriptorsSanity() throws Exception {
List<PropertyDescriptor> expected = Arrays.asList(
MetricsReportingTask.REPORTER_SERVICE,
MetricsReportingTask.PROCESS_GROUP_ID);
assertEquals(expected, testedReportingTask.getSupportedPropertyDescriptors());
}
}

55
nifi-nar-bundles/nifi-metrics-reporting-bundle/pom.xml
View File

@ -1,55 +0,0 @@
<?xml version="1.0" encoding="UTF-8"?>
<!--
Licensed to the Apache Software Foundation (ASF) under one or more
contributor license agreements. See the NOTICE file distributed with
this work for additional information regarding copyright ownership.
The ASF licenses this file to You under the Apache License, Version 2.0
(the "License"); you may not use this file except in compliance with
the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
-->
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/xsd/maven-4.0.0.xsd">
<parent>
<artifactId>nifi-nar-bundles</artifactId>
<groupId>org.apache.nifi</groupId>
<version>2.0.0-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
<artifactId>nifi-metrics-reporting-bundle</artifactId>
<packaging>pom</packaging>
<modules>
<module>nifi-metrics-reporting-task</module>
<module>nifi-metrics-reporting-nar</module>
<module>nifi-metrics-reporter-service-api</module>
<module>nifi-metrics-reporter-service-api-nar</module>
</modules>
<dependencyManagement>
<dependencies>
<dependency>
<groupId>org.apache.nifi</groupId>
<artifactId>nifi-metrics</artifactId>
<version>2.0.0-SNAPSHOT</version>
</dependency>
<dependency>
<groupId>org.apache.nifi</groupId>
<artifactId>nifi-metrics-reporting-task</artifactId>
<version>2.0.0-SNAPSHOT</version>
</dependency>
</dependencies>
</dependencyManagement>
<dependencies>
<dependency>
<groupId>org.apache.nifi</groupId>
<artifactId>nifi-api</artifactId>
<scope>provided</scope>
</dependency>
</dependencies>
</project>

36
nifi-nar-bundles/nifi-riemann-bundle/nifi-riemann-nar/pom.xml
View File

@ -1,36 +0,0 @@
<?xml version="1.0" encoding="UTF-8"?>
<!--
Licensed to the Apache Software Foundation (ASF) under one or more
contributor license agreements. See the NOTICE file distributed with
this work for additional information regarding copyright ownership.
The ASF licenses this file to You under the Apache License, Version 2.0
(the "License"); you may not use this file except in compliance with
the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
-->
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<parent>
<artifactId>nifi-riemann-bundle</artifactId>
<groupId>org.apache.nifi</groupId>
<version>2.0.0-SNAPSHOT</version>
</parent>
<properties>
<maven.javadoc.skip>true</maven.javadoc.skip>
<source.skip>true</source.skip>
</properties>
<artifactId>nifi-riemann-nar</artifactId>
<packaging>nar</packaging>
<dependencies>
<dependency>
<groupId>org.apache.nifi</groupId>
<artifactId>nifi-riemann-processors</artifactId>
</dependency>
</dependencies>
</project>

19
nifi-nar-bundles/nifi-riemann-bundle/nifi-riemann-nar/src/main/resources/META-INF/NOTICE
View File

@ -1,19 +0,0 @@
nifi-riemann-nar
Copyright 2014-2023 The Apache Software Foundation
This product includes software developed at
The Apache Software Foundation (http://www.apache.org/).
===========================================
Apache Software License v2
===========================================
The following binary components are provided under the Apache Software License v2
(ASLv2) Apache Commons Lang
The following NOTICE information applies:
Apache Commons Lang
Copyright 2001-2015 The Apache Software Foundation
This product includes software from the Spring Framework,
under the Apache License 2.0 (see: StringUtils.containsWhitespace())

55
nifi-nar-bundles/nifi-riemann-bundle/nifi-riemann-processors/pom.xml
View File

@ -1,55 +0,0 @@
<?xml version="1.0" encoding="UTF-8"?>
<!--
Licensed to the Apache Software Foundation (ASF) under one or more
contributor license agreements. See the NOTICE file distributed with
this work for additional information regarding copyright ownership.
The ASF licenses this file to You under the Apache License, Version 2.0
(the "License"); you may not use this file except in compliance with
the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
-->
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>org.apache.nifi</groupId>
<artifactId>nifi-riemann-bundle</artifactId>
<version>2.0.0-SNAPSHOT</version>
</parent>
<artifactId>nifi-riemann-processors</artifactId>
<dependencies>
<dependency>
<groupId>io.riemann</groupId>
<artifactId>riemann-java-client</artifactId>
</dependency>
<dependency>
<groupId>org.apache.nifi</groupId>
<artifactId>nifi-api</artifactId>
</dependency>
<dependency>
<groupId>org.apache.nifi</groupId>
<artifactId>nifi-utils</artifactId>
<version>2.0.0-SNAPSHOT</version>
</dependency>
<dependency>
<groupId>org.apache.nifi</groupId>
<artifactId>nifi-properties</artifactId>
</dependency>
<dependency>
<groupId>org.apache.commons</groupId>
<artifactId>commons-lang3</artifactId>
</dependency>
<dependency>
<groupId>org.apache.nifi</groupId>
<artifactId>nifi-mock</artifactId>
<version>2.0.0-SNAPSHOT</version>
<scope>test</scope>
</dependency>
</dependencies>
</project>

382
nifi-nar-bundles/nifi-riemann-bundle/nifi-riemann-processors/src/main/java/org/apache/nifi/processors/riemann/PutRiemann.java
View File

@ -1,382 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.nifi.processors.riemann;
import io.riemann.riemann.Proto;
import io.riemann.riemann.Proto.Event;
import io.riemann.riemann.client.RiemannClient;
import org.apache.commons.lang3.StringUtils;
import org.apache.nifi.annotation.behavior.DynamicProperty;
import org.apache.nifi.annotation.behavior.InputRequirement;
import org.apache.nifi.annotation.behavior.InputRequirement.Requirement;
import org.apache.nifi.annotation.behavior.SupportsBatching;
import org.apache.nifi.annotation.documentation.CapabilityDescription;
import org.apache.nifi.annotation.documentation.Tags;
import org.apache.nifi.annotation.lifecycle.OnScheduled;
import org.apache.nifi.annotation.lifecycle.OnStopped;
import org.apache.nifi.components.PropertyDescriptor;
import org.apache.nifi.components.PropertyValue;
import org.apache.nifi.components.Validator;
import org.apache.nifi.expression.ExpressionLanguageScope;
import org.apache.nifi.flowfile.FlowFile;
import org.apache.nifi.processor.AbstractProcessor;
import org.apache.nifi.processor.ProcessContext;
import org.apache.nifi.processor.ProcessSession;
import org.apache.nifi.processor.Relationship;
import org.apache.nifi.processor.exception.ProcessException;
import org.apache.nifi.processor.util.StandardValidators;
import java.io.IOException;
import java.util.ArrayList;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.TimeUnit;
@Tags({"riemann", "monitoring", "metrics"})
@DynamicProperty(name = "Custom Event Attribute", expressionLanguageScope = ExpressionLanguageScope.FLOWFILE_ATTRIBUTES,
description = "These values will be attached to the Riemann event as a custom attribute",
value = "Any value or expression")
@CapabilityDescription("Send events to Riemann (http://riemann.io) when FlowFiles pass through this processor. " +
"You can use events to notify Riemann that a FlowFile passed through, or you can attach a more " +
"meaningful metric, such as, the time a FlowFile took to get to this processor. All attributes attached to " +
"events support the NiFi Expression Language.")
@SupportsBatching
@InputRequirement(Requirement.INPUT_REQUIRED)
public class PutRiemann extends AbstractProcessor {
protected enum Transport {
TCP, UDP
}
protected volatile RiemannClient riemannClient = null;
protected volatile Transport transport;
public static final Relationship REL_SUCCESS = new Relationship.Builder()
.name("success")
.description("Metrics successfully written to Riemann")
.build();
public static final Relationship REL_FAILURE = new Relationship.Builder()
.name("failure")
.description("Metrics which failed to write to Riemann")
.build();
public static final PropertyDescriptor RIEMANN_HOST = new PropertyDescriptor.Builder()
.name("Riemann Address")
.description("Hostname of Riemann server")
.required(true)
.addValidator(StandardValidators.NON_EMPTY_VALIDATOR)
.build();
public static final PropertyDescriptor RIEMANN_PORT = new PropertyDescriptor.Builder()
.name("Riemann Port")
.description("Port that Riemann is listening on")
.required(true)
.defaultValue("5555")
.addValidator(StandardValidators.PORT_VALIDATOR)
.build();
public static final PropertyDescriptor TRANSPORT_PROTOCOL = new PropertyDescriptor.Builder()
.name("Transport Protocol")
.description("Transport protocol to speak to Riemann in")
.required(true)
.allowableValues(new Transport[]{Transport.TCP, Transport.UDP})
.defaultValue("TCP")
.build();
public static final PropertyDescriptor BATCH_SIZE = new PropertyDescriptor.Builder()
.name("Batch Size")
.description("Batch size for incoming FlowFiles")
.required(false)
.defaultValue("100")
.addValidator(StandardValidators.POSITIVE_INTEGER_VALIDATOR)
.build();
// Attributes Mappings
public static final PropertyDescriptor ATTR_SERVICE = new PropertyDescriptor.Builder()
.name("Service")
.description("Name of service associated to this event (e.g. FTP File Fetched)")
.required(false)
.expressionLanguageSupported(ExpressionLanguageScope.FLOWFILE_ATTRIBUTES)
.addValidator(Validator.VALID)
.build();
public static final PropertyDescriptor ATTR_STATE = new PropertyDescriptor.Builder()
.name("State")
.description("State of service associated to this event in string form (e.g. ok, warning, foo)")
.required(false)
.expressionLanguageSupported(ExpressionLanguageScope.FLOWFILE_ATTRIBUTES)
.addValidator(Validator.VALID)
.build();
public static final PropertyDescriptor ATTR_TIME = new PropertyDescriptor.Builder()
.name("Time")
.description("Time of event in unix epoch seconds (long), default: (current time)")
.required(false)
.expressionLanguageSupported(ExpressionLanguageScope.FLOWFILE_ATTRIBUTES)
.addValidator(Validator.VALID)
.build();
public static final PropertyDescriptor ATTR_HOST = new PropertyDescriptor.Builder()
.name("Host")
.description("A hostname associated to this event (e.g. nifi-app1)")
.required(false)
.defaultValue("${hostname()}")
.expressionLanguageSupported(ExpressionLanguageScope.FLOWFILE_ATTRIBUTES)
.addValidator(Validator.VALID)
.build();
public static final PropertyDescriptor ATTR_TTL = new PropertyDescriptor.Builder()
.name("TTL")
.description("Floating point value in seconds until Riemann considers this event as \"expired\"")
.required(false)
.addValidator(Validator.VALID)
.expressionLanguageSupported(ExpressionLanguageScope.FLOWFILE_ATTRIBUTES)
.build();
public static final PropertyDescriptor ATTR_METRIC = new PropertyDescriptor.Builder()
.name("Metric")
.description("Floating point number associated to this event")
.required(false)
.addValidator(Validator.VALID)
.expressionLanguageSupported(ExpressionLanguageScope.FLOWFILE_ATTRIBUTES)
.build();
public static final PropertyDescriptor ATTR_DESCRIPTION = new PropertyDescriptor.Builder()
.name("Description")
.description("Description associated to the event")
.required(false)
.expressionLanguageSupported(ExpressionLanguageScope.FLOWFILE_ATTRIBUTES)
.addValidator(Validator.VALID)
.build();
public static final PropertyDescriptor ATTR_TAGS = new PropertyDescriptor.Builder()
.name("Tags")
.description("Comma separated list of tags associated to the event")
.required(false)
.expressionLanguageSupported(ExpressionLanguageScope.FLOWFILE_ATTRIBUTES)
.addValidator(Validator.VALID)
.build();
public static final PropertyDescriptor TIMEOUT = new PropertyDescriptor.Builder()
.name("Timeout")
.description("Timeout in milliseconds when writing events to Riemann")
.required(true)
.defaultValue("1000")
.addValidator(StandardValidators.POSITIVE_LONG_VALIDATOR)
.build();
private volatile List<PropertyDescriptor> customAttributes = new ArrayList<>();
private static final Set<Relationship> RELATIONSHIPS = new HashSet<>();
private static final List<PropertyDescriptor> LOCAL_PROPERTIES = new ArrayList<>();
private volatile int batchSize = -1;
private volatile long writeTimeout = 1000;
static {
RELATIONSHIPS.add(REL_SUCCESS);
RELATIONSHIPS.add(REL_FAILURE);
LOCAL_PROPERTIES.add(RIEMANN_HOST);
LOCAL_PROPERTIES.add(RIEMANN_PORT);
LOCAL_PROPERTIES.add(TRANSPORT_PROTOCOL);
LOCAL_PROPERTIES.add(TIMEOUT);
LOCAL_PROPERTIES.add(BATCH_SIZE);
LOCAL_PROPERTIES.add(ATTR_DESCRIPTION);
LOCAL_PROPERTIES.add(ATTR_SERVICE);
LOCAL_PROPERTIES.add(ATTR_STATE);
LOCAL_PROPERTIES.add(ATTR_METRIC);
LOCAL_PROPERTIES.add(ATTR_TTL);
LOCAL_PROPERTIES.add(ATTR_TAGS);
LOCAL_PROPERTIES.add(ATTR_HOST);
LOCAL_PROPERTIES.add(ATTR_TIME);
}
@Override
public Set<Relationship> getRelationships() {
return RELATIONSHIPS;
}
@Override
protected List<PropertyDescriptor> getSupportedPropertyDescriptors() {
return LOCAL_PROPERTIES;
}
@Override
protected PropertyDescriptor getSupportedDynamicPropertyDescriptor(final String propertyDescriptorName) {
return new PropertyDescriptor.Builder()
.name(propertyDescriptorName)
.expressionLanguageSupported(ExpressionLanguageScope.FLOWFILE_ATTRIBUTES)
.addValidator(Validator.VALID)
.required(false)
.dynamic(true)
.build();
}
@OnStopped
public final void cleanUpClient() {
if (riemannClient != null) {
this.riemannClient.close();
}
this.riemannClient = null;
this.batchSize = -1;
this.customAttributes.clear();
}
@OnScheduled
public void onScheduled(ProcessContext context) throws ProcessException {
if (batchSize == -1) {
batchSize = context.getProperty(BATCH_SIZE).asInteger();
}
if (riemannClient == null || !riemannClient.isConnected()) {
transport = Transport.valueOf(context.getProperty(TRANSPORT_PROTOCOL).getValue());
String host = context.getProperty(RIEMANN_HOST).getValue().trim();
int port = context.getProperty(RIEMANN_PORT).asInteger();
writeTimeout = context.getProperty(TIMEOUT).asLong();
RiemannClient client = null;
try {
switch (transport) {
case TCP:
client = RiemannClient.tcp(host, port);
break;
case UDP:
client = RiemannClient.udp(host, port);
break;
}
client.connect();
riemannClient = client;
} catch (IOException e) {
if (client != null) {
client.close();
}
context.yield();
throw new ProcessException(String.format("Unable to connect to Riemann [%s:%d] (%s)\n%s", host, port, transport, e.getMessage()));
}
}
if (customAttributes.size() == 0) {
for (Map.Entry<PropertyDescriptor, String> property : context.getProperties().entrySet()) {
// only custom defined properties
if (!getSupportedPropertyDescriptors().contains(property.getKey())) {
customAttributes.add(property.getKey());
}
}
}
}
@Override
public void onTrigger(ProcessContext context, ProcessSession session) throws ProcessException {
// Check if the client is currently connected, as a previous trigger could have detected a failure
// in the connection.
if (riemannClient == null || !riemannClient.isConnected()) {
// clean up the client and attempt to re-initialize the processor
cleanUpClient();
onScheduled(context);
}
List<FlowFile> incomingFlowFiles = session.get(batchSize);
List<FlowFile> successfulFlowFiles = new ArrayList<>(incomingFlowFiles.size());
List<Event> eventsQueue = new ArrayList<>(incomingFlowFiles.size());
for (FlowFile flowFile : incomingFlowFiles) {
try {
eventsQueue.add(FlowFileToEvent.fromAttributes(context, customAttributes, flowFile));
successfulFlowFiles.add(flowFile);
} catch (NumberFormatException e) {
getLogger().warn("Unable to create Riemann event.", e);
session.transfer(flowFile, REL_FAILURE);
}
}
try {
if (transport == Transport.TCP) {
Proto.Msg returnMessage = riemannClient.sendEvents(eventsQueue).deref(writeTimeout, TimeUnit.MILLISECONDS);
if (returnMessage == null) {
context.yield();
throw new ProcessException("Timed out writing to Riemann!");
}
} else {
riemannClient.sendEvents(eventsQueue);
}
riemannClient.flush();
session.transfer(successfulFlowFiles, REL_SUCCESS);
} catch (Exception e) {
context.yield();
session.transfer(incomingFlowFiles);
throw new ProcessException("Failed writing to Riemann\n" + e.getMessage());
}
}
/**
* Converts a FlowFile into a Riemann Protobuf Event
*/
private static class FlowFileToEvent {
protected static Event fromAttributes(ProcessContext context, List<PropertyDescriptor> customProperties,
FlowFile flowFile) {
Event.Builder builder = Event.newBuilder();
PropertyValue service = context.getProperty(ATTR_SERVICE).evaluateAttributeExpressions(flowFile);
if (StringUtils.isNotBlank(service.getValue())) {
builder.setService(service.getValue());
}
PropertyValue description = context.getProperty(ATTR_DESCRIPTION).evaluateAttributeExpressions(flowFile);
if (StringUtils.isNotBlank(description.getValue())) {
builder.setDescription(description.getValue());
}
PropertyValue metric = context.getProperty(ATTR_METRIC).evaluateAttributeExpressions(flowFile);
if (StringUtils.isNotBlank(metric.getValue())) {
builder.setMetricF(metric.asFloat());
}
PropertyValue time = context.getProperty(ATTR_TIME).evaluateAttributeExpressions(flowFile);
if (StringUtils.isNotBlank(time.getValue())) {
builder.setTime(time.asLong());
}
PropertyValue state = context.getProperty(ATTR_STATE).evaluateAttributeExpressions(flowFile);
if (StringUtils.isNotBlank(state.getValue())) {
builder.setState(state.getValue());
}
PropertyValue ttl = context.getProperty(ATTR_TTL).evaluateAttributeExpressions(flowFile);
if (StringUtils.isNotBlank(ttl.getValue())) {
builder.setTtl(ttl.asFloat());
}
PropertyValue host = context.getProperty(ATTR_HOST).evaluateAttributeExpressions(flowFile);
if (StringUtils.isNotBlank(host.getValue())) {
builder.setHost(host.getValue());
}
PropertyValue tags = context.getProperty(ATTR_TAGS).evaluateAttributeExpressions(flowFile);
if (StringUtils.isNotBlank(tags.getValue())) {
String[] splitTags = tags.getValue().split(",");
for (String splitTag : splitTags) {
builder.addTags(splitTag.trim());
}
}
PropertyValue customAttributeValue;
for (PropertyDescriptor customProperty : customProperties) {
customAttributeValue = context.getProperty(customProperty).evaluateAttributeExpressions(flowFile);
if (StringUtils.isNotBlank(customAttributeValue.getValue())) {
builder.addAttributes(Proto.Attribute.newBuilder()
.setKey(customProperty.getName())
.setValue(customAttributeValue.getValue())
.build());
}
}
return builder.build();
}
}
}

15
nifi-nar-bundles/nifi-riemann-bundle/nifi-riemann-processors/src/main/resources/META-INF/services/org.apache.nifi.processor.Processor
View File

@ -1,15 +0,0 @@
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
org.apache.nifi.processors.riemann.PutRiemann

188
nifi-nar-bundles/nifi-riemann-bundle/nifi-riemann-processors/src/test/java/org/apache/nifi/processors/riemann/TestPutRiemann.java
View File

@ -1,188 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.nifi.processors.riemann;
import io.riemann.riemann.Proto;
import io.riemann.riemann.client.IPromise;
import io.riemann.riemann.client.RiemannClient;
import org.apache.nifi.processor.exception.ProcessException;
import org.apache.nifi.util.MockFlowFile;
import org.apache.nifi.util.TestRunner;
import org.apache.nifi.util.TestRunners;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.mockito.invocation.InvocationOnMock;
import org.mockito.stubbing.Answer;
import java.util.HashMap;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.Queue;
import java.util.concurrent.TimeUnit;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertThrows;
import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.mockito.ArgumentMatchers.anyLong;
import static org.mockito.Mockito.any;
import static org.mockito.Mockito.anyList;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when;
public class TestPutRiemann {
// Holds incoming events to Riemann
private Queue<Proto.Event> eventStream = new LinkedList<Proto.Event>();
@BeforeEach
public void clearEventStream() {
eventStream.clear();
}
private TestRunner getTestRunner() {
return getTestRunner(false);
}
private TestRunner getTestRunner(final boolean failOnWrite) {
RiemannClient riemannClient = mock(RiemannClient.class);
when(riemannClient.sendEvents(anyList())).thenAnswer(new Answer() {
@Override
public Object answer(InvocationOnMock invocationOnMock) throws Throwable {
List<Proto.Event> events = (List<Proto.Event>) invocationOnMock.getArguments()[0];
for (Proto.Event event : events) {
eventStream.add(event);
}
IPromise iPromise = mock(IPromise.class);
if (!failOnWrite) {
when(iPromise.deref(anyLong(), any(TimeUnit.class))).thenReturn(Proto.Msg.getDefaultInstance());
} else {
when(iPromise.deref(anyLong(), any(TimeUnit.class))).thenReturn(null);
}
return iPromise;
}
});
when(riemannClient.isConnected()).thenReturn(true);
PutRiemann riemannProcessor = new PutRiemann();
riemannProcessor.riemannClient = riemannClient;
riemannProcessor.transport = PutRiemann.Transport.TCP;
TestRunner runner = TestRunners.newTestRunner(riemannProcessor);
runner.setProperty(PutRiemann.RIEMANN_HOST, "localhost");
runner.setProperty(PutRiemann.RIEMANN_PORT, "5555");
runner.setProperty(PutRiemann.TRANSPORT_PROTOCOL, "TCP");
runner.setProperty(PutRiemann.BATCH_SIZE, "100");
runner.setProperty(PutRiemann.ATTR_SERVICE, "nifi-test-service");
runner.setProperty(PutRiemann.ATTR_HOST, "${riemann.host}");
runner.setProperty(PutRiemann.ATTR_TTL, "5");
runner.setProperty(PutRiemann.ATTR_DESCRIPTION, "test");
runner.setProperty(PutRiemann.ATTR_TAGS, "tag1, tag2, tag3");
runner.setProperty(PutRiemann.ATTR_METRIC, "${riemann.metric}");
runner.setProperty("custom-attribute-1", "${custom.attribute.1}");
runner.setProperty("custom-attribute-2", "${custom.attribute.2}");
runner.setProperty("custom-attribute-3", "${custom.attribute.3}");
return runner;
}
@Test
public void testBasicEvent() {
TestRunner runner = getTestRunner();
Map<String, String> attributes = new HashMap<>();
attributes.put("riemann.metric", "42");
attributes.put("riemann.host", "basic-host");
MockFlowFile flowFile = new MockFlowFile(1);
flowFile.putAttributes(attributes);
runner.enqueue(flowFile);
runner.run();
runner.assertAllFlowFilesTransferred(PutRiemann.REL_SUCCESS);
Proto.Event event = eventStream.remove();
assertEquals("nifi-test-service", event.getService());
assertTrue(5.0 == event.getTtl());
assertTrue(42.0 == event.getMetricF());
assertEquals("basic-host", event.getHost());
assertEquals("test", event.getDescription());
assertEquals(3, event.getTagsCount());
assertTrue(event.getTagsList().contains("tag1"));
assertTrue(event.getTagsList().contains("tag2"));
assertTrue(event.getTagsList().contains("tag3"));
assertEquals(0, event.getAttributesCount());
}
@Test
public void testBatchedEvents() {
// (2 batches) + (1 remaining event)
int iterations = Integer.parseInt(PutRiemann.BATCH_SIZE.getDefaultValue()) * 2 + 1;
TestRunner runner = getTestRunner();
for (int i = 0; i < iterations; i++) {
Map<String, String> attributes = new HashMap<>();
attributes.put("riemann.metric", Float.toString(i));
attributes.put("riemann.host", "batch-host");
attributes.put("custom.attribute.1", "attr1");
attributes.put("custom.attribute.2", "attr2");
attributes.put("custom.attribute.3", "attr3");
MockFlowFile flowFile = new MockFlowFile(i);
flowFile.putAttributes(attributes);
runner.enqueue(flowFile);
}
runner.run(3);
runner.assertAllFlowFilesTransferred(PutRiemann.REL_SUCCESS);
for (int i = 0; i < iterations; i++) {
Proto.Event event = eventStream.remove();
assertEquals("nifi-test-service", event.getService());
assertTrue(5.0 == event.getTtl());
assertTrue(i == event.getMetricF());
assertEquals("batch-host", event.getHost());
assertEquals("test", event.getDescription());
assertEquals(3, event.getTagsCount());
assertEquals(3, event.getAttributesCount());
assertTrue(event.getTagsList().contains("tag1"));
assertTrue(event.getTagsList().contains("tag2"));
assertTrue(event.getTagsList().contains("tag3"));
}
}
@Test
public void testInvalidEvents() {
TestRunner runner = getTestRunner();
MockFlowFile flowFile = new MockFlowFile(1);
Map<String, String> attributes = new HashMap<>();
attributes.put("riemann.metric", "NOT A NUMBER");
flowFile.putAttributes(attributes);
runner.enqueue(flowFile);
runner.run();
runner.assertAllFlowFilesTransferred(PutRiemann.REL_FAILURE);
}
@Test
public void testFailedDeref() {
TestRunner runner = getTestRunner(true);
MockFlowFile flowFile = new MockFlowFile(1);
Map<String, String> attributes = new HashMap<>();
attributes.put("riemann.metric", "5");
flowFile.putAttributes(attributes);
runner.enqueue(flowFile);
try {
assertThrows(AssertionError.class, () -> runner.run());
} catch (ProcessException e) {
runner.assertQueueNotEmpty();
throw e;
}
}
}

54
nifi-nar-bundles/nifi-riemann-bundle/pom.xml
View File

@ -1,54 +0,0 @@
<?xml version="1.0" encoding="UTF-8"?>
<!--
Licensed to the Apache Software Foundation (ASF) under one or more
contributor license agreements. See the NOTICE file distributed with
this work for additional information regarding copyright ownership.
The ASF licenses this file to You under the Apache License, Version 2.0
(the "License"); you may not use this file except in compliance with
the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
-->
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<parent>
<artifactId>nifi-nar-bundles</artifactId>
<groupId>org.apache.nifi</groupId>
<version>2.0.0-SNAPSHOT</version>
</parent>
<artifactId>nifi-riemann-bundle</artifactId>
<packaging>pom</packaging>
<modules>
<module>nifi-riemann-processors</module>
<module>nifi-riemann-nar</module>
</modules>
<repositories>
<repository>
<id>clojars.org</id>
<url>https://clojars.org/repo</url>
</repository>
</repositories>
<dependencyManagement>
<dependencies>
<dependency>
<groupId>io.riemann</groupId>
<artifactId>riemann-java-client</artifactId>
<version>0.5.3</version>
</dependency>
<dependency>
<groupId>org.apache.nifi</groupId>
<artifactId>nifi-riemann-processors</artifactId>
<version>2.0.0-SNAPSHOT</version>
</dependency>
<dependency>
<groupId>com.google.protobuf</groupId>
<artifactId>protobuf-java</artifactId>
<version>3.23.0</version>
</dependency>
</dependencies>
</dependencyManagement>
</project>

43
nifi-nar-bundles/nifi-rules-action-handler-bundle/nifi-rules-action-handler-nar/pom.xml
View File

@ -1,43 +0,0 @@
<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/xsd/maven-4.0.0.xsd">
<!--
Licensed to the Apache Software Foundation (ASF) under one or more
contributor license agreements. See the NOTICE file distributed with
this work for additional information regarding copyright ownership.
The ASF licenses this file to You under the Apache License, Version 2.0
(the "License"); you may not use this file except in compliance with
the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
-->
<parent>
<artifactId>nifi-rules-action-handler-bundle</artifactId>
<groupId>org.apache.nifi</groupId>
<version>2.0.0-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
<artifactId>nifi-rules-action-handler-nar</artifactId>
<packaging>nar</packaging>
<properties>
<maven.javadoc.skip>true</maven.javadoc.skip>
<source.skip>true</source.skip>
</properties>
<dependencies>
<dependency>
<groupId>org.apache.nifi</groupId>
<artifactId>nifi-rules-action-handler-service</artifactId>
<version>2.0.0-SNAPSHOT</version>
</dependency>
<dependency>
<groupId>org.apache.nifi</groupId>
<artifactId>nifi-standard-services-api-nar</artifactId>
<version>2.0.0-SNAPSHOT</version>
<type>nar</type>
</dependency>
</dependencies>
</project>

236
nifi-nar-bundles/nifi-rules-action-handler-bundle/nifi-rules-action-handler-nar/src/main/resources/LICENSE
View File

@ -1,236 +0,0 @@
Apache License
Version 2.0, January 2004
http://www.apache.org/licenses/
TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
1. Definitions.
"License" shall mean the terms and conditions for use, reproduction,
and distribution as defined by Sections 1 through 9 of this document.
"Licensor" shall mean the copyright owner or entity authorized by
the copyright owner that is granting the License.
"Legal Entity" shall mean the union of the acting entity and all
other entities that control, are controlled by, or are under common
control with that entity. For the purposes of this definition,
"control" means (i) the power, direct or indirect, to cause the
direction or management of such entity, whether by contract or
otherwise, or (ii) ownership of fifty percent (50%) or more of the
outstanding shares, or (iii) beneficial ownership of such entity.
"You" (or "Your") shall mean an individual or Legal Entity
exercising permissions granted by this License.
"Source" form shall mean the preferred form for making modifications,
including but not limited to software source code, documentation
source, and configuration files.
"Object" form shall mean any form resulting from mechanical
transformation or translation of a Source form, including but
not limited to compiled object code, generated documentation,
and conversions to other media types.
"Work" shall mean the work of authorship, whether in Source or
Object form, made available under the License, as indicated by a
copyright notice that is included in or attached to the work
(an example is provided in the Appendix below).
"Derivative Works" shall mean any work, whether in Source or Object
form, that is based on (or derived from) the Work and for which the
editorial revisions, annotations, elaborations, or other modifications
represent, as a whole, an original work of authorship. For the purposes
of this License, Derivative Works shall not include works that remain
separable from, or merely link (or bind by name) to the interfaces of,
the Work and Derivative Works thereof.
"Contribution" shall mean any work of authorship, including
the original version of the Work and any modifications or additions
to that Work or Derivative Works thereof, that is intentionally
submitted to Licensor for inclusion in the Work by the copyright owner
or by an individual or Legal Entity authorized to submit on behalf of
the copyright owner. For the purposes of this definition, "submitted"
means any form of electronic, verbal, or written communication sent
to the Licensor or its representatives, including but not limited to
communication on electronic mailing lists, source code control systems,
and issue tracking systems that are managed by, or on behalf of, the
Licensor for the purpose of discussing and improving the Work, but
excluding communication that is conspicuously marked or otherwise
designated in writing by the copyright owner as "Not a Contribution."
"Contributor" shall mean Licensor and any individual or Legal Entity
on behalf of whom a Contribution has been received by Licensor and
subsequently incorporated within the Work.
2. Grant of Copyright License. Subject to the terms and conditions of
this License, each Contributor hereby grants to You a perpetual,
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
copyright license to reproduce, prepare Derivative Works of,
publicly display, publicly perform, sublicense, and distribute the
Work and such Derivative Works in Source or Object form.
3. Grant of Patent License. Subject to the terms and conditions of
this License, each Contributor hereby grants to You a perpetual,
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
(except as stated in this section) patent license to make, have made,
use, offer to sell, sell, import, and otherwise transfer the Work,
where such license applies only to those patent claims licensable
by such Contributor that are necessarily infringed by their
Contribution(s) alone or by combination of their Contribution(s)
with the Work to which such Contribution(s) was submitted. If You
institute patent litigation against any entity (including a
cross-claim or counterclaim in a lawsuit) alleging that the Work
or a Contribution incorporated within the Work constitutes direct
or contributory patent infringement, then any patent licenses
granted to You under this License for that Work shall terminate
as of the date such litigation is filed.
4. Redistribution. You may reproduce and distribute copies of the
Work or Derivative Works thereof in any medium, with or without
modifications, and in Source or Object form, provided that You
meet the following conditions:
(a) You must give any other recipients of the Work or
Derivative Works a copy of this License; and
(b) You must cause any modified files to carry prominent notices
stating that You changed the files; and
(c) You must retain, in the Source form of any Derivative Works
that You distribute, all copyright, patent, trademark, and
attribution notices from the Source form of the Work,
excluding those notices that do not pertain to any part of
the Derivative Works; and
(d) If the Work includes a "NOTICE" text file as part of its
distribution, then any Derivative Works that You distribute must
include a readable copy of the attribution notices contained
within such NOTICE file, excluding those notices that do not
pertain to any part of the Derivative Works, in at least one
of the following places: within a NOTICE text file distributed
as part of the Derivative Works; within the Source form or
documentation, if provided along with the Derivative Works; or,
within a display generated by the Derivative Works, if and
wherever such third-party notices normally appear. The contents
of the NOTICE file are for informational purposes only and
do not modify the License. You may add Your own attribution
notices within Derivative Works that You distribute, alongside
or as an addendum to the NOTICE text from the Work, provided
that such additional attribution notices cannot be construed
as modifying the License.
You may add Your own copyright statement to Your modifications and
may provide additional or different license terms and conditions
for use, reproduction, or distribution of Your modifications, or
for any such Derivative Works as a whole, provided Your use,
reproduction, and distribution of the Work otherwise complies with
the conditions stated in this License.
5. Submission of Contributions. Unless You explicitly state otherwise,
any Contribution intentionally submitted for inclusion in the Work
by You to the Licensor shall be under the terms and conditions of
this License, without any additional terms or conditions.
Notwithstanding the above, nothing herein shall supersede or modify
the terms of any separate license agreement you may have executed
with Licensor regarding such Contributions.
6. Trademarks. This License does not grant permission to use the trade
names, trademarks, service marks, or product names of the Licensor,
except as required for reasonable and customary use in describing the
origin of the Work and reproducing the content of the NOTICE file.
7. Disclaimer of Warranty. Unless required by applicable law or
agreed to in writing, Licensor provides the Work (and each
Contributor provides its Contributions) on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
implied, including, without limitation, any warranties or conditions
of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
PARTICULAR PURPOSE. You are solely responsible for determining the
appropriateness of using or redistributing the Work and assume any
risks associated with Your exercise of permissions under this License.
8. Limitation of Liability. In no event and under no legal theory,
whether in tort (including negligence), contract, or otherwise,
unless required by applicable law (such as deliberate and grossly
negligent acts) or agreed to in writing, shall any Contributor be
liable to You for damages, including any direct, indirect, special,
incidental, or consequential damages of any character arising as a
result of this License or out of the use or inability to use the
Work (including but not limited to damages for loss of goodwill,
work stoppage, computer failure or malfunction, or any and all
other commercial damages or losses), even if such Contributor
has been advised of the possibility of such damages.
9. Accepting Warranty or Additional Liability. While redistributing
the Work or Derivative Works thereof, You may choose to offer,
and charge a fee for, acceptance of support, warranty, indemnity,
or other liability obligations and/or rights consistent with this
License. However, in accepting such obligations, You may act only
on Your own behalf and on Your sole responsibility, not on behalf
of any other Contributor, and only if You agree to indemnify,
defend, and hold each Contributor harmless for any liability
incurred by, or claims asserted against, such Contributor by reason
of your accepting any such warranty or additional liability.
END OF TERMS AND CONDITIONS
APPENDIX: How to apply the Apache License to your work.
To apply the Apache License to your work, attach the following
boilerplate notice, with the fields enclosed by brackets "[]"
replaced with your own identifying information. (Don't include
the brackets!) The text should be enclosed in the appropriate
comment syntax for the file format. We also recommend that a
file or class name and description of purpose be included on the
same "printed page" as the copyright notice for easier
identification within third-party archives.
Copyright [yyyy] [name of copyright owner]
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
nifi-rules-action-handler-nar includes subcomponents with separate copyright notices and
license terms. Your use of these subcomponents is subject to the terms
and conditions of the following licenses:
The binary distribution of this product bundles the 'ASM' library which is available under a BSD style license.
Copyright (c) 2000-2005 INRIA, France Telecom
All rights reserved.
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions
are met:
1. Redistributions of source code must retain the above copyright
notice, this list of conditions and the following disclaimer.
2. Redistributions in binary form must reproduce the above copyright
notice, this list of conditions and the following disclaimer in the
documentation and/or other materials provided with the distribution.
3. Neither the name of the copyright holders nor the names of its
contributors may be used to endorse or promote products derived from
this software without specific prior written permission.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
THE POSSIBILITY OF SUCH DAMAGE.

18
nifi-nar-bundles/nifi-rules-action-handler-bundle/nifi-rules-action-handler-nar/src/main/resources/NOTICE
View File

@ -1,18 +0,0 @@
nifi-rules-action-handler-nar
Copyright 2014-2023 The Apache Software Foundation
This product includes software developed at
The Apache Software Foundation (http://www.apache.org/).
******************
Apache Software License v2
******************
The following binary components are provided under the Apache Software License v2
(ASLv2) MVEL (MVFLEX Expression Language)
(ASLv2) Spring Framework (Core, Expression)
The following NOTICE information applies:
Spring Framework
Copyright (c) 2002-2019 Pivotal, Inc.

75
nifi-nar-bundles/nifi-rules-action-handler-bundle/nifi-rules-action-handler-service/pom.xml
View File

@ -1,75 +0,0 @@
<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/xsd/maven-4.0.0.xsd">
<!--
Licensed to the Apache Software Foundation (ASF) under one or more
contributor license agreements. See the NOTICE file distributed with
this work for additional information regarding copyright ownership.
The ASF licenses this file to You under the Apache License, Version 2.0
(the "License"); you may not use this file except in compliance with
the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
-->
<parent>
<artifactId>nifi-rules-action-handler-bundle</artifactId>
<groupId>org.apache.nifi</groupId>
<version>2.0.0-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
<artifactId>nifi-rules-action-handler-service</artifactId>
<packaging>jar</packaging>
<dependencies>
<dependency>
<groupId>org.apache.nifi</groupId>
<artifactId>nifi-rules-engine-service-api</artifactId>
<version>2.0.0-SNAPSHOT</version>
</dependency>
<dependency>
<groupId>org.apache.nifi</groupId>
<artifactId>nifi-api</artifactId>
</dependency>
<dependency>
<groupId>org.apache.nifi</groupId>
<artifactId>nifi-record-sink-api</artifactId>
<version>2.0.0-SNAPSHOT</version>
</dependency>
<dependency>
<groupId>org.apache.nifi</groupId>
<artifactId>nifi-record</artifactId>
<version>2.0.0-SNAPSHOT</version>
</dependency>
<dependency>
<groupId>org.apache.nifi</groupId>
<artifactId>nifi-record-serialization-service-api</artifactId>
<version>2.0.0-SNAPSHOT</version>
</dependency>
<dependency>
<groupId>org.apache.commons</groupId>
<artifactId>commons-lang3</artifactId>
</dependency>
<dependency>
<groupId>org.apache.nifi</groupId>
<artifactId>nifi-utils</artifactId>
<version>2.0.0-SNAPSHOT</version>
</dependency>
<dependency>
<groupId>org.mvel</groupId>
<artifactId>mvel2</artifactId>
<version>2.5.0.Final</version>
</dependency>
<dependency>
<groupId>org.springframework</groupId>
<artifactId>spring-expression</artifactId>
</dependency>
<dependency>
<groupId>org.apache.nifi</groupId>
<artifactId>nifi-mock</artifactId>
<scope>test</scope>
</dependency>
</dependencies>
</project>

125
nifi-nar-bundles/nifi-rules-action-handler-bundle/nifi-rules-action-handler-service/src/main/java/org/apache/nifi/rules/handlers/AbstractActionHandlerService.java
View File

@ -1,125 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.nifi.rules.handlers;
import org.apache.commons.lang3.StringUtils;
import org.apache.nifi.annotation.lifecycle.OnEnabled;
import org.apache.nifi.components.PropertyDescriptor;
import org.apache.nifi.context.PropertyContext;
import org.apache.nifi.controller.AbstractControllerService;
import org.apache.nifi.controller.ConfigurationContext;
import org.apache.nifi.expression.ExpressionLanguageScope;
import org.apache.nifi.processor.util.StandardValidators;
import org.apache.nifi.reporting.InitializationException;
import org.apache.nifi.rules.Action;
import org.apache.nifi.rules.PropertyContextActionHandler;
import java.util.Arrays;
import java.util.List;
import java.util.Map;
import java.util.stream.Collectors;
public abstract class AbstractActionHandlerService extends AbstractControllerService implements PropertyContextActionHandler {
protected List<String> enforceActionTypes;
protected EnforceActionTypeLevel enforceActionTypeLevel;
public enum DebugLevels {
trace, debug, info, warn, error
}
public enum EnforceActionTypeLevel {
IGNORE, WARN, EXCEPTION
}
public static final PropertyDescriptor ENFORCE_ACTION_TYPE = new PropertyDescriptor.Builder()
.name("action-handler-enforce-type")
.displayName("Enforce Action Type")
.required(false)
.description("The Action Type(s) that should be supported by this handler. If provided any other type an " +
"exception will be thrown. This can support a comma delimited list of types (e.g. ALERT,LOG)")
.addValidator(StandardValidators.NON_BLANK_VALIDATOR)
.expressionLanguageSupported(ExpressionLanguageScope.VARIABLE_REGISTRY)
.build();
public static final PropertyDescriptor ENFORCE_ACTION_TYPE_LEVEL = new PropertyDescriptor.Builder()
.name("action-handler-enforce-type-level")
.displayName("Enforce Level")
.required(false)
.description("If specific action types are enforced, this setting specifies whether the action should be ignored," +
" a warning should be logged or if an exception is thrown. Default is to ignore the received action.")
.addValidator(StandardValidators.NON_BLANK_VALIDATOR)
.allowableValues(EnforceActionTypeLevel.values())
.defaultValue("IGNORE")
.build();
public void execute(Action action, Map<String, Object> facts) {
if (actionTypeNotSupported(action)) {
handleActionEnforcement(action);
} else {
executeAction(action, facts);
}
}
public void execute(PropertyContext context, Action action, Map<String, Object> facts) {
if (actionTypeNotSupported(action)) {
handleActionEnforcement(action);
} else {
executeAction(context, action, facts);
}
}
protected void executeAction(Action action, Map<String, Object> facts) {
throw new UnsupportedOperationException("This method is not supported by this handler.");
}
protected void executeAction(PropertyContext propertyContext, Action action, Map<String, Object> facts) {
throw new UnsupportedOperationException("This method is not supported by this handler");
}
protected boolean actionTypeNotSupported(Action action) {
return enforceActionTypes != null && !enforceActionTypes.contains(action.getType());
}
protected void handleActionEnforcement(Action action) {
String message = "This Action Handler does not support actions with the provided type: " + action.getType();
if (enforceActionTypeLevel.equals(EnforceActionTypeLevel.WARN)) {
getLogger().warn(message);
} else if (enforceActionTypeLevel.equals(EnforceActionTypeLevel.EXCEPTION)) {
throw new UnsupportedOperationException(message);
} else if (getLogger().isDebugEnabled()) {
getLogger().debug(message);
}
}
@OnEnabled
public void onEnabled(final ConfigurationContext context) throws InitializationException {
String actionTypes = context.getProperty(ENFORCE_ACTION_TYPE).evaluateAttributeExpressions().getValue();
if(StringUtils.isNotEmpty(actionTypes)){
enforceActionTypes = Arrays.stream(actionTypes.split(","))
.map(String::trim)
.filter(StringUtils::isNotEmpty)
.collect(Collectors.toList());
}
String level = context.getProperty(ENFORCE_ACTION_TYPE_LEVEL).getValue();
if(StringUtils.isNotEmpty(level)) {
enforceActionTypeLevel = EnforceActionTypeLevel.valueOf(level);
}
}
}

124
nifi-nar-bundles/nifi-rules-action-handler-bundle/nifi-rules-action-handler-service/src/main/java/org/apache/nifi/rules/handlers/ActionHandlerLookup.java
View File

@ -1,124 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.nifi.rules.handlers;
import org.apache.nifi.annotation.behavior.DynamicProperty;
import org.apache.nifi.annotation.documentation.CapabilityDescription;
import org.apache.nifi.annotation.documentation.Tags;
import org.apache.nifi.annotation.lifecycle.OnDisabled;
import org.apache.nifi.annotation.lifecycle.OnEnabled;
import org.apache.nifi.components.PropertyDescriptor;
import org.apache.nifi.components.ValidationContext;
import org.apache.nifi.components.ValidationResult;
import org.apache.nifi.context.PropertyContext;
import org.apache.nifi.controller.ConfigurationContext;
import org.apache.nifi.expression.ExpressionLanguageScope;
import org.apache.nifi.processor.exception.ProcessException;
import org.apache.nifi.processor.util.StandardValidators;
import org.apache.nifi.rules.Action;
import org.apache.nifi.rules.PropertyContextActionHandler;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
@Tags({"rules", "rules engine", "action", "action handler","lookup"})
@CapabilityDescription("Provides an Action Handler that can be used to dynamically select another Action Handler. " +
"This service will allow multiple ActionHandlers to be defined and registered by action type. When actions are provided the handlers can " +
"be dynamically determined and executed at runtime.")
@DynamicProperty(name = "actionType ", value = "Action Handler Service", expressionLanguageScope = ExpressionLanguageScope.NONE, description = "")
public class ActionHandlerLookup extends AbstractActionHandlerService{
private volatile Map<String, PropertyContextActionHandler> actionHandlerMap;
@Override
protected PropertyDescriptor getSupportedDynamicPropertyDescriptor(final String propertyDescriptorName) {
return new PropertyDescriptor.Builder()
.name(propertyDescriptorName)
.description("The Action handler to return when action type = '" + propertyDescriptorName + "'")
.identifiesControllerService(PropertyContextActionHandler.class)
.addValidator(StandardValidators.NON_BLANK_VALIDATOR)
.build();
}
@Override
protected Collection<ValidationResult> customValidate(ValidationContext context) {
final List<ValidationResult> results = new ArrayList<>();
int numDefinedServices = 0;
for (final PropertyDescriptor descriptor : context.getProperties().keySet()) {
if (descriptor.isDynamic()) {
numDefinedServices++;
}
final String referencedId = context.getProperty(descriptor).getValue();
if (this.getIdentifier().equals(referencedId)) {
results.add(new ValidationResult.Builder()
.subject(descriptor.getDisplayName())
.explanation("the current service cannot be registered as an ActionHandler to lookup")
.valid(false)
.build());
}
}
if (numDefinedServices == 0) {
results.add(new ValidationResult.Builder()
.subject(this.getClass().getSimpleName())
.explanation("at least one Action Handler must be defined via dynamic properties")
.valid(false)
.build());
}
return results;
}
@OnEnabled
public void onEnabled(final ConfigurationContext context) {
final Map<String,PropertyContextActionHandler> serviceMap = new HashMap<>();
for (final PropertyDescriptor descriptor : context.getProperties().keySet()) {
if (descriptor.isDynamic()) {
final PropertyContextActionHandler propertyContextActionHandler = context.getProperty(descriptor).asControllerService(PropertyContextActionHandler.class);
serviceMap.put(descriptor.getName(), propertyContextActionHandler);
}
}
actionHandlerMap = Collections.unmodifiableMap(serviceMap);
}
@OnDisabled
public void onDisabled() {
actionHandlerMap = null;
}
@Override
public void execute(Action action, Map<String, Object> facts) {
execute(null,action,facts);
}
@Override
public void execute(PropertyContext context, Action action, Map<String, Object> facts) {
PropertyContextActionHandler actionHandler = actionHandlerMap.get(action.getType());
if (actionHandler == null) {
throw new ProcessException("No Action Handler was found for Action Type:" + action.getType());
}
actionHandler.execute(context,action,facts);
}
}

166
nifi-nar-bundles/nifi-rules-action-handler-bundle/nifi-rules-action-handler-service/src/main/java/org/apache/nifi/rules/handlers/AlertHandler.java
View File

@ -1,166 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.nifi.rules.handlers;
import org.apache.nifi.annotation.documentation.CapabilityDescription;
import org.apache.nifi.annotation.documentation.Tags;
import org.apache.nifi.annotation.lifecycle.OnEnabled;
import org.apache.nifi.components.PropertyDescriptor;
import org.apache.nifi.context.PropertyContext;
import org.apache.nifi.controller.ConfigurationContext;
import org.apache.nifi.controller.ControllerServiceInitializationContext;
import org.apache.nifi.logging.ComponentLog;
import org.apache.nifi.processor.util.StandardValidators;
import org.apache.nifi.reporting.BulletinRepository;
import org.apache.nifi.reporting.InitializationException;
import org.apache.nifi.reporting.ReportingContext;
import org.apache.nifi.reporting.Severity;
import org.apache.nifi.rules.Action;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import java.util.Map;
import java.util.Set;
@Tags({"rules", "rules engine", "action", "action handler", "logging", "alerts", "bulletins"})
@CapabilityDescription("Creates alerts as bulletins based on a provided action (usually created by a rules engine). " +
"Action objects executed with this Handler should contain \"category\", \"message\", and \"logLevel\" attributes.")
public class AlertHandler extends AbstractActionHandlerService {
public static final PropertyDescriptor DEFAULT_LOG_LEVEL = new PropertyDescriptor.Builder()
.name("alert-default-log-level")
.displayName("Default Alert Log Level")
.required(true)
.description("The default Log Level that will be used to log an alert message" +
" if a log level was not provided in the received action's attributes.")
.allowableValues(DebugLevels.values())
.defaultValue("info")
.build();
public static final PropertyDescriptor DEFAULT_CATEGORY = new PropertyDescriptor.Builder()
.name("alert-default-category")
.displayName("Default Category")
.required(true)
.description("The default category to use when logging alert message "+
" if a category was not provided in the received action's attributes.")
.defaultValue("Rules Triggered Alert")
.addValidator(StandardValidators.NON_EMPTY_VALIDATOR)
.build();
public static final PropertyDescriptor DEFAULT_MESSAGE = new PropertyDescriptor.Builder()
.name("alert-default-message")
.displayName("Default Message")
.required(true)
.description("The default message to include in alert if an alert message was " +
"not provided in the received action's attributes")
.defaultValue("An alert was triggered by a rules-based action.")
.addValidator(StandardValidators.NON_EMPTY_VALIDATOR)
.build();
private static final PropertyDescriptor INCLUDE_FACTS = new PropertyDescriptor.Builder()
.name("alert-include-facts")
.displayName("Include Fact Data")
.required(true)
.description("If true, the alert message will include the facts which triggered this action. Default is false.")
.defaultValue("true")
.allowableValues("true", "false")
.build();
private List<PropertyDescriptor> properties;
private String defaultCategory;
private String defaultLogLevel;
private String defaultMessage;
private Boolean includeFacts;
@Override
protected void init(ControllerServiceInitializationContext config) throws InitializationException {
super.init(config);
final List<PropertyDescriptor> properties = new ArrayList<>();
properties.add(DEFAULT_LOG_LEVEL);
properties.add(DEFAULT_CATEGORY);
properties.add(DEFAULT_MESSAGE);
properties.add(INCLUDE_FACTS);
properties.add(ENFORCE_ACTION_TYPE);
properties.add(ENFORCE_ACTION_TYPE_LEVEL);
this.properties = Collections.unmodifiableList(properties);
}
@Override
@OnEnabled
public void onEnabled(final ConfigurationContext context) throws InitializationException {
super.onEnabled(context);
defaultLogLevel = context.getProperty(DEFAULT_LOG_LEVEL).getValue().toUpperCase();
defaultCategory = context.getProperty(DEFAULT_CATEGORY).getValue();
defaultMessage = context.getProperty(DEFAULT_MESSAGE).getValue();
includeFacts = context.getProperty(INCLUDE_FACTS).asBoolean();
}
@Override
protected List<PropertyDescriptor> getSupportedPropertyDescriptors() {
return properties;
}
@Override
protected void executeAction(PropertyContext propertyContext, Action action, Map<String, Object> facts) {
ComponentLog logger = getLogger();
if (propertyContext instanceof ReportingContext) {
ReportingContext context = (ReportingContext) propertyContext;
Map<String, String> attributes = action.getAttributes();
if (context.getBulletinRepository() != null) {
final String category = attributes.getOrDefault("category", defaultCategory);
final String message = getMessage(attributes.getOrDefault("message", defaultMessage), facts);
final String level = attributes.getOrDefault("severity", attributes.getOrDefault("logLevel", defaultLogLevel));
Severity severity;
try {
severity = Severity.valueOf(level.toUpperCase());
} catch (IllegalArgumentException iae) {
severity = Severity.INFO;
}
BulletinRepository bulletinRepository = context.getBulletinRepository();
bulletinRepository.addBulletin(context.createBulletin(category, severity, message));
} else {
logger.warn("Bulletin Repository is not available which is unusual. Cannot send a bulletin.");
}
} else {
logger.warn("Reporting context was not provided to create bulletins.");
}
}
protected String getMessage(String alertMessage, Map<String, Object> facts){
if (includeFacts) {
final StringBuilder message = new StringBuilder(alertMessage);
final Set<String> fields = facts.keySet();
message.append("\n");
message.append("Alert Facts:\n");
fields.forEach(field -> {
message.append("Field: ");
message.append(field);
message.append(", Value: ");
message.append(facts.get(field));
message.append("\n");
});
return message.toString();
}else{
return alertMessage;
}
}
}

131
nifi-nar-bundles/nifi-rules-action-handler-bundle/nifi-rules-action-handler-service/src/main/java/org/apache/nifi/rules/handlers/ExpressionHandler.java
View File

@ -1,131 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.nifi.rules.handlers;
import org.apache.commons.lang3.StringUtils;
import org.apache.nifi.annotation.documentation.CapabilityDescription;
import org.apache.nifi.annotation.documentation.Tags;
import org.apache.nifi.annotation.lifecycle.OnEnabled;
import org.apache.nifi.components.PropertyDescriptor;
import org.apache.nifi.context.PropertyContext;
import org.apache.nifi.controller.ConfigurationContext;
import org.apache.nifi.controller.ControllerServiceInitializationContext;
import org.apache.nifi.reporting.InitializationException;
import org.apache.nifi.rules.Action;
import org.mvel2.MVEL;
import org.springframework.expression.Expression;
import org.springframework.expression.ExpressionParser;
import org.springframework.expression.spel.standard.SpelExpressionParser;
import org.springframework.expression.spel.support.StandardEvaluationContext;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import java.util.Map;
@Tags({"rules", "rules engine", "action", "action handler", "expression language","MVEL","SpEL"})
@CapabilityDescription("Executes an action containing an expression written in MVEL or SpEL. The action " +
"is usually created by a rules engine. Action objects executed with this Handler should contain \"command\" and \"type\" attributes.")
public class ExpressionHandler extends AbstractActionHandlerService {
enum ExpresssionType {
MVEL, SPEL;
}
public static final PropertyDescriptor DEFAULT_EXPRESSION_LANGUAGE_TYPE = new PropertyDescriptor.Builder()
.name("default-expression-language-type")
.displayName("Default Expression Language Type")
.required(true)
.description("If an expression language type is not provided as an attribute within an Action, the default expression language that " +
"should be used to compile and execute action. Supported languages are MVEL and Spring Expression Language (SpEL).")
.allowableValues(ExpresssionType.values())
.defaultValue("MVEL")
.build();
private List<PropertyDescriptor> properties;
private ExpresssionType type;
@Override
protected void init(ControllerServiceInitializationContext config) throws InitializationException {
super.init(config);
final List<PropertyDescriptor> properties = new ArrayList<>();
properties.add(DEFAULT_EXPRESSION_LANGUAGE_TYPE);
properties.add(ENFORCE_ACTION_TYPE);
properties.add(ENFORCE_ACTION_TYPE_LEVEL);
this.properties = Collections.unmodifiableList(properties);
}
@Override
@OnEnabled
public void onEnabled(final ConfigurationContext context) throws InitializationException {
super.onEnabled(context);
type = ExpresssionType.valueOf(context.getProperty(DEFAULT_EXPRESSION_LANGUAGE_TYPE).getValue());
}
@Override
protected List<PropertyDescriptor> getSupportedPropertyDescriptors() {
return properties;
}
@Override
protected void executeAction(PropertyContext propertyContext, Action action, Map<String, Object> facts) {
executeAction(action, facts);
}
@Override
protected void executeAction(Action action, Map<String, Object> facts) {
Map<String, String> attributes = action.getAttributes();
final String command = attributes.get("command");
if(StringUtils.isNotEmpty(command)) {
try {
final String type = attributes.getOrDefault("type",this.type.toString());
ExpresssionType expresssionType = ExpresssionType.valueOf(type);
if (expresssionType.equals(ExpresssionType.MVEL)) {
executeMVEL(command, facts);
} else {
executeSPEL(command, facts);
}
} catch (Exception ex) {
getLogger().warn("Error occurred when attempting to execute expression. Action: {}, Facts - {}", action, facts, ex);
}
}else{
getLogger().warn("Command attribute was not provided. Action: {}, Facts - {}",
new Object[]{action, facts});
}
}
private void executeMVEL(String command, Map<String, Object> facts) {
MVEL.executeExpression(MVEL.compileExpression(command), facts);
if(getLogger().isDebugEnabled()) {
getLogger().debug("Expression was executed successfully: {}: {}", new Object[]{type, command});
}
}
private void executeSPEL(String command, Map<String, Object> facts) {
final ExpressionParser parser = new SpelExpressionParser();
StandardEvaluationContext context = new StandardEvaluationContext();
context.setRootObject(facts);
context.setVariables(facts);
Expression expression = parser.parseExpression(command);
Object value = expression.getValue(context);
if(getLogger().isDebugEnabled()) {
getLogger().debug("Expression was executed successfully with result: {}. {}: {}", new Object[]{value, type, command});
}
}
}

186
nifi-nar-bundles/nifi-rules-action-handler-bundle/nifi-rules-action-handler-service/src/main/java/org/apache/nifi/rules/handlers/LogHandler.java
View File

@ -1,186 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.nifi.rules.handlers;
import org.apache.commons.lang3.StringUtils;
import org.apache.nifi.annotation.documentation.CapabilityDescription;
import org.apache.nifi.annotation.documentation.Tags;
import org.apache.nifi.annotation.lifecycle.OnEnabled;
import org.apache.nifi.components.PropertyDescriptor;
import org.apache.nifi.context.PropertyContext;
import org.apache.nifi.controller.ConfigurationContext;
import org.apache.nifi.controller.ControllerServiceInitializationContext;
import org.apache.nifi.expression.ExpressionLanguageScope;
import org.apache.nifi.logging.ComponentLog;
import org.apache.nifi.logging.LogLevel;
import org.apache.nifi.processor.util.StandardValidators;
import org.apache.nifi.reporting.InitializationException;
import org.apache.nifi.rules.Action;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import java.util.Map;
import java.util.Set;
@Tags({"rules", "rules engine", "action", "action handler", "logging"})
@CapabilityDescription("Logs messages and fact information based on a provided action (usually created by a rules engine). " +
" Action objects executed with this Handler should contain \"logLevel\" and \"message\" attributes.")
public class LogHandler extends AbstractActionHandlerService {
public static final PropertyDescriptor DEFAULT_LOG_LEVEL = new PropertyDescriptor.Builder()
.name("logger-default-log-level")
.displayName("Default Log Level")
.required(true)
.description("If a log level is not provided as an attribute within an Action, the default log level will be used.")
.allowableValues(DebugLevels.values())
.defaultValue("info")
.build();
public static final PropertyDescriptor DEFAULT_LOG_MESSAGE = new PropertyDescriptor.Builder()
.name("logger-default-log-message")
.displayName("Default Log Message")
.required(true)
.description("If a log message is not provided as an attribute within an Action, the default log message will be used.")
.defaultValue("Rules Action Triggered Log.")
.addValidator(StandardValidators.NON_EMPTY_VALIDATOR)
.expressionLanguageSupported(ExpressionLanguageScope.VARIABLE_REGISTRY)
.build();
private static final PropertyDescriptor LOG_FACTS = new PropertyDescriptor.Builder()
.name("log-facts")
.displayName("Log Facts")
.required(true)
.description("If true, the log message will include the facts which triggered this log action.")
.defaultValue("true")
.allowableValues("true", "false")
.build();
private static final PropertyDescriptor LOG_PREFIX = new PropertyDescriptor.Builder()
.name("log-prefix")
.displayName("Log Prefix")
.required(false)
.description("Log prefix appended to the log lines. It helps to distinguish the output of multiple LogAttribute processors.")
.addValidator(StandardValidators.NON_EMPTY_VALIDATOR)
.expressionLanguageSupported(ExpressionLanguageScope.VARIABLE_REGISTRY)
.build();
private List<PropertyDescriptor> properties;
private String logPrefix;
private Boolean logFacts;
private String defaultLogLevel;
private String defaultLogMessage;
@Override
protected void init(ControllerServiceInitializationContext config) throws InitializationException {
super.init(config);
final List<PropertyDescriptor> properties = new ArrayList<>();
properties.add(LOG_PREFIX);
properties.add(LOG_FACTS);
properties.add(DEFAULT_LOG_LEVEL);
properties.add(DEFAULT_LOG_MESSAGE);
properties.add(ENFORCE_ACTION_TYPE);
properties.add(ENFORCE_ACTION_TYPE_LEVEL);
this.properties = Collections.unmodifiableList(properties);
}
@Override
@OnEnabled
public void onEnabled(final ConfigurationContext context) throws InitializationException {
super.onEnabled(context);
logPrefix = context.getProperty(LOG_PREFIX).evaluateAttributeExpressions().getValue();
logFacts = context.getProperty(LOG_FACTS).asBoolean();
defaultLogLevel = context.getProperty(DEFAULT_LOG_LEVEL).getValue().toUpperCase();
defaultLogMessage = context.getProperty(DEFAULT_LOG_MESSAGE).evaluateAttributeExpressions().getValue();
}
@Override
protected List<PropertyDescriptor> getSupportedPropertyDescriptors() {
return properties;
}
@Override
protected void executeAction(PropertyContext propertyContext, Action action, Map<String, Object> facts) {
executeAction(action, facts);
}
@Override
protected void executeAction(Action action, Map<String, Object> facts) {
ComponentLog logger = getLogger();
Map<String, String> attributes = action.getAttributes();
final String logLevel = attributes.get("logLevel");
final LogLevel level = getLogLevel(logLevel, LogLevel.valueOf(defaultLogLevel));
final String eventMessage = StringUtils.isNotEmpty(attributes.get("message")) ? attributes.get("message") : defaultLogMessage;
final String factsMessage = createFactsLogMessage(facts, eventMessage);
logger.log(level, factsMessage);
}
private LogLevel getLogLevel(String logLevel, LogLevel defaultLevel) {
LogLevel level;
if (StringUtils.isNotEmpty(logLevel)) {
try {
level = LogLevel.valueOf(logLevel.toUpperCase());
} catch (IllegalArgumentException iea) {
level = defaultLevel;
}
} else {
level = defaultLevel;
}
return level;
}
protected String createFactsLogMessage(Map<String, Object> facts, String eventMessage) {
final Set<String> fields = facts.keySet();
final StringBuilder message = new StringBuilder();
String dashedLine;
if (StringUtils.isBlank(logPrefix)) {
dashedLine = StringUtils.repeat('-', 50);
} else {
// abbreviate long lines
logPrefix = StringUtils.abbreviate(logPrefix, 40);
// center the logPrefix and pad with dashes
logPrefix = StringUtils.center(logPrefix, 40, '-');
// five dashes on the left and right side, plus the dashed logPrefix
dashedLine = StringUtils.repeat('-', 5) + logPrefix + StringUtils.repeat('-', 5);
}
message.append("\n");
message.append(dashedLine);
message.append("\n");
message.append("Log Message: ");
message.append(eventMessage);
message.append("\n");
if (logFacts) {
message.append("Log Facts:\n");
fields.forEach(field -> {
message.append("Field: ");
message.append(field);
message.append(", Value: ");
message.append(facts.get(field));
message.append("\n");
});
}
return message.toString().trim();
}
}

160
nifi-nar-bundles/nifi-rules-action-handler-bundle/nifi-rules-action-handler-service/src/main/java/org/apache/nifi/rules/handlers/RecordSinkHandler.java
View File

@ -1,160 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.nifi.rules.handlers;
import org.apache.commons.lang3.math.NumberUtils;
import org.apache.nifi.annotation.documentation.CapabilityDescription;
import org.apache.nifi.annotation.documentation.Tags;
import org.apache.nifi.annotation.lifecycle.OnEnabled;
import org.apache.nifi.components.PropertyDescriptor;
import org.apache.nifi.context.PropertyContext;
import org.apache.nifi.controller.ConfigurationContext;
import org.apache.nifi.controller.ControllerServiceInitializationContext;
import org.apache.nifi.record.sink.RecordSinkService;
import org.apache.nifi.reporting.InitializationException;
import org.apache.nifi.rules.Action;
import org.apache.nifi.serialization.SimpleRecordSchema;
import org.apache.nifi.serialization.WriteResult;
import org.apache.nifi.serialization.record.DataType;
import org.apache.nifi.serialization.record.ListRecordSet;
import org.apache.nifi.serialization.record.MapRecord;
import org.apache.nifi.serialization.record.RecordField;
import org.apache.nifi.serialization.record.RecordFieldType;
import org.apache.nifi.serialization.record.RecordSchema;
import org.apache.nifi.serialization.record.RecordSet;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.List;
import java.util.Map;
import java.util.stream.Collectors;
@Tags({"rules", "rules engine", "action", "action handler", "record", "record sink"})
@CapabilityDescription("Sends fact information to sink based on a provided action (usually created by a rules engine)." +
" Action objects executed with this Handler should contain \"sendZeroResult\" attribute.")
public class RecordSinkHandler extends AbstractActionHandlerService{
static final PropertyDescriptor RECORD_SINK_SERVICE = new PropertyDescriptor.Builder()
.name("record-sink-service")
.displayName("Record Sink Service")
.description("Specifies the Controller Service used to support the SEND event action. If not set SEND events will be ignored.")
.identifiesControllerService(RecordSinkService.class)
.required(true)
.build();
private RecordSinkService recordSinkService;
private List<PropertyDescriptor> properties;
@Override
protected void init(ControllerServiceInitializationContext config) throws InitializationException {
super.init(config);
final List<PropertyDescriptor> properties = new ArrayList<>();
properties.add(RECORD_SINK_SERVICE);
properties.add(ENFORCE_ACTION_TYPE);
properties.add(ENFORCE_ACTION_TYPE_LEVEL);
this.properties = Collections.unmodifiableList(properties);
}
@Override
protected List<PropertyDescriptor> getSupportedPropertyDescriptors() {
return properties;
}
@Override
@OnEnabled
public void onEnabled(final ConfigurationContext context) throws InitializationException {
super.onEnabled(context);
if(context.getProperty(RECORD_SINK_SERVICE).isSet()) {
recordSinkService = context.getProperty(RECORD_SINK_SERVICE).asControllerService(RecordSinkService.class);
}
}
@Override
protected void executeAction(PropertyContext propertyContext, Action action, Map<String, Object> facts) {
executeAction(action, facts);
}
@Override
protected void executeAction(Action action, Map<String, Object> facts) {
Map<String, String> attributes = action.getAttributes();
boolean sendZeroResults = attributes.containsKey("sentZeroResults") && Boolean.parseBoolean(attributes.get("sendZeroResults"));
final RecordSet recordSet = getRecordSet(facts);
try {
WriteResult result = recordSinkService.sendData(recordSet, attributes, sendZeroResults);
if (getLogger().isDebugEnabled() && result != null) {
getLogger().debug("Records written to sink service: {}", new Object[]{result.getRecordCount()});
}
} catch (Exception ex) {
getLogger().warn("Exception encountered when attempting to send metrics", ex);
}
}
private RecordSet getRecordSet(Map<String, Object> metrics){
List<RecordField> recordFields = metrics.entrySet().stream().map(entry ->
new RecordField(entry.getKey(),getDataType(String.valueOf(entry.getValue())))
).collect(Collectors.toList());
final RecordSchema recordSchema = new SimpleRecordSchema(recordFields);
return new ListRecordSet(recordSchema, Arrays.asList( new MapRecord(recordSchema,metrics)));
}
private DataType getDataType(final String value) {
if (value == null || value.isEmpty()) {
return null;
}
if (NumberUtils.isParsable(value)) {
if (value.contains(".")) {
try {
final double doubleValue = Double.parseDouble(value);
if (doubleValue == Double.POSITIVE_INFINITY || doubleValue == Double.NEGATIVE_INFINITY) {
return RecordFieldType.DECIMAL.getDecimalDataType(value.length() - 1, value.length() - 1 - value.indexOf("."));
}
if (doubleValue > Float.MAX_VALUE || doubleValue < Float.MIN_VALUE) {
return RecordFieldType.DOUBLE.getDataType();
}
return RecordFieldType.FLOAT.getDataType();
} catch (final NumberFormatException nfe) {
return RecordFieldType.STRING.getDataType();
}
}
try {
final long longValue = Long.parseLong(value);
if (longValue > Integer.MAX_VALUE || longValue < Integer.MIN_VALUE) {
return RecordFieldType.LONG.getDataType();
}
return RecordFieldType.INT.getDataType();
} catch (final NumberFormatException nfe) {
return RecordFieldType.STRING.getDataType();
}
}
if (value.equalsIgnoreCase("true") || value.equalsIgnoreCase("false")) {
return RecordFieldType.BOOLEAN.getDataType();
}
return RecordFieldType.STRING.getDataType();
}
}

19
nifi-nar-bundles/nifi-rules-action-handler-bundle/nifi-rules-action-handler-service/src/main/resources/META-INF/services/org.apache.nifi.controller.ControllerService
View File

@ -1,19 +0,0 @@
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
org.apache.nifi.rules.handlers.ActionHandlerLookup
org.apache.nifi.rules.handlers.ExpressionHandler
org.apache.nifi.rules.handlers.LogHandler
org.apache.nifi.rules.handlers.RecordSinkHandler
org.apache.nifi.rules.handlers.AlertHandler

39
nifi-nar-bundles/nifi-rules-action-handler-bundle/nifi-rules-action-handler-service/src/main/resources/docs/org.apache.nifi.rules.handlers.AlertHandler/additionalDetails.html
View File

@ -1,39 +0,0 @@
<!DOCTYPE html>
<html lang="en">
<!--
Licensed to the Apache Software Foundation (ASF) under one or more
contributor license agreements. See the NOTICE file distributed with
this work for additional information regarding copyright ownership.
The ASF licenses this file to You under the Apache License, Version 2.0
(the "License"); you may not use this file except in compliance with
the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
-->
<head>
<meta charset="utf-8" />
<title>AlertHandler</title>
<!--link rel="stylesheet" href="../../css/component-usage.css" type="text/css" /-->
<link rel="stylesheet" href="/nifi-docs/css/component-usage.css" type="text/css" />
</head>
<body>
<h2>Summary</h2>
<p>
The AlertHandler is used to broadcast alerts (bulletins) as dictated by the action object. Action objects can include attributes to configure
the handler otherwise default values will be used. Possible attribute values are listed below.
</p>
<h3>ExpressionHandler Service Attributes</h3>
<table title="AlertHandler Attributes" border="1" width="500">
<tr><th>Attribute</th><th>Description</th></tr>
<tr><td>category</td><td>The category the alert should be grouped under.</td></tr>
<tr><td>logLevel</td><td>Log Level for the alert. Possible values are trace, debug, info, warn, error.</td></tr>
<tr><td>message</td><td>Message for the alert.</td></tr>
</table>
<br/>
</body>
</html>

38
nifi-nar-bundles/nifi-rules-action-handler-bundle/nifi-rules-action-handler-service/src/main/resources/docs/org.apache.nifi.rules.handlers.ExpressionHandler/additionalDetails.html
View File

@ -1,38 +0,0 @@
<!DOCTYPE html>
<html lang="en">
<!--
Licensed to the Apache Software Foundation (ASF) under one or more
contributor license agreements. See the NOTICE file distributed with
this work for additional information regarding copyright ownership.
The ASF licenses this file to You under the Apache License, Version 2.0
(the "License"); you may not use this file except in compliance with
the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
-->
<head>
<meta charset="utf-8" />
<title>ExpressionHandler</title>
<!--link rel="stylesheet" href="../../css/component-usage.css" type="text/css" /-->
<link rel="stylesheet" href="/nifi-docs/css/component-usage.css" type="text/css" />
</head>
<body>
<h2>Summary</h2>
<p>
The ExpressionHandler is used to execute dynamic commands writtin in MVEL or SpEL expression language. Action objects must include attributes to configure
the handler otherwise an exception will be thrown. Possible attribute values are listed below.
</p>
<h3>ExpressionHandler Service Attributes</h3>
<table title="ExpressionHandler Attributes" border="1" width="500">
<tr><th>Attribute</th><th>Description</th></tr>
<tr><td>type</td><td>The expression language type of the command to be executed. Possible values are MVEL and SpEl (MVEL will be applied by default if type is not provided).</td></tr>
<tr><td>command</td><td>The expression language command that should be executed</td></tr>
</table>
<br/>
</body>
</html>

38
nifi-nar-bundles/nifi-rules-action-handler-bundle/nifi-rules-action-handler-service/src/main/resources/docs/org.apache.nifi.rules.handlers.LogHandler/additionalDetails.html
View File

@ -1,38 +0,0 @@
<!DOCTYPE html>
<html lang="en">
<!--
Licensed to the Apache Software Foundation (ASF) under one or more
contributor license agreements. See the NOTICE file distributed with
this work for additional information regarding copyright ownership.
The ASF licenses this file to You under the Apache License, Version 2.0
(the "License"); you may not use this file except in compliance with
the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
-->
<head>
<meta charset="utf-8" />
<title>LogHandler</title>
<!--link rel="stylesheet" href="../../css/component-usage.css" type="text/css" /-->
<link rel="stylesheet" href="/nifi-docs/css/component-usage.css" type="text/css" />
</head>
<body>
<h2>Summary</h2>
<p>
The LogHandler is used to execute actions that dictate to log a message and/or metrics. LogHandler can be invoked with any Action object.
Action objects can include attributes to configure the LogHandler or rely on the handler's default settings. Possible attribute values are listed below.
</p>
<h3>LogHandler Service Attributes</h3>
<table title="LogHandler Attributes" border="1" width="500">
<tr><th>Attribute</th><th>Description</th></tr>
<tr><td>logLevel</td><td>Log Level for logged message. Possible values are trace, debug, info, warn, error.</td></tr>
<tr><td>message</td><td>Message for log.</td></tr>
</table>
<br/>
</body>
</html>

37
nifi-nar-bundles/nifi-rules-action-handler-bundle/nifi-rules-action-handler-service/src/main/resources/docs/org.apache.nifi.rules.handlers.RecordSinkHandler/additionalDetails.html
View File

@ -1,37 +0,0 @@
<!DOCTYPE html>
<html lang="en">
<!--
Licensed to the Apache Software Foundation (ASF) under one or more
contributor license agreements. See the NOTICE file distributed with
this work for additional information regarding copyright ownership.
The ASF licenses this file to You under the Apache License, Version 2.0
(the "License"); you may not use this file except in compliance with
the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
-->
<head>
<meta charset="utf-8" />
<title>RecordSinkHandler</title>
<!--link rel="stylesheet" href="../../css/component-usage.css" type="text/css" /-->
<link rel="stylesheet" href="/nifi-docs/css/component-usage.css" type="text/css" />
</head>
<body>
<h2>Summary</h2>
<p>
The RecordSinkHandler is used to execute actions that send metrics information to a configured sink. RecordSinkHandler can be invoked with any Action object.
Action objects can include attributes to configure the handler. Possible attribute values are listed below.
</p>
<h3>RecordSinkHandler Service Attributes</h3>
<table title="RecordSinkHandler Attributes" border="1" width="500">
<tr><th>Attribute</th><th>Description</th></tr>
<tr><td>sendZeroResults</td><td>Allow empty results to be sent to sink. Possible values are true and false (default is false).</td></tr>
</table>
<br/>
</body>
</html>

200
nifi-nar-bundles/nifi-rules-action-handler-bundle/nifi-rules-action-handler-service/src/test/java/org/apache/nifi/rules/handlers/MockComponentLog.java
View File

@ -1,200 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.nifi.rules.handlers;
import org.apache.nifi.logging.ComponentLog;
import org.apache.nifi.logging.LogLevel;
import org.apache.nifi.logging.LogMessage;
public class MockComponentLog implements ComponentLog {
String infoMessage;
String warnMessage;
String debugMessage;
String traceMessage;
String errorMessage;
protected String convertMessage(String msg, Object[] os){
String replaceMsg = msg;
for (Object o : os) {
replaceMsg = replaceMsg.replaceFirst("\\{\\}", os.toString());
}
return replaceMsg;
}
@Override
public void warn(String msg, Throwable t) {
warn(msg);
}
@Override
public void warn(String msg, Object... os) {
warn(msg);
}
@Override
public void warn(String msg) {
warnMessage = msg;
}
@Override
public void warn(LogMessage logMessage) {
warnMessage = logMessage.getMessage();
}
@Override
public void trace(String msg, Throwable t) {
trace(msg);
}
@Override
public void trace(String msg, Object... os) {
trace(msg);
}
@Override
public void trace(String msg) {
traceMessage = msg;
}
@Override
public void trace(LogMessage logMessage) {
traceMessage = logMessage.getMessage();
}
@Override
public boolean isWarnEnabled() {
return true;
}
@Override
public boolean isTraceEnabled() {
return false;
}
@Override
public boolean isInfoEnabled() {
return false;
}
@Override
public boolean isErrorEnabled() {
return false;
}
@Override
public boolean isDebugEnabled() {
return true;
}
@Override
public void info(String msg, Throwable t) {
info(msg);
}
@Override
public void info(String msg, Object... os) {
info(msg);
}
@Override
public void info(String msg) {
infoMessage = msg;
}
@Override
public void info(LogMessage message) {
infoMessage = message.getMessage();
}
@Override
public String getName() {
return null;
}
@Override
public void error(String msg, Throwable t) {
error(msg);
}
@Override
public void error(String msg, Object... os) {
error(convertMessage(msg, os));
}
@Override
public void error(String msg) {
errorMessage = msg;
}
@Override
public void error(LogMessage message) {
errorMessage = message.getMessage();
}
@Override
public void debug(String msg, Throwable t) {
debug(msg);
}
@Override
public void debug(String msg, Object... os) {
debug(convertMessage(msg, os));
}
@Override
public void debug(String msg) {
debugMessage = msg;
}
@Override
public void debug(LogMessage message) {
debugMessage = message.getMessage();
}
@Override
public void log(LogLevel level, String msg, Throwable t) {
}
@Override
public void log(LogLevel level, String msg, Object... os) {
}
public String getInfoMessage() {
return infoMessage;
}
public String getWarnMessage() {
return warnMessage;
}
public String getDebugMessage() {
return debugMessage;
}
public String getTraceMessage() {
return traceMessage;
}
public String getErrorMessage() {
return errorMessage;
}
}

135
nifi-nar-bundles/nifi-rules-action-handler-bundle/nifi-rules-action-handler-service/src/test/java/org/apache/nifi/rules/handlers/TestActionHandlerLookup.java
View File

@ -1,135 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.nifi.rules.handlers;
import org.apache.nifi.components.PropertyDescriptor;
import org.apache.nifi.context.PropertyContext;
import org.apache.nifi.processor.exception.ProcessException;
import org.apache.nifi.reporting.InitializationException;
import org.apache.nifi.rules.Action;
import org.apache.nifi.util.TestRunner;
import org.apache.nifi.util.TestRunners;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import java.util.Arrays;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import static org.junit.jupiter.api.Assertions.assertThrows;
import static org.junit.jupiter.api.Assertions.assertTrue;
public class TestActionHandlerLookup {
private MockPropertyActionHandler alertHandler;
private MockPropertyActionHandler logHandler;
private ActionHandlerLookup actionHandlerLookup;
private TestRunner runner;
@BeforeEach
public void setup() throws InitializationException {
alertHandler = new MockPropertyActionHandler();
logHandler = new MockPropertyActionHandler();
actionHandlerLookup = new ActionHandlerLookup();
runner = TestRunners.newTestRunner(TestProcessor.class);
final String alertIdentifier = "alert-handler";
runner.addControllerService(alertIdentifier, alertHandler);
final String logIdentifier = "log-handler";
runner.addControllerService(logIdentifier, logHandler);
runner.addControllerService("action-handler-lookup", actionHandlerLookup);
runner.setProperty(actionHandlerLookup, "ALERT", alertIdentifier);
runner.setProperty(actionHandlerLookup, "LOG", logIdentifier);
runner.enableControllerService(alertHandler);
runner.enableControllerService(logHandler);
runner.enableControllerService(actionHandlerLookup);
}
@Test
public void testLookupAlert() {
final Map<String, String> attributes = new HashMap<>();
final Map<String, Object> metrics = new HashMap<>();
attributes.put("logLevel", "INFO");
attributes.put("message", "This should be not sent as an alert!");
metrics.put("jvmHeap", "1000000");
metrics.put("cpu", "90");
final Action action = new Action();
action.setType("ALERT");
action.setAttributes(attributes);
actionHandlerLookup.execute(null, action, metrics);
assertTrue(alertHandler.getExecuteContextCalled());
}
@Test
public void testLookupLog() {
final Map<String, String> attributes = new HashMap<>();
final Map<String, Object> metrics = new HashMap<>();
attributes.put("logLevel", "INFO");
attributes.put("message", "This should be not sent as an alert!");
metrics.put("jvmHeap", "1000000");
metrics.put("cpu", "90");
final Action action = new Action();
action.setType("LOG");
action.setAttributes(attributes);
actionHandlerLookup.execute(null, action, metrics);
assertTrue(logHandler.getExecuteContextCalled());
}
@Test
public void testLookupInvalidActionType() {
final Map<String, String> attributes = new HashMap<>();
final Map<String, Object> metrics = new HashMap<>();
attributes.put("logLevel", "INFO");
attributes.put("message", "This should be not sent as an alert!");
metrics.put("jvmHeap", "1000000");
metrics.put("cpu", "90");
final Action action = new Action();
action.setType("FAKE");
assertThrows(ProcessException.class, () -> actionHandlerLookup.execute(null,action,metrics));
}
private static class MockPropertyActionHandler extends AbstractActionHandlerService {
Boolean executeContextCalled = false;
@Override
public void execute(Action action, Map<String, Object> facts) {
execute(null,action,facts);
}
@Override
public void execute(PropertyContext context, Action action, Map<String, Object> facts) {
executeContextCalled = true;
}
public Boolean getExecuteContextCalled() {
return executeContextCalled;
}
@Override
protected List<PropertyDescriptor> getSupportedPropertyDescriptors() {
return Arrays.asList(ENFORCE_ACTION_TYPE, ENFORCE_ACTION_TYPE_LEVEL);
}
}
}

366
nifi-nar-bundles/nifi-rules-action-handler-bundle/nifi-rules-action-handler-service/src/test/java/org/apache/nifi/rules/handlers/TestAlertHandler.java
View File

@ -1,366 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.nifi.rules.handlers;
import org.apache.commons.lang3.StringUtils;
import org.apache.nifi.components.PropertyDescriptor;
import org.apache.nifi.components.PropertyValue;
import org.apache.nifi.context.PropertyContext;
import org.apache.nifi.logging.ComponentLog;
import org.apache.nifi.reporting.Bulletin;
import org.apache.nifi.reporting.BulletinFactory;
import org.apache.nifi.reporting.BulletinRepository;
import org.apache.nifi.reporting.InitializationException;
import org.apache.nifi.reporting.ReportingContext;
import org.apache.nifi.reporting.Severity;
import org.apache.nifi.rules.Action;
import org.apache.nifi.util.MockBulletinRepository;
import org.apache.nifi.util.TestRunner;
import org.apache.nifi.util.TestRunners;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.mockito.Mockito;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import static org.hamcrest.MatcherAssert.assertThat;
import static org.hamcrest.core.IsInstanceOf.instanceOf;
import static org.junit.jupiter.api.Assertions.assertDoesNotThrow;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertThrows;
import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.mockito.ArgumentMatchers.anyString;
public class TestAlertHandler {
private TestRunner runner;
private MockComponentLog mockComponentLog;
private ReportingContext reportingContext;
private AlertHandler alertHandler;
private MockAlertBulletinRepository mockAlertBulletinRepository;
@BeforeEach
public void setup() throws InitializationException {
runner = TestRunners.newTestRunner(TestProcessor.class);
mockComponentLog = new MockComponentLog();
AlertHandler handler = new MockAlertHandler(mockComponentLog);
mockAlertBulletinRepository = new MockAlertBulletinRepository();
runner.addControllerService("MockAlertHandler", handler);
runner.enableControllerService(handler);
alertHandler = (AlertHandler) runner.getProcessContext()
.getControllerServiceLookup()
.getControllerService("MockAlertHandler");
reportingContext = Mockito.mock(ReportingContext.class);
Mockito.when(reportingContext.getBulletinRepository()).thenReturn(mockAlertBulletinRepository);
Mockito.when(reportingContext.createBulletin(anyString(), Mockito.any(Severity.class), anyString()))
.thenAnswer(invocation ->
BulletinFactory.createBulletin(invocation.getArgument(0), invocation.getArgument(1).toString(), invocation.getArgument(2)));
}
@Test
public void testValidService() {
runner.assertValid(alertHandler);
assertThat(alertHandler, instanceOf(AlertHandler.class));
}
@Test
public void testAlertNoReportingContext() {
final Map<String, String> attributes = new HashMap<>();
final Map<String, Object> metrics = new HashMap<>();
attributes.put("logLevel", "INFO");
attributes.put("message", "This should be not sent as an alert!");
metrics.put("jvmHeap", "1000000");
metrics.put("cpu", "90");
final Action action = new Action();
action.setType("ALERT");
action.setAttributes(attributes);
assertThrows(UnsupportedOperationException.class, () -> alertHandler.execute(action, metrics));
}
@Test
public void testAlertWithBulletinLevel() {
final Map<String, String> attributes = new HashMap<>();
final Map<String, Object> metrics = new HashMap<>();
final String category = "Rules Alert";
final String message = "This should be sent as an alert!";
final String severity = "INFO";
attributes.put("category", category);
attributes.put("message", message);
attributes.put("severity", severity);
metrics.put("jvmHeap", "1000000");
metrics.put("cpu", "90");
final String expectedOutput = "This should be sent as an alert!\n" +
"Alert Facts:\n" +
"Field: cpu, Value: 90\n" +
"Field: jvmHeap, Value: 1000000\n";
final Action action = new Action();
action.setType("ALERT");
action.setAttributes(attributes);
alertHandler.execute(reportingContext, action, metrics);
BulletinRepository bulletinRepository = reportingContext.getBulletinRepository();
List<Bulletin> bulletins = bulletinRepository.findBulletinsForController();
assertFalse(bulletins.isEmpty());
Bulletin bulletin = bulletins.get(0);
assertEquals(bulletin.getCategory(), category);
assertEquals(bulletin.getMessage(), expectedOutput);
assertEquals(bulletin.getLevel(), severity);
}
@Test
public void testAlertWithDefaultValues() {
final Map<String, String> attributes = new HashMap<>();
final Map<String, Object> metrics = new HashMap<>();
final String category = "Rules Triggered Alert";
final String message = "An alert was triggered by a rules based action.";
final String severity = "INFO";
metrics.put("jvmHeap", "1000000");
metrics.put("cpu", "90");
final String expectedOutput = "An alert was triggered by a rules-based action.\n" +
"Alert Facts:\n" +
"Field: cpu, Value: 90\n" +
"Field: jvmHeap, Value: 1000000\n";
final Action action = new Action();
action.setType("ALERT");
action.setAttributes(attributes);
alertHandler.execute(reportingContext, action, metrics);
BulletinRepository bulletinRepository = reportingContext.getBulletinRepository();
List<Bulletin> bulletins = bulletinRepository.findBulletinsForController();
assertFalse(bulletins.isEmpty());
Bulletin bulletin = bulletins.get(0);
assertEquals(bulletin.getCategory(), category);
assertEquals(bulletin.getMessage(), expectedOutput);
assertEquals(bulletin.getLevel(), severity);
}
@Test
public void testInvalidContext(){
final Map<String, String> attributes = new HashMap<>();
final Map<String, Object> metrics = new HashMap<>();
final String category = "Rules Alert";
final String message = "This should be sent as an alert!";
final String severity = "INFO";
attributes.put("category", category);
attributes.put("message", message);
attributes.put("severity", severity);
metrics.put("jvmHeap", "1000000");
metrics.put("cpu", "90");
final Action action = new Action();
action.setType("ALERT");
action.setAttributes(attributes);
PropertyContext fakeContext = new PropertyContext() {
@Override
public PropertyValue getProperty(PropertyDescriptor descriptor) {
return null;
}
@Override
public Map<String, String> getAllProperties() {
return null;
}
};
alertHandler.execute(fakeContext, action, metrics);
final String debugMessage = mockComponentLog.getWarnMessage();
assertTrue(StringUtils.isNotEmpty(debugMessage));
assertEquals(debugMessage,"Reporting context was not provided to create bulletins.");
}
@Test
public void testEmptyBulletinRepository(){
final Map<String, String> attributes = new HashMap<>();
final Map<String, Object> metrics = new HashMap<>();
final String category = "Rules Alert";
final String message = "This should be sent as an alert!";
final String severity = "INFO";
attributes.put("category", category);
attributes.put("message", message);
attributes.put("severity", severity);
metrics.put("jvmHeap", "1000000");
metrics.put("cpu", "90");
final Action action = new Action();
action.setType("ALERT");
action.setAttributes(attributes);
ReportingContext fakeContext = Mockito.mock(ReportingContext.class);
Mockito.when(reportingContext.getBulletinRepository()).thenReturn(null);
alertHandler.execute(fakeContext, action, metrics);
final String warnMessage = mockComponentLog.getWarnMessage();
assertTrue(StringUtils.isNotEmpty(warnMessage));
assertEquals(warnMessage,"Bulletin Repository is not available which is unusual. Cannot send a bulletin.");
}
@Test
public void testInvalidActionTypeException(){
runner.disableControllerService(alertHandler);
runner.setProperty(alertHandler, AlertHandler.ENFORCE_ACTION_TYPE, "ALERT");
runner.setProperty(alertHandler, AlertHandler.ENFORCE_ACTION_TYPE_LEVEL, "EXCEPTION");
runner.enableControllerService(alertHandler);
final Map<String, String> attributes = new HashMap<>();
final Map<String, Object> metrics = new HashMap<>();
final String category = "Rules Alert";
final String message = "This should be sent as an alert!";
final String severity = "INFO";
attributes.put("category", category);
attributes.put("message", message);
attributes.put("severity", severity);
metrics.put("jvmHeap", "1000000");
metrics.put("cpu", "90");
final Action action = new Action();
action.setType("FAKE");
action.setAttributes(attributes);
assertThrows(UnsupportedOperationException.class, () -> alertHandler.execute(reportingContext, action, metrics));
}
@Test
public void testInvalidActionTypeWarn(){
runner.disableControllerService(alertHandler);
runner.setProperty(alertHandler, AlertHandler.ENFORCE_ACTION_TYPE, "ALERT");
runner.setProperty(alertHandler, AlertHandler.ENFORCE_ACTION_TYPE_LEVEL, "WARN");
runner.enableControllerService(alertHandler);
final Map<String, String> attributes = new HashMap<>();
final Map<String, Object> metrics = new HashMap<>();
final String category = "Rules Alert";
final String message = "This should be sent as an alert!";
final String severity = "INFO";
attributes.put("category", category);
attributes.put("message", message);
attributes.put("severity", severity);
metrics.put("jvmHeap", "1000000");
metrics.put("cpu", "90");
final Action action = new Action();
action.setType("FAKE");
action.setAttributes(attributes);
assertDoesNotThrow(() -> alertHandler.execute(reportingContext,action, metrics));
final String warnMessage = mockComponentLog.getWarnMessage();
assertTrue(StringUtils.isNotEmpty(warnMessage));
assertEquals("This Action Handler does not support actions with the provided type: FAKE",warnMessage);
}
@Test
public void testInvalidActionTypeIgnore(){
runner.disableControllerService(alertHandler);
runner.setProperty(alertHandler, AlertHandler.ENFORCE_ACTION_TYPE, "ALERT");
runner.setProperty(alertHandler, AlertHandler.ENFORCE_ACTION_TYPE_LEVEL, "IGNORE");
runner.enableControllerService(alertHandler);
final Map<String, String> attributes = new HashMap<>();
final Map<String, Object> metrics = new HashMap<>();
final String category = "Rules Alert";
final String message = "This should be sent as an alert!";
final String severity = "INFO";
attributes.put("category", category);
attributes.put("message", message);
attributes.put("severity", severity);
metrics.put("jvmHeap", "1000000");
metrics.put("cpu", "90");
final Action action = new Action();
action.setType("FAKE");
action.setAttributes(attributes);
assertDoesNotThrow(() -> alertHandler.execute(reportingContext,action, metrics));
final String debugMessage = mockComponentLog.getDebugMessage();
assertTrue(StringUtils.isNotEmpty(debugMessage));
assertEquals("This Action Handler does not support actions with the provided type: FAKE",debugMessage);
}
@Test
public void testValidActionType(){
runner.disableControllerService(alertHandler);
runner.setProperty(alertHandler, AlertHandler.ENFORCE_ACTION_TYPE, "ALERT, LOG, ");
runner.enableControllerService(alertHandler);
final Map<String, String> attributes = new HashMap<>();
final Map<String, Object> metrics = new HashMap<>();
final String category = "Rules Alert";
final String message = "This should be sent as an alert!";
final String severity = "INFO";
attributes.put("category", category);
attributes.put("message", message);
attributes.put("severity", severity);
metrics.put("jvmHeap", "1000000");
metrics.put("cpu", "90");
final Action action = new Action();
action.setType("ALERT");
action.setAttributes(attributes);
assertDoesNotThrow(() -> alertHandler.execute(reportingContext,action, metrics));
}
private static class MockAlertHandler extends AlertHandler {
private ComponentLog testLogger;
public MockAlertHandler(ComponentLog testLogger) {
this.testLogger = testLogger;
}
@Override
protected ComponentLog getLogger() {
return testLogger;
}
}
private static class MockAlertBulletinRepository extends MockBulletinRepository {
List<Bulletin> bulletinList;
public MockAlertBulletinRepository() {
bulletinList = new ArrayList<>();
}
@Override
public void addBulletin(Bulletin bulletin) {
bulletinList.add(bulletin);
}
@Override
public List<Bulletin> findBulletinsForController() {
return bulletinList;
}
}
}

236
nifi-nar-bundles/nifi-rules-action-handler-bundle/nifi-rules-action-handler-service/src/test/java/org/apache/nifi/rules/handlers/TestExpressionHandler.java
View File

@ -1,236 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.nifi.rules.handlers;
import org.apache.commons.lang3.StringUtils;
import org.apache.nifi.logging.ComponentLog;
import org.apache.nifi.reporting.InitializationException;
import org.apache.nifi.rules.Action;
import org.apache.nifi.util.TestRunner;
import org.apache.nifi.util.TestRunners;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import java.util.HashMap;
import java.util.Map;
import static org.hamcrest.MatcherAssert.assertThat;
import static org.hamcrest.core.IsInstanceOf.instanceOf;
import static org.junit.jupiter.api.Assertions.assertDoesNotThrow;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertThrows;
import static org.junit.jupiter.api.Assertions.assertTrue;
public class TestExpressionHandler {
private TestRunner runner;
private MockComponentLog mockComponentLog;
private ExpressionHandler expressionHandler;
@BeforeEach
public void setup() throws InitializationException {
runner = TestRunners.newTestRunner(TestProcessor.class);
mockComponentLog = new MockComponentLog();
ExpressionHandler handler = new MockExpressionHandler(mockComponentLog);
runner.addControllerService("MockExpressionHandler", handler);
runner.enableControllerService(handler);
expressionHandler = (ExpressionHandler) runner.getProcessContext()
.getControllerServiceLookup()
.getControllerService("MockExpressionHandler");
}
@Test
public void testValidService() {
runner.assertValid(expressionHandler);
assertThat(expressionHandler, instanceOf(ExpressionHandler.class));
}
@Test
public void testMvelExpression(){
final Map<String,String> attributes = new HashMap<>();
final Map<String,Object> metrics = new HashMap<>();
final String expectedMessage = "Expression was executed successfully:";
attributes.put("command","System.out.println(jvmHeap)");
attributes.put("type","MVEL");
metrics.put("jvmHeap","1000000");
metrics.put("cpu","90");
final Action action = new Action();
action.setType("EXPRESSION");
action.setAttributes(attributes);
expressionHandler.execute(action, metrics);
String logMessage = mockComponentLog.getDebugMessage();
assertTrue(StringUtils.isNotEmpty(logMessage));
assertTrue(logMessage.startsWith(expectedMessage));
}
@Test
public void testSpelExpression(){
final Map<String,String> attributes = new HashMap<>();
final Map<String,Object> metrics = new HashMap<>();
final String expectedMessage = "Expression was executed successfully with result:";
attributes.put("command","#jvmHeap + ' is large'");
attributes.put("type","SPEL");
metrics.put("jvmHeap","1000000");
metrics.put("cpu","90");
final Action action = new Action();
action.setType("EXPRESSION");
action.setAttributes(attributes);
expressionHandler.execute(action, metrics);
String logMessage = mockComponentLog.getDebugMessage();
assertTrue(StringUtils.isNotEmpty(logMessage));
assertTrue(logMessage.startsWith(expectedMessage));
}
@Test
public void testInvalidType() {
final Map<String,String> attributes = new HashMap<>();
final Map<String,Object> metrics = new HashMap<>();
final String expectedMessage = "Error occurred when attempting to execute expression.";
attributes.put("command","#jvmHeap + ' is large'");
attributes.put("type","FAKE");
metrics.put("jvmHeap","1000000");
metrics.put("cpu","90");
final Action action = new Action();
action.setType("EXPRESSION");
action.setAttributes(attributes);
expressionHandler.execute(action, metrics);
String logMessage = mockComponentLog.getWarnMessage();
assertTrue(StringUtils.isNotEmpty(logMessage));
assertTrue(logMessage.startsWith(expectedMessage));
}
@Test
public void testNoCommandProvided() {
final Map<String,String> attributes = new HashMap<>();
final Map<String,Object> metrics = new HashMap<>();
final String expectedMessage = "Command attribute was not provided.";
attributes.put("type","FAKE");
metrics.put("jvmHeap","1000000");
metrics.put("cpu","90");
final Action action = new Action();
action.setType("EXPRESSION");
action.setAttributes(attributes);
expressionHandler.execute(action, metrics);
String logMessage = mockComponentLog.getWarnMessage();
assertTrue(StringUtils.isNotEmpty(logMessage));
assertTrue(logMessage.startsWith(expectedMessage));
}
@Test
public void testInvalidActionTypeException() {
runner.disableControllerService(expressionHandler);
runner.setProperty(expressionHandler, AlertHandler.ENFORCE_ACTION_TYPE, "EXPRESSION");
runner.setProperty(expressionHandler, AlertHandler.ENFORCE_ACTION_TYPE_LEVEL, "EXCEPTION");
runner.enableControllerService(expressionHandler);
final Map<String,String> attributes = new HashMap<>();
final Map<String,Object> metrics = new HashMap<>();
attributes.put("type","FAKE");
metrics.put("jvmHeap","1000000");
metrics.put("cpu","90");
final Action action = new Action();
action.setType("FAKE");
action.setAttributes(attributes);
assertThrows(UnsupportedOperationException.class, () -> expressionHandler.execute(action, metrics));
}
@Test
public void testInvalidActionTypeWarning() {
runner.disableControllerService(expressionHandler);
runner.setProperty(expressionHandler, AlertHandler.ENFORCE_ACTION_TYPE, "EXPRESSION");
runner.setProperty(expressionHandler, AlertHandler.ENFORCE_ACTION_TYPE_LEVEL, "WARN");
runner.enableControllerService(expressionHandler);
final Map<String,String> attributes = new HashMap<>();
final Map<String,Object> metrics = new HashMap<>();
attributes.put("type","FAKE");
metrics.put("jvmHeap","1000000");
metrics.put("cpu","90");
final Action action = new Action();
action.setType("FAKE");
action.setAttributes(attributes);
assertDoesNotThrow(() -> expressionHandler.execute(action, metrics));
final String warnMessage = mockComponentLog.getWarnMessage();
assertTrue(StringUtils.isNotEmpty(warnMessage));
assertEquals("This Action Handler does not support actions with the provided type: FAKE",warnMessage);
}
@Test
public void testInvalidActionTypeIgnore() {
runner.disableControllerService(expressionHandler);
runner.setProperty(expressionHandler, AlertHandler.ENFORCE_ACTION_TYPE, "EXPRESSION");
runner.setProperty(expressionHandler, AlertHandler.ENFORCE_ACTION_TYPE_LEVEL, "IGNORE");
runner.enableControllerService(expressionHandler);
final Map<String,String> attributes = new HashMap<>();
final Map<String,Object> metrics = new HashMap<>();
attributes.put("type","FAKE");
metrics.put("jvmHeap","1000000");
metrics.put("cpu","90");
final Action action = new Action();
action.setType("FAKE");
action.setAttributes(attributes);
assertDoesNotThrow(() -> expressionHandler.execute(action, metrics));
final String debugMessage = mockComponentLog.getDebugMessage();
assertTrue(StringUtils.isNotEmpty(debugMessage));
assertEquals("This Action Handler does not support actions with the provided type: FAKE",debugMessage);
}
@Test
public void testValidActionType() {
runner.disableControllerService(expressionHandler);
runner.setProperty(expressionHandler, AlertHandler.ENFORCE_ACTION_TYPE, "EXPRESSION");
runner.enableControllerService(expressionHandler);
final Map<String,String> attributes = new HashMap<>();
final Map<String,Object> metrics = new HashMap<>();
attributes.put("type","FAKE");
metrics.put("jvmHeap","1000000");
metrics.put("cpu","90");
final Action action = new Action();
action.setType("EXPRESSION");
action.setAttributes(attributes);
assertDoesNotThrow(() -> expressionHandler.execute(action, metrics));
}
private static class MockExpressionHandler extends ExpressionHandler{
private ComponentLog testLogger;
public MockExpressionHandler(ComponentLog testLogger) {
this.testLogger = testLogger;
}
@Override
protected ComponentLog getLogger() {
return testLogger;
}
}
}

269
nifi-nar-bundles/nifi-rules-action-handler-bundle/nifi-rules-action-handler-service/src/test/java/org/apache/nifi/rules/handlers/TestLogHandler.java
View File

@ -1,269 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.nifi.rules.handlers;
import org.apache.commons.lang3.StringUtils;
import org.apache.nifi.logging.ComponentLog;
import org.apache.nifi.reporting.InitializationException;
import org.apache.nifi.rules.Action;
import org.apache.nifi.util.TestRunner;
import org.apache.nifi.util.TestRunners;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import java.util.HashMap;
import java.util.Map;
import static org.hamcrest.MatcherAssert.assertThat;
import static org.hamcrest.core.IsInstanceOf.instanceOf;
import static org.junit.jupiter.api.Assertions.assertDoesNotThrow;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertThrows;
import static org.junit.jupiter.api.Assertions.assertTrue;
public class TestLogHandler {
TestRunner runner;
MockComponentLog mockComponentLog;
LogHandler logHandler;
@BeforeEach
public void setup() throws InitializationException {
runner = TestRunners.newTestRunner(TestProcessor.class);
mockComponentLog = new MockComponentLog();
LogHandler handler = new MockLogHandler(mockComponentLog);
runner.addControllerService("MockLogHandler", handler);
runner.enableControllerService(handler);
logHandler = (LogHandler) runner.getProcessContext()
.getControllerServiceLookup()
.getControllerService("MockLogHandler");
}
@Test
public void testValidService() {
runner.assertValid(logHandler);
assertThat(logHandler, instanceOf(LogHandler.class));
}
@Test
public void testWarningLogged() {
final Map<String, String> attributes = new HashMap<>();
final Map<String, Object> metrics = new HashMap<>();
final String expectedMessage = "--------------------------------------------------\n" +
"Log Message: This is a warning\n" +
"Log Facts:\n" +
"Field: cpu, Value: 90\n" +
"Field: jvmHeap, Value: 1000000";
attributes.put("logLevel", "warn");
attributes.put("message", "This is a warning");
metrics.put("jvmHeap", "1000000");
metrics.put("cpu", "90");
final Action action = new Action();
action.setType("LOG");
action.setAttributes(attributes);
logHandler.execute(action, metrics);
String logMessage = mockComponentLog.getWarnMessage();
assertTrue(StringUtils.isNotEmpty(logMessage));
assertEquals(expectedMessage, logMessage);
}
@Test
public void testNoLogAttributesProvided() {
final Map<String, String> attributes = new HashMap<>();
final Map<String, Object> metrics = new HashMap<>();
final String expectedMessage = "--------------------------------------------------\n" +
"Log Message: Rules Action Triggered Log.\n" +
"Log Facts:\n" +
"Field: cpu, Value: 90\n" +
"Field: jvmHeap, Value: 1000000";
metrics.put("jvmHeap", "1000000");
metrics.put("cpu", "90");
final Action action = new Action();
action.setType("LOG");
action.setAttributes(attributes);
logHandler.execute(action, metrics);
String logMessage = mockComponentLog.getInfoMessage();
assertTrue(StringUtils.isNotEmpty(logMessage));
assertEquals(expectedMessage, logMessage);
}
@Test
public void testInvalidLogLevelProvided() {
final Map<String, String> attributes = new HashMap<>();
final Map<String, Object> metrics = new HashMap<>();
attributes.put("logLevel", "FAKE");
final String expectedMessage = "--------------------------------------------------\n" +
"Log Message: Rules Action Triggered Log.\n" +
"Log Facts:\n" +
"Field: cpu, Value: 90\n" +
"Field: jvmHeap, Value: 1000000";
metrics.put("jvmHeap", "1000000");
metrics.put("cpu", "90");
final Action action = new Action();
action.setType("LOG");
action.setAttributes(attributes);
logHandler.execute(action, metrics);
String logMessage = mockComponentLog.getInfoMessage();
assertTrue(StringUtils.isNotEmpty(logMessage));
assertEquals(expectedMessage, logMessage);
}
@Test
public void testInvalidActionTypeException() {
runner.disableControllerService(logHandler);
runner.setProperty(logHandler, AlertHandler.ENFORCE_ACTION_TYPE, "LOG");
runner.setProperty(logHandler, AlertHandler.ENFORCE_ACTION_TYPE_LEVEL, "EXCEPTION");
runner.enableControllerService(logHandler);
final Map<String, String> attributes = new HashMap<>();
final Map<String, Object> metrics = new HashMap<>();
attributes.put("logLevel", "FAKE");
final String expectedMessage = "--------------------------------------------------\n" +
"Log Message: Rules Action Triggered Log.\n" +
"Log Facts:\n" +
"Field: cpu, Value: 90\n" +
"Field: jvmHeap, Value: 1000000";
metrics.put("jvmHeap", "1000000");
metrics.put("cpu", "90");
final Action action = new Action();
action.setType("FAKE");
action.setAttributes(attributes);
assertThrows(UnsupportedOperationException.class, () -> logHandler.execute(action, metrics));
}
@Test
public void testInvalidActionTypeWarning() {
runner.disableControllerService(logHandler);
runner.setProperty(logHandler, AlertHandler.ENFORCE_ACTION_TYPE, "LOG");
runner.setProperty(logHandler, AlertHandler.ENFORCE_ACTION_TYPE_LEVEL, "WARN");
runner.enableControllerService(logHandler);
final Map<String, String> attributes = new HashMap<>();
final Map<String, Object> metrics = new HashMap<>();
attributes.put("logLevel", "FAKE");
final String expectedMessage = "--------------------------------------------------\n" +
"Log Message: Rules Action Triggered Log.\n" +
"Log Facts:\n" +
"Field: cpu, Value: 90\n" +
"Field: jvmHeap, Value: 1000000";
metrics.put("jvmHeap", "1000000");
metrics.put("cpu", "90");
final Action action = new Action();
action.setType("FAKE");
action.setAttributes(attributes);
assertDoesNotThrow(() -> logHandler.execute(action, metrics));
final String warnMessage = mockComponentLog.getWarnMessage();
assertTrue(StringUtils.isNotEmpty(warnMessage));
assertEquals("This Action Handler does not support actions with the provided type: FAKE",warnMessage);
}
@Test
public void testInvalidActionTypeDebug() {
runner.disableControllerService(logHandler);
runner.setProperty(logHandler, AlertHandler.ENFORCE_ACTION_TYPE, "LOG");
runner.setProperty(logHandler, AlertHandler.ENFORCE_ACTION_TYPE_LEVEL, "IGNORE");
runner.enableControllerService(logHandler);
final Map<String, String> attributes = new HashMap<>();
final Map<String, Object> metrics = new HashMap<>();
attributes.put("logLevel", "FAKE");
final String expectedMessage = "--------------------------------------------------\n" +
"Log Message: Rules Action Triggered Log.\n" +
"Log Facts:\n" +
"Field: cpu, Value: 90\n" +
"Field: jvmHeap, Value: 1000000";
metrics.put("jvmHeap", "1000000");
metrics.put("cpu", "90");
final Action action = new Action();
action.setType("FAKE");
action.setAttributes(attributes);
assertDoesNotThrow(() -> logHandler.execute(action, metrics));
final String debugMessage = mockComponentLog.getDebugMessage();
assertTrue(StringUtils.isNotEmpty(debugMessage));
assertEquals("This Action Handler does not support actions with the provided type: FAKE",debugMessage);
}
@Test
public void testValidActionType() {
runner.disableControllerService(logHandler);
runner.setProperty(logHandler, AlertHandler.ENFORCE_ACTION_TYPE, "LOG");
runner.enableControllerService(logHandler);
final Map<String, String> attributes = new HashMap<>();
final Map<String, Object> metrics = new HashMap<>();
attributes.put("logLevel", "FAKE");
final String expectedMessage = "--------------------------------------------------\n" +
"Log Message: Rules Action Triggered Log.\n" +
"Log Facts:\n" +
"Field: cpu, Value: 90\n" +
"Field: jvmHeap, Value: 1000000";
metrics.put("jvmHeap", "1000000");
metrics.put("cpu", "90");
final Action action = new Action();
action.setType("LOG");
action.setAttributes(attributes);
assertDoesNotThrow(() -> logHandler.execute(action, metrics));
}
private static class MockLogHandler extends LogHandler {
private ComponentLog testLogger;
public MockLogHandler(ComponentLog testLogger) {
this.testLogger = testLogger;
}
@Override
protected ComponentLog getLogger() {
return testLogger;
}
}
}

46
nifi-nar-bundles/nifi-rules-action-handler-bundle/nifi-rules-action-handler-service/src/test/java/org/apache/nifi/rules/handlers/TestProcessor.java
View File

@ -1,46 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.nifi.rules.handlers;
import org.apache.nifi.components.PropertyDescriptor;
import org.apache.nifi.processor.AbstractProcessor;
import org.apache.nifi.processor.ProcessContext;
import org.apache.nifi.processor.ProcessSession;
import org.apache.nifi.processor.exception.ProcessException;
import java.util.ArrayList;
import java.util.List;
public class TestProcessor extends AbstractProcessor {
@Override
public void onTrigger(ProcessContext context, ProcessSession session) throws ProcessException {
}
@Override
protected List<PropertyDescriptor> getSupportedPropertyDescriptors() {
List<PropertyDescriptor> properties = new ArrayList<>();
properties.add(new PropertyDescriptor.Builder()
.name("log-action-handler-test")
.description("Logging Action Handler")
.identifiesControllerService(LogHandler.class)
.required(true)
.build());
return properties;
}
}

152
nifi-nar-bundles/nifi-rules-action-handler-bundle/nifi-rules-action-handler-service/src/test/java/org/apache/nifi/rules/handlers/TestRecordSinkHandler.java
View File

@ -1,152 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.nifi.rules.handlers;
import org.apache.commons.lang3.StringUtils;
import org.apache.nifi.components.AbstractConfigurableComponent;
import org.apache.nifi.controller.ControllerServiceInitializationContext;
import org.apache.nifi.logging.ComponentLog;
import org.apache.nifi.record.sink.RecordSinkService;
import org.apache.nifi.reporting.InitializationException;
import org.apache.nifi.rules.Action;
import org.apache.nifi.serialization.WriteResult;
import org.apache.nifi.serialization.record.Record;
import org.apache.nifi.serialization.record.RecordSchema;
import org.apache.nifi.serialization.record.RecordSet;
import org.apache.nifi.util.TestRunner;
import org.apache.nifi.util.TestRunners;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import java.io.IOException;
import java.math.BigDecimal;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import static org.hamcrest.MatcherAssert.assertThat;
import static org.hamcrest.core.IsInstanceOf.instanceOf;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertTrue;
public class TestRecordSinkHandler {
private TestRunner runner;
private MockComponentLog mockComponentLog;
private RecordSinkHandler recordSinkHandler;
private MockRecordSinkService recordSinkService;
@BeforeEach
public void setup() throws InitializationException {
runner = TestRunners.newTestRunner(TestProcessor.class);
mockComponentLog = new MockComponentLog();
RecordSinkHandler handler = new MockRecordSinkHandler(mockComponentLog);
recordSinkService = new MockRecordSinkService();
runner.addControllerService("MockRecordSinkService", recordSinkService);
runner.enableControllerService(recordSinkService);
runner.addControllerService("MockRecordSinkHandler", handler);
runner.setProperty(handler, MockRecordSinkHandler.RECORD_SINK_SERVICE,"MockRecordSinkService");
runner.enableControllerService(handler);
recordSinkHandler = (RecordSinkHandler) runner.getProcessContext()
.getControllerServiceLookup()
.getControllerService("MockRecordSinkHandler");
}
@Test
public void testValidService() {
runner.assertValid(recordSinkHandler);
assertThat(recordSinkHandler, instanceOf(RecordSinkHandler.class));
}
@Test
public void testRecordSendViaSink() throws InitializationException, IOException {
final Map<String,String> attributes = new HashMap<>();
final Map<String,Object> metrics = new HashMap<>();
final String expectedMessage = "Records written to sink service:";
final BigDecimal bigDecimalValue = new BigDecimal(String.join("", Collections.nCopies(400, "1")) + ".2");
attributes.put("sendZeroResults","false");
metrics.put("jvmHeap","1000000");
metrics.put("cpu","90");
metrics.put("custom", bigDecimalValue);
final Action action = new Action();
action.setType("SEND");
action.setAttributes(attributes);
recordSinkHandler.execute(action, metrics);
String logMessage = mockComponentLog.getDebugMessage();
List<Map<String, Object>> rows = recordSinkService.getRows();
assertTrue(StringUtils.isNotEmpty(logMessage));
assertTrue(logMessage.startsWith(expectedMessage));
assertFalse(rows.isEmpty());
Map<String,Object> record = rows.get(0);
assertEquals("90", (record.get("cpu")));
assertEquals("1000000", (record.get("jvmHeap")));
assertEquals(bigDecimalValue, (record.get("custom")));
}
private static class MockRecordSinkHandler extends RecordSinkHandler {
private ComponentLog testLogger;
public MockRecordSinkHandler(ComponentLog testLogger) {
this.testLogger = testLogger;
}
@Override
protected ComponentLog getLogger() {
return testLogger;
}
}
private static class MockRecordSinkService extends AbstractConfigurableComponent implements RecordSinkService {
private List<Map<String, Object>> rows = new ArrayList<>();
@Override
public WriteResult sendData(RecordSet recordSet, Map<String,String> attributes, boolean sendZeroResults) throws IOException {
int numRecordsWritten = 0;
RecordSchema recordSchema = recordSet.getSchema();
Record record;
while ((record = recordSet.next()) != null) {
Map<String, Object> row = new HashMap<>();
final Record finalRecord = record;
recordSchema.getFieldNames().forEach((fieldName) -> row.put(fieldName, finalRecord.getValue(fieldName)));
rows.add(row);
numRecordsWritten++;
}
return WriteResult.of(numRecordsWritten, Collections.emptyMap());
}
@Override
public String getIdentifier() {
return "MockRecordSinkService";
}
@Override
public void initialize(ControllerServiceInitializationContext context) throws InitializationException {
}
public List<Map<String, Object>> getRows() {
return rows;
}
}
}

40
nifi-nar-bundles/nifi-rules-action-handler-bundle/pom.xml
View File

@ -1,40 +0,0 @@
<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/xsd/maven-4.0.0.xsd">
<!--
Licensed to the Apache Software Foundation (ASF) under one or more
contributor license agreements. See the NOTICE file distributed with
this work for additional information regarding copyright ownership.
The ASF licenses this file to You under the Apache License, Version 2.0
(the "License"); you may not use this file except in compliance with
the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
-->
<parent>
<artifactId>nifi-nar-bundles</artifactId>
<groupId>org.apache.nifi</groupId>
<version>2.0.0-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
<artifactId>nifi-rules-action-handler-bundle</artifactId>
<packaging>pom</packaging>
<modules>
<module>nifi-rules-action-handler-nar</module>
<module>nifi-rules-action-handler-service</module>
</modules>
<dependencyManagement>
<dependencies>
<dependency>
<groupId>org.apache.nifi</groupId>
<artifactId>nifi-rules-action-handler-service</artifactId>
<version>2.0.0-SNAPSHOT</version>
</dependency>
</dependencies>
</dependencyManagement>
</project>

5
nifi-nar-bundles/nifi-scripting-bundle/nifi-scripting-processors/pom.xml
View File

@ -69,11 +69,6 @@
<artifactId>nifi-record-sink-api</artifactId>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>org.apache.nifi</groupId>
<artifactId>nifi-rules-engine-service-api</artifactId>
<version>2.0.0-SNAPSHOT</version>
</dependency>
<dependency>
<groupId>org.apache.ivy</groupId>
<artifactId>ivy</artifactId>

224
nifi-nar-bundles/nifi-scripting-bundle/nifi-scripting-processors/src/main/java/org/apache/nifi/rules/engine/script/ScriptedRulesEngine.java
View File

@ -1,224 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.nifi.rules.engine.script;
import org.apache.nifi.annotation.behavior.DynamicProperty;
import org.apache.nifi.annotation.behavior.Restricted;
import org.apache.nifi.annotation.behavior.Restriction;
import org.apache.nifi.annotation.documentation.CapabilityDescription;
import org.apache.nifi.annotation.documentation.Tags;
import org.apache.nifi.annotation.lifecycle.OnEnabled;
import org.apache.nifi.components.PropertyDescriptor;
import org.apache.nifi.components.RequiredPermission;
import org.apache.nifi.components.ValidationResult;
import org.apache.nifi.controller.ConfigurationContext;
import org.apache.nifi.expression.ExpressionLanguageScope;
import org.apache.nifi.logging.ComponentLog;
import org.apache.nifi.processor.exception.ProcessException;
import org.apache.nifi.rules.Action;
import org.apache.nifi.rules.engine.RulesEngineService;
import org.apache.nifi.script.AbstractScriptedControllerService;
import org.apache.nifi.script.ScriptingComponentHelper;
import javax.script.Invocable;
import javax.script.ScriptContext;
import javax.script.ScriptEngine;
import javax.script.ScriptException;
import java.util.Collection;
import java.util.Collections;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.concurrent.atomic.AtomicReference;
@Tags({"rules", "rules engine", "script", "invoke", "groovy", "python", "jython"})
@CapabilityDescription("Allows the user to provide a scripted RulesEngineService for custom firing of rules depending on the supplied facts. The script must set a variable 'rulesEngine' to an "
+ "implementation of RulesEngineService.")
@DynamicProperty(name = "Script Engine Binding property", value = "Binding property value passed to Script Runner",
expressionLanguageScope = ExpressionLanguageScope.VARIABLE_REGISTRY,
description = "Updates a script engine property specified by the Dynamic Property's key with the value specified by the Dynamic Property's value")
@Restricted(
restrictions = {
@Restriction(
requiredPermission = RequiredPermission.EXECUTE_CODE,
explanation = "Provides operator the ability to execute arbitrary code assuming all permissions that NiFi has.")
}
)
public class ScriptedRulesEngine extends AbstractScriptedControllerService implements RulesEngineService {
protected final AtomicReference<RulesEngineService> rulesEngine = new AtomicReference<>();
/**
* Returns a list of property descriptors supported by this processor. The list always includes properties such as
* script engine name, script file name, script body name, script arguments, and an external module path. If the
* scripted processor also defines supported properties, those are added to the list as well.
*
* @return a List of PropertyDescriptor objects supported by this processor
*/
@Override
protected List<PropertyDescriptor> getSupportedPropertyDescriptors() {
synchronized (scriptingComponentHelper.isInitialized) {
if (!scriptingComponentHelper.isInitialized.get()) {
scriptingComponentHelper.createResources();
}
}
return Collections.unmodifiableList(scriptingComponentHelper.getDescriptors());
}
public void setup() {
if (scriptNeedsReload.get() || rulesEngine.get() == null) {
if (ScriptingComponentHelper.isFile(scriptingComponentHelper.getScriptPath())) {
scriptNeedsReload.set(!reloadScriptFile(scriptingComponentHelper.getScriptPath()));
} else {
scriptNeedsReload.set(!reloadScriptBody(scriptingComponentHelper.getScriptBody()));
}
}
}
/**
* Reloads the script RulesEngineService. This must be called within the lock.
*
* @param scriptBody An input stream associated with the script content
* @return Whether the script was successfully reloaded
*/
protected boolean reloadScript(final String scriptBody) {
// note we are starting here with a fresh listing of validation
// results since we are (re)loading a new/updated script. any
// existing validation results are not relevant
final Collection<ValidationResult> results = new HashSet<>();
try {
// Create a single script engine, the Processor object is reused by each task
if (scriptRunner == null) {
scriptingComponentHelper.setupScriptRunners(1, scriptBody, getLogger());
scriptRunner = scriptingComponentHelper.scriptRunnerQ.poll();
}
if (scriptRunner == null) {
throw new ProcessException("No script runner available!");
}
// get the engine and ensure its invocable
ScriptEngine scriptEngine = scriptRunner.getScriptEngine();
if (scriptEngine instanceof Invocable) {
final Invocable invocable = (Invocable) scriptEngine;
// evaluate the script
scriptRunner.run(scriptEngine.getBindings(ScriptContext.ENGINE_SCOPE));
// get configured processor from the script (if it exists)
final Object obj = scriptEngine.get("rulesEngine");
if (obj != null) {
final ComponentLog logger = getLogger();
try {
// set the logger if the processor wants it
invocable.invokeMethod(obj, "setLogger", logger);
} catch (final NoSuchMethodException nsme) {
if (logger.isDebugEnabled()) {
logger.debug("Configured script RulesEngineService does not contain a setLogger method.");
}
}
if (configurationContext != null) {
try {
// set the logger if the processor wants it
invocable.invokeMethod(obj, "setConfigurationContext", configurationContext);
} catch (final NoSuchMethodException nsme) {
if (logger.isDebugEnabled()) {
logger.debug("Configured script RulesEngineService does not contain a setConfigurationContext method.");
}
}
}
// record the processor for use later
final RulesEngineService scriptedReader = invocable.getInterface(obj, RulesEngineService.class);
rulesEngine.set(scriptedReader);
} else {
throw new ScriptException("No RecordReader was defined by the script.");
}
}
} catch (final Exception ex) {
final ComponentLog logger = getLogger();
final String message = "Unable to load script: " + ex.getLocalizedMessage();
logger.error(message, ex);
results.add(new ValidationResult.Builder()
.subject("ScriptValidation")
.valid(false)
.explanation("Unable to load script due to " + ex.getLocalizedMessage())
.input(scriptingComponentHelper.getScriptPath())
.build());
}
// store the updated validation results
validationResults.set(results);
// return whether there was any issues loading the configured script
return results.isEmpty();
}
@Override
@OnEnabled
public void onEnabled(final ConfigurationContext context) {
synchronized (scriptingComponentHelper.isInitialized) {
if (!scriptingComponentHelper.isInitialized.get()) {
scriptingComponentHelper.createResources();
}
}
super.onEnabled(context);
// Call an non-interface method onEnabled(context), to allow a scripted RulesEngineService the chance to set up as necessary
if (scriptRunner != null) {
final ScriptEngine scriptEngine = scriptRunner.getScriptEngine();
final Invocable invocable = (Invocable) scriptEngine;
if (configurationContext != null) {
try {
// Get the actual object from the script engine, versus the proxy stored in RulesEngineService. The object may have additional methods,
// where RulesEngineService is a proxied interface
final Object obj = scriptEngine.get("rulesEngine");
if (obj != null) {
try {
invocable.invokeMethod(obj, "onEnabled", context);
} catch (final NoSuchMethodException nsme) {
if (getLogger().isDebugEnabled()) {
getLogger().debug("Configured script RulesEngineService does not contain an onEnabled() method.");
}
}
} else {
throw new ScriptException("No RulesEngineService was defined by the script.");
}
} catch (ScriptException se) {
throw new ProcessException("Error executing onEnabled(context) method: " + se.getMessage(), se);
}
}
} else {
throw new ProcessException("Error creating ScriptRunner");
}
}
@Override
public List<Action> fireRules(Map<String, Object> facts) {
if (rulesEngine.get() != null) {
return rulesEngine.get().fireRules(facts);
}
return null;
}
}

257
nifi-nar-bundles/nifi-scripting-bundle/nifi-scripting-processors/src/main/java/org/apache/nifi/rules/handlers/script/ScriptedActionHandler.java
View File

@ -1,257 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.nifi.rules.handlers.script;
import org.apache.nifi.annotation.behavior.DynamicProperty;
import org.apache.nifi.annotation.behavior.Restricted;
import org.apache.nifi.annotation.behavior.Restriction;
import org.apache.nifi.annotation.documentation.CapabilityDescription;
import org.apache.nifi.annotation.documentation.Tags;
import org.apache.nifi.annotation.lifecycle.OnEnabled;
import org.apache.nifi.components.PropertyDescriptor;
import org.apache.nifi.components.RequiredPermission;
import org.apache.nifi.components.ValidationResult;
import org.apache.nifi.context.PropertyContext;
import org.apache.nifi.controller.ConfigurationContext;
import org.apache.nifi.expression.ExpressionLanguageScope;
import org.apache.nifi.logging.ComponentLog;
import org.apache.nifi.processor.exception.ProcessException;
import org.apache.nifi.rules.Action;
import org.apache.nifi.rules.ActionHandler;
import org.apache.nifi.rules.PropertyContextActionHandler;
import org.apache.nifi.script.AbstractScriptedControllerService;
import org.apache.nifi.script.ScriptingComponentHelper;
import javax.script.Invocable;
import javax.script.ScriptContext;
import javax.script.ScriptEngine;
import javax.script.ScriptException;
import java.util.Collection;
import java.util.Collections;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.concurrent.atomic.AtomicReference;
@Tags({"rules", "rules engine", "action", "action handler", "script", "invoke", "groovy", "python", "jython"})
@CapabilityDescription("Allows the user to provide a scripted ActionHandler for custom firing of rules depending on the supplied facts. The script must set a variable 'actionHandler' to an "
+ "implementation of ActionHandler.")
@DynamicProperty(name = "Script Engine Binding property", value = "Binding property value passed to Script Runner",
expressionLanguageScope = ExpressionLanguageScope.VARIABLE_REGISTRY,
description = "Updates a script engine property specified by the Dynamic Property's key with the value specified by the Dynamic Property's value")
@Restricted(
restrictions = {
@Restriction(
requiredPermission = RequiredPermission.EXECUTE_CODE,
explanation = "Provides operator the ability to execute arbitrary code assuming all permissions that NiFi has.")
}
)
public class ScriptedActionHandler extends AbstractScriptedControllerService implements PropertyContextActionHandler {
protected final AtomicReference<ActionHandler> actionHandler = new AtomicReference<>();
/**
* Returns a list of property descriptors supported by this processor. The list always includes properties such as
* script engine name, script file name, script body name, script arguments, and an external module path. If the
* scripted processor also defines supported properties, those are added to the list as well.
*
* @return a List of PropertyDescriptor objects supported by this processor
*/
@Override
protected List<PropertyDescriptor> getSupportedPropertyDescriptors() {
synchronized (scriptingComponentHelper.isInitialized) {
if (!scriptingComponentHelper.isInitialized.get()) {
scriptingComponentHelper.createResources();
}
}
return Collections.unmodifiableList(scriptingComponentHelper.getDescriptors());
}
public void setup() {
if (scriptNeedsReload.get() || actionHandler.get() == null) {
if (ScriptingComponentHelper.isFile(scriptingComponentHelper.getScriptPath())) {
scriptNeedsReload.set(!reloadScriptFile(scriptingComponentHelper.getScriptPath()));
} else {
scriptNeedsReload.set(!reloadScriptBody(scriptingComponentHelper.getScriptBody()));
}
}
}
/**
* Reloads the script ActionHandler. This must be called within the lock.
*
* @param scriptBody An input stream associated with the script content
* @return Whether the script was successfully reloaded
*/
protected boolean reloadScript(final String scriptBody) {
// note we are starting here with a fresh listing of validation
// results since we are (re)loading a new/updated script. any
// existing validation results are not relevant
final Collection<ValidationResult> results = new HashSet<>();
try {
// Create a single script engine, the Processor object is reused by each task
if (scriptRunner == null) {
scriptingComponentHelper.setupScriptRunners(1, scriptBody, getLogger());
scriptRunner = scriptingComponentHelper.scriptRunnerQ.poll();
}
if (scriptRunner == null) {
throw new ProcessException("No script runner available!");
}
// get the engine and ensure its invocable
ScriptEngine scriptEngine = scriptRunner.getScriptEngine();
if (scriptEngine instanceof Invocable) {
final Invocable invocable = (Invocable) scriptEngine;
// evaluate the script
scriptRunner.run(scriptEngine.getBindings(ScriptContext.ENGINE_SCOPE));
// get configured processor from the script (if it exists)
final Object obj = scriptRunner.getScriptEngine().get("actionHandler");
if (obj != null) {
final ComponentLog logger = getLogger();
try {
// set the logger if the processor wants it
invocable.invokeMethod(obj, "setLogger", logger);
} catch (final NoSuchMethodException nsme) {
if (logger.isDebugEnabled()) {
logger.debug("Configured script ActionHandler does not contain a setLogger method.");
}
}
if (configurationContext != null) {
try {
// set the logger if the processor wants it
invocable.invokeMethod(obj, "setConfigurationContext", configurationContext);
} catch (final NoSuchMethodException nsme) {
if (logger.isDebugEnabled()) {
logger.debug("Configured script ActionHandler does not contain a setConfigurationContext method.");
}
}
}
// record the processor for use later
final ActionHandler scriptedReader = invocable.getInterface(obj, ActionHandler.class);
actionHandler.set(scriptedReader);
} else {
throw new ScriptException("No RecordReader was defined by the script.");
}
}
} catch (final Exception ex) {
final ComponentLog logger = getLogger();
final String message = "Unable to load script: " + ex.getLocalizedMessage();
logger.error(message, ex);
results.add(new ValidationResult.Builder()
.subject("ScriptValidation")
.valid(false)
.explanation("Unable to load script due to " + ex.getLocalizedMessage())
.input(scriptingComponentHelper.getScriptPath())
.build());
}
// store the updated validation results
validationResults.set(results);
// return whether there was any issues loading the configured script
return results.isEmpty();
}
@Override
@OnEnabled
public void onEnabled(final ConfigurationContext context) {
synchronized (scriptingComponentHelper.isInitialized) {
if (!scriptingComponentHelper.isInitialized.get()) {
scriptingComponentHelper.createResources();
}
}
super.onEnabled(context);
// Call an non-interface method onEnabled(context), to allow a scripted ActionHandler the chance to set up as necessary
if (scriptRunner != null) {
final ScriptEngine scriptEngine = scriptRunner.getScriptEngine();
final Invocable invocable = (Invocable) scriptEngine;
if (configurationContext != null) {
try {
// Get the actual object from the script engine, versus the proxy stored in ActionHandler. The object may have additional methods,
// where ActionHandler is a proxied interface
final Object obj = scriptRunner.getScriptEngine().get("actionHandler");
if (obj != null) {
try {
invocable.invokeMethod(obj, "onEnabled", context);
} catch (final NoSuchMethodException nsme) {
if (getLogger().isDebugEnabled()) {
getLogger().debug("Configured script ActionHandler does not contain an onEnabled() method.");
}
}
} else {
throw new ScriptException("No ActionHandler was defined by the script.");
}
} catch (ScriptException se) {
throw new ProcessException("Error executing onEnabled(context) method", se);
}
}
} else {
throw new ProcessException("Error creating ScriptRunner");
}
}
public void execute(PropertyContext context, Action action, Map<String, Object> facts) {
// Attempt to call a non-ActionHandler interface method (i.e. execute(context, action, facts) from PropertyContextActionHandler)
if (scriptRunner != null) {
final ScriptEngine scriptEngine = scriptRunner.getScriptEngine();
final Invocable invocable = (Invocable) scriptEngine;
try {
// Get the actual object from the script engine, versus the proxy stored in ActionHandler. The object may have additional methods,
// where ActionHandler is a proxied interface
final Object obj = scriptRunner.getScriptEngine().get("actionHandler");
if (obj != null) {
try {
invocable.invokeMethod(obj, "execute", context, action, facts);
} catch (final NoSuchMethodException nsme) {
if (getLogger().isDebugEnabled()) {
getLogger().debug("Configured script ActionHandler is not a PropertyContextActionHandler and has no execute(context, action, facts) method, falling back to"
+ "execute(action, facts).");
}
execute(action, facts);
}
} else {
throw new ScriptException("No ActionHandler was defined by the script.");
}
} catch (ScriptException se) {
throw new ProcessException("Error executing onEnabled(context) method: " + se.getMessage(), se);
}
} else {
throw new ProcessException("Error creating ScriptRunner");
}
}
@Override
public void execute(Action action, Map<String, Object> facts) {
if (actionHandler.get() != null) {
actionHandler.get().execute(action, facts);
}
}
}

3
nifi-nar-bundles/nifi-scripting-bundle/nifi-scripting-processors/src/main/resources/META-INF/services/org.apache.nifi.controller.ControllerService
View File

@ -12,11 +12,8 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
org.apache.nifi.rules.handlers.script.ScriptedActionHandler
org.apache.nifi.record.script.ScriptedReader
org.apache.nifi.record.script.ScriptedRecordSetWriter
org.apache.nifi.record.sink.script.ScriptedRecordSink
org.apache.nifi.rules.engine.script.ScriptedRulesEngine
org.apache.nifi.lookup.script.ScriptedLookupService
org.apache.nifi.lookup.script.SimpleScriptedLookupService

92
nifi-nar-bundles/nifi-scripting-bundle/nifi-scripting-processors/src/test/java/org/apache/nifi/rules/engine/script/ScriptedRulesEngineTest.java
View File

@ -1,92 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.nifi.rules.engine.script;
import org.apache.nifi.processor.AbstractProcessor;
import org.apache.nifi.processor.ProcessContext;
import org.apache.nifi.processor.ProcessSession;
import org.apache.nifi.processor.exception.ProcessException;
import org.apache.nifi.processors.script.AccessibleScriptingComponentHelper;
import org.apache.nifi.reporting.InitializationException;
import org.apache.nifi.rules.Action;
import org.apache.nifi.script.ScriptingComponentHelper;
import org.apache.nifi.script.ScriptingComponentUtils;
import org.apache.nifi.serialization.record.MockRecordWriter;
import org.apache.nifi.util.TestRunner;
import org.apache.nifi.util.TestRunners;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import java.io.IOException;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import static org.junit.jupiter.api.Assertions.assertEquals;
public class ScriptedRulesEngineTest {
private Map<String, Object> facts = new HashMap<>();
@BeforeEach
public void setup() {
facts.put("predictedQueuedCount", 60);
facts.put("predictedTimeToBytesBackpressureMillis", 299999);
}
@Test
public void testRules() throws IOException, InitializationException {
ScriptedRulesEngine task = initTask();
List<Action> actions = task.fireRules(facts);
assertEquals(2, actions.size());
assertEquals("LOG", actions.get(0).getType());
assertEquals("DEBUG", actions.get(0).getAttributes().get("level"));
assertEquals("ALERT", actions.get(1).getType());
assertEquals("Time to backpressure < 5 mins", actions.get(1).getAttributes().get("message"));
}
private MockScriptedRulesEngine initTask() throws InitializationException {
final TestRunner runner = TestRunners.newTestRunner(new AbstractProcessor() {
@Override
public void onTrigger(final ProcessContext context, final ProcessSession session) throws ProcessException {
}
});
final MockRecordWriter writer = new MockRecordWriter(null, false); // No header, don"t quote values
runner.addControllerService("writer", writer);
runner.enableControllerService(writer);
final MockScriptedRulesEngine rulesEngine = new MockScriptedRulesEngine();
runner.addControllerService("rulesEngine", rulesEngine);
runner.setProperty(rulesEngine, "Script Engine", "Groovy");
runner.setProperty(rulesEngine, ScriptingComponentUtils.SCRIPT_FILE, "src/test/resources/groovy/test_rules_engine.groovy");
runner.setProperty(rulesEngine, ScriptingComponentUtils.SCRIPT_BODY, (String) null);
runner.setProperty(rulesEngine, ScriptingComponentUtils.MODULES, (String) null);
runner.enableControllerService(rulesEngine);
return rulesEngine;
}
public static class MockScriptedRulesEngine extends ScriptedRulesEngine implements AccessibleScriptingComponentHelper {
@Override
public ScriptingComponentHelper getScriptingComponentHelper() {
return this.scriptingComponentHelper;
}
}
}

222
nifi-nar-bundles/nifi-scripting-bundle/nifi-scripting-processors/src/test/java/org/apache/nifi/rules/handlers/script/ScriptedActionHandlerTest.java
View File

@ -1,222 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License") you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.nifi.rules.handlers.script;
import org.apache.nifi.processor.AbstractProcessor;
import org.apache.nifi.processor.ProcessContext;
import org.apache.nifi.processor.ProcessSession;
import org.apache.nifi.processor.exception.ProcessException;
import org.apache.nifi.processors.script.AccessibleScriptingComponentHelper;
import org.apache.nifi.reporting.Bulletin;
import org.apache.nifi.reporting.BulletinFactory;
import org.apache.nifi.reporting.BulletinRepository;
import org.apache.nifi.reporting.InitializationException;
import org.apache.nifi.reporting.ReportingContext;
import org.apache.nifi.reporting.Severity;
import org.apache.nifi.rules.Action;
import org.apache.nifi.script.ScriptingComponentHelper;
import org.apache.nifi.script.ScriptingComponentUtils;
import org.apache.nifi.serialization.record.MockRecordWriter;
import org.apache.nifi.util.MockBulletinRepository;
import org.apache.nifi.util.TestRunner;
import org.apache.nifi.util.TestRunners;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.mockito.Mockito;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import static org.junit.jupiter.api.Assertions.assertInstanceOf;
import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.mockito.ArgumentMatchers.anyString;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when;
public class ScriptedActionHandlerTest {
private TestRunner runner;
private ReportingContext reportingContext;
private MockScriptedActionHandler actionHandler;
private MockScriptedBulletinRepository mockScriptedBulletinRepository;
private Map<String, Object> facts = new HashMap<>();
private Map<String, String> attrs = new HashMap<>();
@BeforeEach
public void setup() {
facts.put("predictedQueuedCount", 60);
facts.put("predictedTimeToBytesBackpressureMillis", 299999);
attrs.put("level", "DEBUG");
attrs.put("message", "Time to backpressure < 5 mins");
}
@Test
public void testActions() throws InitializationException {
actionHandler = initTask("src/test/resources/groovy/test_action_handler.groovy");
List<Action> actions = Arrays.asList(new Action("LOG", attrs), new Action("ALERT", attrs));
actions.forEach((action) -> actionHandler.execute(action, facts));
// Verify a fact was added (not the intended operation of ActionHandler, but testable)
assertEquals(42, facts.get("testFact"));
}
@Test
public void testActionHandlerNotPropertyContextActionHandler() throws InitializationException {
actionHandler = initTask("src/test/resources/groovy/test_action_handler.groovy");
mockScriptedBulletinRepository = new MockScriptedBulletinRepository();
reportingContext = mock(ReportingContext.class);
when(reportingContext.getBulletinRepository()).thenReturn(mockScriptedBulletinRepository);
when(reportingContext.createBulletin(anyString(), Mockito.any(Severity.class), anyString()))
.thenAnswer(invocation -> BulletinFactory.createBulletin(invocation.getArgument(0), invocation.getArgument(1).toString(), invocation.getArgument(2)));
List<Action> actions = Arrays.asList(new Action("LOG", attrs), new Action("ALERT", attrs));
actions.forEach(action -> actionHandler.execute(reportingContext, action, facts));
// Verify instead of a bulletin being added, a fact was added (not the intended operation of ActionHandler, but testable)
assertTrue(mockScriptedBulletinRepository.bulletinList.isEmpty());
assertEquals(42, facts.get("testFact"));
}
@Test
public void testPropertyContextActionHandler() throws InitializationException {
actionHandler = initTask("src/test/resources/groovy/test_propertycontext_action_handler.groovy");
mockScriptedBulletinRepository = new MockScriptedBulletinRepository();
reportingContext = mock(ReportingContext.class);
when(reportingContext.getBulletinRepository()).thenReturn(mockScriptedBulletinRepository);
when(reportingContext.createBulletin(anyString(), Mockito.any(Severity.class), anyString()))
.thenAnswer(invocation -> BulletinFactory.createBulletin(invocation.getArgument(0), invocation.getArgument(1).toString(), invocation.getArgument(2)));
List<Action> actions = Arrays.asList(new Action("LOG", attrs), new Action("ALERT", attrs));
actions.forEach(action -> actionHandler.execute(reportingContext, action, facts));
// Verify instead of a bulletin being added, a fact was added (not the intended operation of ActionHandler, but testable)
List<Bulletin> bulletinList = mockScriptedBulletinRepository.bulletinList;
assertEquals(2, bulletinList.size());
}
@Test
public void testValidService() throws Exception {
setupTestRunner();
runner.assertValid(actionHandler);
assertInstanceOf(ScriptedActionHandler.class, actionHandler);
}
@Test
public void testAlertWithBulletinLevel() throws Exception {
setupTestRunner();
final Map<String, String> attributes = new HashMap<>();
final Map<String, Object> metrics = new HashMap<>();
final String category = "Rules Alert";
final String message = "This should be sent as an alert!";
final String severity = "INFO";
attributes.put("category", category);
attributes.put("message", message);
attributes.put("severity", severity);
metrics.put("jvmHeap", "1000000");
metrics.put("cpu", "90");
final String expectedOutput = "This should be sent as an alert!\n" +
"Alert Facts:\n" +
"Field: cpu, Value: 90\n" +
"Field: jvmHeap, Value: 1000000\n";
final Action action = new Action();
action.setType("ALERT");
action.setAttributes(attributes);
actionHandler.execute(reportingContext, action, metrics);
BulletinRepository bulletinRepository = reportingContext.getBulletinRepository();
List<Bulletin> bulletins = bulletinRepository.findBulletinsForController();
assertFalse(bulletins.isEmpty());
Bulletin bulletin = bulletins.get(0);
assertEquals(bulletin.getCategory(), category);
assertEquals(bulletin.getMessage(), expectedOutput);
assertEquals(bulletin.getLevel(), severity);
}
private static class MockScriptedBulletinRepository extends MockBulletinRepository {
List<Bulletin> bulletinList;
MockScriptedBulletinRepository() {
bulletinList = new ArrayList<>();
}
@Override
public void addBulletin(Bulletin bulletin) {
bulletinList.add(bulletin);
}
@Override
public List<Bulletin> findBulletinsForController() {
return bulletinList;
}
}
private void setupTestRunner() throws Exception {
runner = TestRunners.newTestRunner(TestProcessor.class);
MockScriptedActionHandler handler = initTask("src/test/resources/groovy/test_propertycontext_action_handler.groovy");
mockScriptedBulletinRepository = new MockScriptedBulletinRepository();
Map<String, String> properties = new HashMap<>();
properties.put(handler.getScriptingComponentHelper().SCRIPT_ENGINE.getName(), "Groovy");
properties.put(ScriptingComponentUtils.SCRIPT_FILE.getName(), "src/test/resources/groovy/test_propertycontext_action_handler.groovy");
runner.addControllerService("MockAlertHandler", handler, properties);
runner.enableControllerService(handler);
actionHandler = (MockScriptedActionHandler) runner.getProcessContext()
.getControllerServiceLookup()
.getControllerService("MockAlertHandler");
reportingContext = mock(ReportingContext.class);
when(reportingContext.getBulletinRepository()).thenReturn(mockScriptedBulletinRepository);
when(reportingContext.createBulletin(anyString(), Mockito.any(Severity.class), anyString()))
.thenAnswer(invocation -> BulletinFactory.createBulletin(invocation.getArgument(0), invocation.getArgument(1).toString(), invocation.getArgument(2)));
}
private MockScriptedActionHandler initTask(String scriptFile) throws InitializationException {
final TestRunner runner = TestRunners.newTestRunner(new AbstractProcessor() {
@Override
public void onTrigger(final ProcessContext context, final ProcessSession session) throws ProcessException {
}
});
final MockRecordWriter writer = new MockRecordWriter(null, false); // No header, don"t quote values
runner.addControllerService("writer", writer);
runner.enableControllerService(writer);
final MockScriptedActionHandler actionHandler = new MockScriptedActionHandler();
runner.addControllerService("actionHandler", actionHandler);
runner.setProperty(actionHandler, "Script Engine", "Groovy");
runner.setProperty(actionHandler, ScriptingComponentUtils.SCRIPT_FILE, scriptFile);
runner.setProperty(actionHandler, ScriptingComponentUtils.SCRIPT_BODY, (String) null);
runner.setProperty(actionHandler, ScriptingComponentUtils.MODULES, (String) null);
runner.enableControllerService(actionHandler);
return actionHandler;
}
public static class MockScriptedActionHandler extends ScriptedActionHandler implements AccessibleScriptingComponentHelper {
@Override
public ScriptingComponentHelper getScriptingComponentHelper() {
return this.scriptingComponentHelper;
}
}
}

47
nifi-nar-bundles/nifi-scripting-bundle/nifi-scripting-processors/src/test/java/org/apache/nifi/rules/handlers/script/TestProcessor.java
View File

@ -1,47 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.nifi.rules.handlers.script;
import org.apache.nifi.components.PropertyDescriptor;
import org.apache.nifi.processor.AbstractProcessor;
import org.apache.nifi.processor.ProcessContext;
import org.apache.nifi.processor.ProcessSession;
import org.apache.nifi.processor.exception.ProcessException;
import java.util.ArrayList;
import java.util.List;
public class TestProcessor extends AbstractProcessor {
private static PropertyDescriptor prop = new PropertyDescriptor.Builder()
.name("scripted-action-handler-test")
.description("Scripted Action Handler")
.identifiesControllerService(ScriptedActionHandler.class)
.required(true)
.build();
@Override
public void onTrigger(ProcessContext context, ProcessSession session) throws ProcessException {
}
@Override
protected List<PropertyDescriptor> getSupportedPropertyDescriptors() {
List<PropertyDescriptor> properties = new ArrayList<>();
properties.add(prop);
return properties;
}
}

32
nifi-nar-bundles/nifi-scripting-bundle/nifi-scripting-processors/src/test/resources/groovy/test_action_handler.groovy
View File

@ -1,32 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the 'License') you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an 'AS IS' BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import org.apache.nifi.controller.AbstractControllerService
import org.apache.nifi.rules.Action
import org.apache.nifi.rules.ActionHandler
class GroovyActionHandler extends AbstractControllerService implements ActionHandler {
@Override
void execute(Action action, Map<String, Object> facts) {
// Add a fact for verification that execute was successfully performed
facts['testFact'] = 42
}
}
actionHandler = new GroovyActionHandler()

49
nifi-nar-bundles/nifi-scripting-bundle/nifi-scripting-processors/src/test/resources/groovy/test_propertycontext_action_handler.groovy
View File

@ -1,49 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the 'License') you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an 'AS IS' BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import org.apache.nifi.context.PropertyContext
import org.apache.nifi.controller.AbstractControllerService
import org.apache.nifi.reporting.BulletinRepository
import org.apache.nifi.reporting.ReportingContext
import org.apache.nifi.reporting.Severity
import org.apache.nifi.rules.Action
import org.apache.nifi.rules.PropertyContextActionHandler
class GroovyPropertyContextActionHandler extends AbstractControllerService implements PropertyContextActionHandler {
@Override
void execute(Action action, Map<String, Object> facts) {
// Add a fact for verification that execute was successfully performed
facts['testFact'] = 42
}
@Override
void execute(PropertyContext propertyContext, Action action, Map<String, Object> facts) {
// Add a fact for verification that execute was successfully performed
if (propertyContext instanceof ReportingContext) {
ReportingContext context = (ReportingContext) propertyContext
BulletinRepository bulletinRepository = context.bulletinRepository
bulletinRepository.addBulletin(context.createBulletin('Rules Alert', Severity.INFO, 'This should be sent as an alert!\n'
+ 'Alert Facts:\n'
+ 'Field: cpu, Value: 90\n'
+ 'Field: jvmHeap, Value: 1000000\n'))
}
}
}
actionHandler = new GroovyPropertyContextActionHandler()

39
nifi-nar-bundles/nifi-scripting-bundle/nifi-scripting-processors/src/test/resources/groovy/test_rules_engine.groovy
View File

@ -1,39 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the 'License') you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an 'AS IS' BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import org.apache.nifi.controller.AbstractControllerService
import org.apache.nifi.rules.Action
import org.apache.nifi.rules.engine.RulesEngineService
class GroovyRulesEngine extends AbstractControllerService implements RulesEngineService {
@Override
List<Action> fireRules(Map<String, Object> facts) {
def actions = [] as List<Action>
if (facts['predictedQueuedCount'] > 50) {
actions << new Action('LOG', ['level': 'DEBUG'])
}
if (facts['predictedTimeToBytesBackpressureMillis'] < 300000) {
actions << new Action('ALERT', ['message': 'Time to backpressure < 5 mins'])
}
actions
}
}
rulesEngine = new GroovyRulesEngine()

6
nifi-nar-bundles/nifi-sql-reporting-bundle/nifi-sql-reporting-tasks/pom.xml
View File

@ -96,12 +96,6 @@
<version>2.0.0-SNAPSHOT</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.apache.nifi</groupId>
<artifactId>nifi-rules-engine-service-api</artifactId>
<version>2.0.0-SNAPSHOT</version>
<scope>provided</scope>
</dependency>
</dependencies>
<build>
<plugins>

119
nifi-nar-bundles/nifi-sql-reporting-bundle/nifi-sql-reporting-tasks/src/main/java/org/apache/nifi/reporting/sql/MetricsEventReportingTask.java
View File

@ -1,119 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.nifi.reporting.sql;
import org.apache.nifi.annotation.behavior.Stateful;
import org.apache.nifi.annotation.documentation.CapabilityDescription;
import org.apache.nifi.annotation.documentation.Tags;
import org.apache.nifi.annotation.lifecycle.OnScheduled;
import org.apache.nifi.components.PropertyDescriptor;
import org.apache.nifi.components.state.Scope;
import org.apache.nifi.controller.ConfigurationContext;
import org.apache.nifi.reporting.AbstractReportingTask;
import org.apache.nifi.reporting.ReportingContext;
import org.apache.nifi.reporting.ReportingInitializationContext;
import org.apache.nifi.reporting.sql.util.QueryMetricsUtil;
import org.apache.nifi.rules.Action;
import org.apache.nifi.rules.PropertyContextActionHandler;
import org.apache.nifi.rules.engine.RulesEngineService;
import org.apache.nifi.serialization.record.Record;
import org.apache.nifi.serialization.record.ResultSetRecordSet;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import static org.apache.nifi.reporting.sql.util.TrackedQueryTime.BULLETIN_END_TIME;
import static org.apache.nifi.reporting.sql.util.TrackedQueryTime.BULLETIN_START_TIME;
import static org.apache.nifi.reporting.sql.util.TrackedQueryTime.PROVENANCE_END_TIME;
import static org.apache.nifi.reporting.sql.util.TrackedQueryTime.PROVENANCE_START_TIME;
import static org.apache.nifi.util.db.JdbcProperties.VARIABLE_REGISTRY_ONLY_DEFAULT_PRECISION;
import static org.apache.nifi.util.db.JdbcProperties.VARIABLE_REGISTRY_ONLY_DEFAULT_SCALE;
@Tags({"reporting", "rules", "action", "action handler", "status", "connection", "processor", "jvm", "metrics", "history", "bulletin", "sql"})
@CapabilityDescription("Triggers rules-driven actions based on metrics values ")
@Stateful(scopes = Scope.LOCAL, description = "Stores the Reporting Task's last execution time so that on restart the task knows where it left off.")
public class MetricsEventReportingTask extends AbstractReportingTask implements QueryTimeAware {
private List<PropertyDescriptor> properties;
private MetricsQueryService metricsQueryService;
private volatile RulesEngineService rulesEngineService;
private volatile PropertyContextActionHandler actionHandler;
@Override
protected List<PropertyDescriptor> getSupportedPropertyDescriptors() {
return properties;
}
@Override
protected void init(final ReportingInitializationContext config) {
final List<PropertyDescriptor> properties = new ArrayList<>();
properties.add(QueryMetricsUtil.QUERY);
properties.add(QueryMetricsUtil.RULES_ENGINE);
properties.add(QueryMetricsUtil.ACTION_HANDLER);
properties.add(VARIABLE_REGISTRY_ONLY_DEFAULT_PRECISION);
properties.add(VARIABLE_REGISTRY_ONLY_DEFAULT_SCALE);
this.properties = Collections.unmodifiableList(properties);
}
@OnScheduled
public void setup(final ConfigurationContext context) {
actionHandler = context.getProperty(QueryMetricsUtil.ACTION_HANDLER).asControllerService(PropertyContextActionHandler.class);
rulesEngineService = context.getProperty(QueryMetricsUtil.RULES_ENGINE).asControllerService(RulesEngineService.class);
final Integer defaultPrecision = context.getProperty(VARIABLE_REGISTRY_ONLY_DEFAULT_PRECISION).evaluateAttributeExpressions().asInteger();
final Integer defaultScale = context.getProperty(VARIABLE_REGISTRY_ONLY_DEFAULT_SCALE).evaluateAttributeExpressions().asInteger();
metricsQueryService = new MetricsSqlQueryService(getLogger(), defaultPrecision, defaultScale);
}
@Override
public void onTrigger(ReportingContext context) {
String sql = context.getProperty(QueryMetricsUtil.QUERY).evaluateAttributeExpressions().getValue();
try {
sql = processStartAndEndTimes(context, sql, BULLETIN_START_TIME, BULLETIN_END_TIME);
sql = processStartAndEndTimes(context, sql, PROVENANCE_START_TIME, PROVENANCE_END_TIME);
fireRules(context, actionHandler, rulesEngineService, sql);
} catch (Exception e) {
getLogger().error("Error opening loading rules: {}", e.getMessage(), e);
}
}
private void fireRules(ReportingContext context, PropertyContextActionHandler actionHandler, RulesEngineService engine, String query) throws Exception {
getLogger().debug("Executing query: {}", query);
QueryResult queryResult = metricsQueryService.query(context, query);
ResultSetRecordSet recordSet = metricsQueryService.getResultSetRecordSet(queryResult);
Record record;
try {
while ((record = recordSet.next()) != null) {
final Map<String, Object> facts = new HashMap<>();
for (String fieldName : record.getRawFieldNames()) {
facts.put(fieldName, record.getValue(fieldName));
}
List<Action> actions = engine.fireRules(facts);
if (actions == null || actions.isEmpty()) {
getLogger().debug("No actions required for provided facts.");
} else {
actions.forEach(action -> actionHandler.execute(context, action, facts));
}
}
} finally {
metricsQueryService.closeQuietly(recordSet);
}
}
}

19
nifi-nar-bundles/nifi-sql-reporting-bundle/nifi-sql-reporting-tasks/src/main/java/org/apache/nifi/reporting/sql/util/QueryMetricsUtil.java
View File

@ -24,9 +24,6 @@ import org.apache.nifi.components.ValidationResult;
import org.apache.nifi.components.Validator;
import org.apache.nifi.expression.ExpressionLanguageScope;
import org.apache.nifi.record.sink.RecordSinkService;
import org.apache.nifi.rules.PropertyContextActionHandler;
import org.apache.nifi.rules.engine.RulesEngineService;
public class QueryMetricsUtil {
@ -59,22 +56,6 @@ public class QueryMetricsUtil {
.required(true)
.build();
public static final PropertyDescriptor RULES_ENGINE = new PropertyDescriptor.Builder()
.name("rules-engine-service")
.displayName("Rules Engine Service")
.description("Specifies the Controller Service to use for applying rules to metrics.")
.identifiesControllerService(RulesEngineService.class)
.required(true)
.build();
public static final PropertyDescriptor ACTION_HANDLER = new PropertyDescriptor.Builder()
.name("action-handler")
.displayName("Event Action Handler")
.description("Handler that will execute the defined action returned from rules engine (if Action type is supported by the handler)")
.identifiesControllerService(PropertyContextActionHandler.class)
.required(true)
.build();
public static class SqlValidator implements Validator {
@Override
public ValidationResult validate(final String subject, final String input, final ValidationContext context) {

1
nifi-nar-bundles/nifi-sql-reporting-bundle/nifi-sql-reporting-tasks/src/main/resources/META-INF/services/org.apache.nifi.reporting.ReportingTask
View File

@ -14,4 +14,3 @@
# limitations under the License.
org.apache.nifi.reporting.sql.QueryNiFiReportingTask
org.apache.nifi.reporting.sql.MetricsEventReportingTask

63
nifi-nar-bundles/nifi-sql-reporting-bundle/nifi-sql-reporting-tasks/src/main/resources/docs/org.apache.nifi.reporting.sql.MetricsEventReportingTask/additionalDetails.html
View File

@ -1,63 +0,0 @@
<!DOCTYPE html>
<html lang="en">
<!--
Licensed to the Apache Software Foundation (ASF) under one or more
contributor license agreements. See the NOTICE file distributed with
this work for additional information regarding copyright ownership.
The ASF licenses this file to You under the Apache License, Version 2.0
(the "License"); you may not use this file except in compliance with
the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
-->
<head>
<meta charset="utf-8" />
<title>Metrics Event Reporting Task</title>
<!--link rel="stylesheet" href="../../css/component-usage.css" type="text/css" /-->
<link rel="stylesheet" href="/nifi-docs/css/component-usage.css" type="text/css" />
</head>
<body>
<h2>Summary</h2>
<p>
This reporting task can be used to issue SQL queries against various NiFi metrics information, submit returned data to a rules engine (which will determine if any actions should be performed)
and execute the prescribed actions using action handlers. This task requires a RulesEngineService (which will identify any actions that should be performed) and an ActionHandler which will execute the action(s).
A distinct ActionHandler can be used to service all events or an ActionHandlerLookup can be used for dynamic handler lookup. NOTE: Optimally action handler should be associated with the expected action types
returned from the rules engine.
</p>
<p>
The reporting task can uniquely handle items from the bulletin and provenance repositories. This means that an item will only be processed once when the query is set to unique.
The query can be set to unique by defining a time window with special sql placeholders ($bulletinStartTime, $bulletinEndTime, $provenanceStartTime, $provenanceEndTime)
that the reporting task will evaluate runtime. See the SQL Query Examples section.
</p>
<br/><br/>
<h2>SQL Query Examples</h2>
<p>
<b>Example:</b> Select all fields from the <code>CONNECTION_STATUS</code> table:<br/>
<pre>SELECT * FROM CONNECTION_STATUS</pre>
</p>
<br/>
<p>
<b>Example:</b> Select connection IDs where time-to-backpressure (based on queue count) is less than 5 minutes:<br/>
<pre>SELECT connectionId FROM CONNECTION_STATUS_PREDICTIONS WHERE predictedTimeToCountBackpressureMillis < 300000</pre>
</p>
<br/>
<p>
<b>Example:</b> Get the unique bulletin categories associated with errors:<br/>
<pre>SELECT DISTINCT(bulletinCategory) FROM BULLETINS WHERE bulletinLevel = "ERROR"</pre>
</p>
<p>
<b>Example:</b> Select all fields from the <code>BULLETINS</code> table with time window:<br/>
<pre>SELECT * from BULLETINS WHERE bulletinTimestamp > $bulletinStartTime AND bulletinTimestamp <= $bulletinEndTime</pre>
</p>
<p>
<b>Example:</b> Select all fields from the <code>PROVENANCE</code> table with time window:<br/>
<pre>SELECT * from PROVENANCE where timestampMillis > $provenanceStartTime and timestampMillis <= $provenanceEndTime</pre>
</p>
<br/>
</body>
</html>

397
nifi-nar-bundles/nifi-sql-reporting-bundle/nifi-sql-reporting-tasks/src/test/java/org/apache/nifi/reporting/sql/TestMetricsEventReportingTask.java
View File

@ -1,397 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.nifi.reporting.sql;
import org.apache.nifi.attribute.expression.language.StandardPropertyValue;
import org.apache.nifi.components.PropertyDescriptor;
import org.apache.nifi.components.PropertyValue;
import org.apache.nifi.components.state.Scope;
import org.apache.nifi.context.PropertyContext;
import org.apache.nifi.controller.ConfigurationContext;
import org.apache.nifi.controller.status.ConnectionStatus;
import org.apache.nifi.controller.status.ProcessGroupStatus;
import org.apache.nifi.controller.status.ProcessorStatus;
import org.apache.nifi.controller.status.analytics.ConnectionStatusPredictions;
import org.apache.nifi.logging.ComponentLog;
import org.apache.nifi.processor.Processor;
import org.apache.nifi.provenance.MockProvenanceRepository;
import org.apache.nifi.provenance.ProvenanceEventRecord;
import org.apache.nifi.provenance.ProvenanceEventType;
import org.apache.nifi.reporting.Bulletin;
import org.apache.nifi.reporting.BulletinFactory;
import org.apache.nifi.reporting.BulletinQuery;
import org.apache.nifi.reporting.ComponentType;
import org.apache.nifi.reporting.EventAccess;
import org.apache.nifi.reporting.InitializationException;
import org.apache.nifi.reporting.ReportingContext;
import org.apache.nifi.reporting.ReportingInitializationContext;
import org.apache.nifi.reporting.sql.util.QueryMetricsUtil;
import org.apache.nifi.reporting.sql.util.TrackedQueryTime;
import org.apache.nifi.rules.MockPropertyContextActionHandler;
import org.apache.nifi.rules.PropertyContextActionHandler;
import org.apache.nifi.rules.engine.MockRulesEngineService;
import org.apache.nifi.rules.engine.RulesEngineService;
import org.apache.nifi.state.MockStateManager;
import org.apache.nifi.util.MockBulletinRepository;
import org.apache.nifi.util.MockFlowFile;
import org.apache.nifi.util.MockProcessSession;
import org.apache.nifi.util.MockPropertyValue;
import org.apache.nifi.util.SharedSessionState;
import org.apache.nifi.util.db.JdbcProperties;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.mockito.Mockito;
import org.mockito.stubbing.Answer;
import java.io.IOException;
import java.time.Instant;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.UUID;
import java.util.concurrent.atomic.AtomicLong;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.mockito.Mockito.mock;
class TestMetricsEventReportingTask {
private ReportingContext context;
private MockMetricsEventReportingTask reportingTask;
private MockPropertyContextActionHandler actionHandler;
private ProcessGroupStatus status;
private MockQueryBulletinRepository mockBulletinRepository;
private MockProvenanceRepository mockProvenanceRepository;
private AtomicLong currentTime;
private MockStateManager mockStateManager;
@BeforeEach
public void setup() {
currentTime = new AtomicLong();
status = new ProcessGroupStatus();
actionHandler = new MockPropertyContextActionHandler();
status.setId("1234");
status.setFlowFilesReceived(5);
status.setBytesReceived(10000);
status.setFlowFilesSent(10);
status.setBytesRead(20000L);
status.setBytesSent(20000);
status.setQueuedCount(100);
status.setQueuedContentSize(1024L);
status.setBytesWritten(80000L);
status.setActiveThreadCount(5);
// create a processor status with processing time
ProcessorStatus procStatus = new ProcessorStatus();
procStatus.setId("proc");
procStatus.setProcessingNanos(123456789);
Collection<ProcessorStatus> processorStatuses = new ArrayList<>();
processorStatuses.add(procStatus);
status.setProcessorStatus(processorStatuses);
ConnectionStatusPredictions connectionStatusPredictions = new ConnectionStatusPredictions();
connectionStatusPredictions.setPredictedTimeToCountBackpressureMillis(1000);
connectionStatusPredictions.setPredictedTimeToBytesBackpressureMillis(1000);
connectionStatusPredictions.setNextPredictedQueuedCount(1000000000);
connectionStatusPredictions.setNextPredictedQueuedBytes(1000000000000000L);
ConnectionStatus root1ConnectionStatus = new ConnectionStatus();
root1ConnectionStatus.setId("root1");
root1ConnectionStatus.setQueuedCount(1000);
root1ConnectionStatus.setPredictions(connectionStatusPredictions);
ConnectionStatus root2ConnectionStatus = new ConnectionStatus();
root2ConnectionStatus.setId("root2");
root2ConnectionStatus.setQueuedCount(500);
root2ConnectionStatus.setPredictions(connectionStatusPredictions);
Collection<ConnectionStatus> rootConnectionStatuses = new ArrayList<>();
rootConnectionStatuses.add(root1ConnectionStatus);
rootConnectionStatuses.add(root2ConnectionStatus);
status.setConnectionStatus(rootConnectionStatuses);
}
@Test
void testConnectionStatusTable() throws InitializationException {
final Map<PropertyDescriptor, String> properties = new HashMap<>();
properties.put(QueryMetricsUtil.QUERY, "select connectionId, predictedQueuedCount, predictedTimeToBytesBackpressureMillis from CONNECTION_STATUS_PREDICTIONS");
reportingTask = initTask(properties);
reportingTask.onTrigger(context);
List<PropertyContext> propertyContexts = actionHandler.getPropertyContexts();
assertEquals(2, actionHandler.getRows().size());
assertEquals(2, propertyContexts.size());
}
@Test
void testUniqueBulletinQueryIsInTimeWindow() throws InitializationException {
final Map<PropertyDescriptor, String> properties = new HashMap<>();
properties.put(QueryMetricsUtil.QUERY, "select bulletinCategory from BULLETINS where bulletinTimestamp > $bulletinStartTime and bulletinTimestamp <= $bulletinEndTime");
reportingTask = initTask(properties);
currentTime.set(Instant.now().toEpochMilli());
reportingTask.onTrigger(context);
assertEquals(1, actionHandler.getRows().size());
actionHandler.reset();
final Bulletin bulletin = BulletinFactory.createBulletin(ComponentType.CONTROLLER_SERVICE.name().toLowerCase(), "WARN", "test bulletin 2", "testFlowFileUuid");
mockBulletinRepository.addBulletin(bulletin);
currentTime.set(bulletin.getTimestamp().getTime());
reportingTask.onTrigger(context);
assertEquals(1, actionHandler.getRows().size());
}
@Test
void testUniqueBulletinQueryIsOutOfTimeWindow() throws InitializationException {
final Map<PropertyDescriptor, String> properties = new HashMap<>();
properties.put(QueryMetricsUtil.QUERY, "select bulletinCategory from BULLETINS where bulletinTimestamp > $bulletinStartTime and bulletinTimestamp <= $bulletinEndTime");
reportingTask = initTask(properties);
currentTime.set(Instant.now().toEpochMilli());
reportingTask.onTrigger(context);
assertEquals(1, actionHandler.getRows().size());
actionHandler.reset();
final Bulletin bulletin = BulletinFactory.createBulletin(ComponentType.CONTROLLER_SERVICE.name().toLowerCase(), "WARN", "test bulletin 2", "testFlowFileUuid");
mockBulletinRepository.addBulletin(bulletin);
currentTime.set(bulletin.getTimestamp().getTime() - 1);
reportingTask.onTrigger(context);
assertEquals(0, actionHandler.getRows().size());
}
@Test
void testUniqueProvenanceQueryIsInTimeWindow() throws InitializationException {
final Map<PropertyDescriptor, String> properties = new HashMap<>();
properties.put(QueryMetricsUtil.QUERY, "select componentId from PROVENANCE where timestampMillis > $provenanceStartTime and timestampMillis <= $provenanceEndTime");
reportingTask = initTask(properties);
currentTime.set(Instant.now().toEpochMilli());
reportingTask.onTrigger(context);
assertEquals(1, actionHandler.getRows().size());
actionHandler.reset();
MockFlowFile mockFlowFile = new MockFlowFile(2L);
ProvenanceEventRecord prov2 = mockProvenanceRepository.eventBuilder()
.setEventType(ProvenanceEventType.CREATE)
.fromFlowFile(mockFlowFile)
.setComponentId("2")
.setComponentType("ReportingTask")
.setFlowFileUUID("I am FlowFile 2")
.setEventTime(Instant.now().toEpochMilli())
.setEventDuration(100)
.setTransitUri("test://")
.setSourceSystemFlowFileIdentifier("I am FlowFile 2")
.setAlternateIdentifierUri("remote://test")
.build();
mockProvenanceRepository.registerEvent(prov2);
currentTime.set(prov2.getEventTime());
reportingTask.onTrigger(context);
assertEquals(1, actionHandler.getRows().size());
}
@Test
void testUniqueProvenanceQueryIsOutOfTimeWindow() throws InitializationException {
final Map<PropertyDescriptor, String> properties = new HashMap<>();
properties.put(QueryMetricsUtil.QUERY, "select componentId from PROVENANCE where timestampMillis > $provenanceStartTime and timestampMillis <= $provenanceEndTime");
reportingTask = initTask(properties);
currentTime.set(Instant.now().toEpochMilli());
reportingTask.onTrigger(context);
assertEquals(1, actionHandler.getRows().size());
actionHandler.reset();
MockFlowFile mockFlowFile = new MockFlowFile(2L);
ProvenanceEventRecord prov2 = mockProvenanceRepository.eventBuilder()
.setEventType(ProvenanceEventType.CREATE)
.fromFlowFile(mockFlowFile)
.setComponentId("2")
.setComponentType("ReportingTask")
.setFlowFileUUID("I am FlowFile 2")
.setEventTime(Instant.now().toEpochMilli())
.setEventDuration(100)
.setTransitUri("test://")
.setSourceSystemFlowFileIdentifier("I am FlowFile 2")
.setAlternateIdentifierUri("remote://test")
.build();
mockProvenanceRepository.registerEvent(prov2);
currentTime.set(prov2.getEventTime() - 1);
reportingTask.onTrigger(context);
assertEquals(0, actionHandler.getRows().size());
}
@Test
void testTimeWindowFromStateMap() throws IOException, InitializationException {
final Map<PropertyDescriptor, String> properties = new HashMap<>();
properties.put(QueryMetricsUtil.RECORD_SINK, "mock-record-sink");
properties.put(QueryMetricsUtil.QUERY, "select * from BULLETINS, PROVENANCE where " +
"bulletinTimestamp > $bulletinStartTime and bulletinTimestamp <= $bulletinEndTime " +
"and timestampMillis > $provenanceStartTime and timestampMillis <= $provenanceEndTime");
reportingTask = initTask(properties);
long testBulletinStartTime = 1609538145L;
long testProvenanceStartTime = 1641074145L;
final Map<String, String> stateMap = new HashMap<>();
stateMap.put(TrackedQueryTime.BULLETIN_START_TIME.name(), String.valueOf(testBulletinStartTime));
stateMap.put(TrackedQueryTime.PROVENANCE_START_TIME.name(), String.valueOf(testProvenanceStartTime));
mockStateManager.setState(stateMap, Scope.LOCAL);
final long bulletinStartTime = Long.parseLong(context.getStateManager().getState(Scope.LOCAL).get(TrackedQueryTime.BULLETIN_START_TIME.name()));
final long provenanceStartTime = Long.parseLong(context.getStateManager().getState(Scope.LOCAL).get(TrackedQueryTime.PROVENANCE_START_TIME.name()));
assertEquals(testBulletinStartTime, bulletinStartTime);
assertEquals(testProvenanceStartTime, provenanceStartTime);
final long currentTime = Instant.now().toEpochMilli();
this.currentTime.set(currentTime);
reportingTask.onTrigger(context);
final long updatedBulletinStartTime = Long.parseLong(context.getStateManager().getState(Scope.LOCAL).get(TrackedQueryTime.BULLETIN_START_TIME.name()));
final long updatedProvenanceStartTime = Long.parseLong(context.getStateManager().getState(Scope.LOCAL).get(TrackedQueryTime.PROVENANCE_START_TIME.name()));
assertEquals(currentTime, updatedBulletinStartTime);
assertEquals(currentTime, updatedProvenanceStartTime);
}
private MockMetricsEventReportingTask initTask(Map<PropertyDescriptor, String> customProperties) throws InitializationException {
final ComponentLog logger = Mockito.mock(ComponentLog.class);
reportingTask = new MockMetricsEventReportingTask();
final ReportingInitializationContext initContext = Mockito.mock(ReportingInitializationContext.class);
Mockito.when(initContext.getIdentifier()).thenReturn(UUID.randomUUID().toString());
Mockito.when(initContext.getLogger()).thenReturn(logger);
reportingTask.initialize(initContext);
Map<PropertyDescriptor, String> properties = new HashMap<>();
for (final PropertyDescriptor descriptor : reportingTask.getSupportedPropertyDescriptors()) {
properties.put(descriptor, descriptor.getDefaultValue());
}
properties.putAll(customProperties);
context = Mockito.mock(ReportingContext.class);
Mockito.when(context.isAnalyticsEnabled()).thenReturn(true);
mockStateManager = new MockStateManager(reportingTask);
Mockito.when(context.getStateManager()).thenReturn(mockStateManager);
Mockito.doAnswer((Answer<PropertyValue>) invocation -> {
final PropertyDescriptor descriptor = invocation.getArgument(0, PropertyDescriptor.class);
return new MockPropertyValue(properties.get(descriptor));
}).when(context).getProperty(Mockito.any(PropertyDescriptor.class));
final EventAccess eventAccess = Mockito.mock(EventAccess.class);
Mockito.when(context.getEventAccess()).thenReturn(eventAccess);
Mockito.when(eventAccess.getControllerStatus()).thenReturn(status);
final PropertyValue pValue = Mockito.mock(StandardPropertyValue.class);
actionHandler = new MockPropertyContextActionHandler();
Mockito.when(pValue.asControllerService(PropertyContextActionHandler.class)).thenReturn(actionHandler);
final PropertyValue resValue = Mockito.mock(StandardPropertyValue.class);
MockRulesEngineService rulesEngineService = new MockRulesEngineService();
Mockito.when(resValue.asControllerService(RulesEngineService.class)).thenReturn(rulesEngineService);
ConfigurationContext configContext = Mockito.mock(ConfigurationContext.class);
Mockito.when(configContext.getProperty(QueryMetricsUtil.RULES_ENGINE)).thenReturn(resValue);
Mockito.when(configContext.getProperty(QueryMetricsUtil.ACTION_HANDLER)).thenReturn(pValue);
Mockito.when(configContext.getProperty(JdbcProperties.VARIABLE_REGISTRY_ONLY_DEFAULT_PRECISION)).thenReturn(new MockPropertyValue("10"));
Mockito.when(configContext.getProperty(JdbcProperties.VARIABLE_REGISTRY_ONLY_DEFAULT_SCALE)).thenReturn(new MockPropertyValue("0"));
reportingTask.setup(configContext);
setupMockProvenanceRepository(eventAccess);
setupMockBulletinRepository();
return reportingTask;
}
private final class MockMetricsEventReportingTask extends MetricsEventReportingTask {
@Override
public long getCurrentTime() {
return currentTime.get();
}
}
private void setupMockBulletinRepository() {
mockBulletinRepository = new MockQueryBulletinRepository();
mockBulletinRepository.addBulletin(BulletinFactory.createBulletin(ComponentType.PROCESSOR.name().toLowerCase(), "WARN", "test bulletin 1", "testFlowFileUuid"));
Mockito.when(context.getBulletinRepository()).thenReturn(mockBulletinRepository);
}
private void setupMockProvenanceRepository(final EventAccess eventAccess) {
mockProvenanceRepository = new MockProvenanceRepository();
long currentTimeMillis = System.currentTimeMillis();
Map<String, String> previousAttributes = new HashMap<>();
previousAttributes.put("mime.type", "application/json");
previousAttributes.put("test.value", "A");
Map<String, String> updatedAttributes = new HashMap<>(previousAttributes);
updatedAttributes.put("test.value", "B");
// Generate provenance events and put them in a repository
Processor processor = mock(Processor.class);
SharedSessionState sharedState = new SharedSessionState(processor, new AtomicLong(0));
MockProcessSession processSession = new MockProcessSession(sharedState, processor);
MockFlowFile mockFlowFile = processSession.createFlowFile("Test content".getBytes());
ProvenanceEventRecord prov1 = mockProvenanceRepository.eventBuilder()
.setEventType(ProvenanceEventType.CREATE)
.fromFlowFile(mockFlowFile)
.setComponentId("1")
.setComponentType("ReportingTask")
.setFlowFileUUID("I am FlowFile 1")
.setEventTime(currentTimeMillis)
.setEventDuration(100)
.setTransitUri("test://")
.setSourceSystemFlowFileIdentifier("I am FlowFile 1")
.setAlternateIdentifierUri("remote://test")
.setAttributes(previousAttributes, updatedAttributes)
.build();
mockProvenanceRepository.registerEvent(prov1);
Mockito.when(eventAccess.getProvenanceRepository()).thenReturn(mockProvenanceRepository);
}
private static class MockQueryBulletinRepository extends MockBulletinRepository {
Map<String, List<Bulletin>> bulletins = new HashMap<>();
@Override
public void addBulletin(Bulletin bulletin) {
bulletins.computeIfAbsent(bulletin.getCategory(), key -> new ArrayList<>())
.add(bulletin);
}
@Override
public List<Bulletin> findBulletins(BulletinQuery bulletinQuery) {
return new ArrayList<>(
Optional.ofNullable(bulletins.get(bulletinQuery.getSourceType().name().toLowerCase()))
.orElse(Collections.emptyList())
);
}
@Override
public List<Bulletin> findBulletinsForController() {
return Optional.ofNullable(bulletins.get("controller"))
.orElse(Collections.emptyList());
}
}
}

67
nifi-nar-bundles/nifi-sql-reporting-bundle/nifi-sql-reporting-tasks/src/test/java/org/apache/nifi/rules/MockPropertyContextActionHandler.java
View File

@ -1,67 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.nifi.rules;
import org.apache.nifi.components.AbstractConfigurableComponent;
import org.apache.nifi.context.PropertyContext;
import org.apache.nifi.controller.ControllerServiceInitializationContext;
import org.apache.nifi.reporting.InitializationException;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
public class MockPropertyContextActionHandler extends AbstractConfigurableComponent implements PropertyContextActionHandler {
private final List<Map<String, Object>> rows = new ArrayList<>();
private final List<PropertyContext> propertyContexts = new ArrayList<>();
@Override
public void execute(PropertyContext context, Action action, Map<String, Object> facts) {
propertyContexts.add(context);
execute(action, facts);
}
@Override
public void execute(Action action, Map<String, Object> facts) {
rows.add(facts);
}
@Override
public void initialize(ControllerServiceInitializationContext context) throws InitializationException {
}
public List<Map<String, Object>> getRows() {
return rows;
}
public List<PropertyContext> getPropertyContexts() {
return propertyContexts;
}
@Override
public String getIdentifier() {
return "MockPropertyContextActionHandler";
}
public void reset() {
rows.clear();
propertyContexts.clear();
}
}

42
nifi-nar-bundles/nifi-sql-reporting-bundle/nifi-sql-reporting-tasks/src/test/java/org/apache/nifi/rules/engine/MockRulesEngineService.java
View File

@ -1,42 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.nifi.rules.engine;
import org.apache.nifi.components.AbstractConfigurableComponent;
import org.apache.nifi.controller.ControllerServiceInitializationContext;
import org.apache.nifi.rules.Action;
import org.mockito.Mockito;
import java.util.Collections;
import java.util.List;
import java.util.Map;
public class MockRulesEngineService extends AbstractConfigurableComponent implements RulesEngineService {
@Override
public List<Action> fireRules(Map<String, Object> facts) {
return Collections.singletonList(Mockito.mock(Action.class));
}
@Override
public void initialize(ControllerServiceInitializationContext context) {
}
@Override
public String getIdentifier() {
return "MockRulesEngineService";
}
}

Some files were not shown because too many files have changed in this diff Show More