Apache
/
nifi
mirror of https://github.com/apache/nifi.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

NIFI-9231 - Add support for custom extensions in ParseCEF 

This closes #5451

Signed-off-by: David Handermann <exceptionfactory@apache.org>
This commit is contained in:
Pierre Villard 2021-10-11 12:27:52 +02:00 committed by exceptionfactory
parent ebcb1cc0d5
commit 9588758aac
No known key found for this signature in database
GPG Key ID: 29B6A52D2AAE8DBA
4 changed files with 48 additions and 7 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/pom.xml
View File

@ -344,7 +344,7 @@
</dependency>
<dependency>
<groupId>com.fluenda</groupId>
<artifactId>ParCEFone</artifactId>
<artifactId>parcefone</artifactId>
</dependency>
<dependency>
<groupId>com.github.wnameless.json</groupId>

16
nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/ParseCEF.java
View File

@ -136,6 +136,16 @@ public class ParseCEF extends AbstractProcessor {
.defaultValue("true")
.build();
public static final PropertyDescriptor INCLUDE_CUSTOM_EXTENSIONS = new PropertyDescriptor.Builder()
.name("INCLUDE_CUSTOM_EXTENSIONS")
.displayName("Include custom extensions")
.description("If set to true, custom extensions (not specified in the CEF specifications) will be "
+ "included in the generated data/attributes.")
.addValidator(StandardValidators.BOOLEAN_VALIDATOR)
.required(true)
.defaultValue("false")
.build();
public static final String UTC = "UTC";
public static final String LOCAL_TZ = "Local Timezone (system Default)";
public static final PropertyDescriptor TIME_REPRESENTATION = new PropertyDescriptor.Builder()
@ -176,6 +186,7 @@ public class ParseCEF extends AbstractProcessor {
final List<PropertyDescriptor>properties = new ArrayList<>();
properties.add(FIELDS_DESTINATION);
properties.add(APPEND_RAW_MESSAGE_TO_JSON);
properties.add(INCLUDE_CUSTOM_EXTENSIONS);
properties.add(TIME_REPRESENTATION);
properties.add(DATETIME_REPRESENTATION);
return properties;
@ -258,6 +269,7 @@ public class ParseCEF extends AbstractProcessor {
try {
final String destination = context.getProperty(FIELDS_DESTINATION).getValue();
final boolean includeCustomExtensions = context.getProperty(INCLUDE_CUSTOM_EXTENSIONS).asBoolean();
switch (destination) {
case DESTINATION_ATTRIBUTES:
@ -270,7 +282,7 @@ public class ParseCEF extends AbstractProcessor {
}
// Process KVs composing the Extension field
for (Map.Entry<String, Object> entry : event.getExtension(true).entrySet()) {
for (Map.Entry<String, Object> entry : event.getExtension(true, includeCustomExtensions).entrySet()) {
attributes.put("cef.extension." + entry.getKey(), prettyResult(entry.getValue(), tzId));
flowFile = session.putAllAttributes(flowFile, attributes);
@ -283,7 +295,7 @@ public class ParseCEF extends AbstractProcessor {
// Add two JSON objects containing one CEF field each
results.set("header", mapper.valueToTree(event.getHeader()));
results.set("extension", mapper.valueToTree(event.getExtension(true)));
results.set("extension", mapper.valueToTree(event.getExtension(true, includeCustomExtensions)));
// Add the original content to original CEF content
// to the resulting JSON

33
nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/TestParseCEF.java
View File

@ -206,7 +206,6 @@ public class TestParseCEF {
Assert.assertEquals("chocolate!", inner.get("test_test_test").asText());
}
@Test
public void testSuccessfulParseToContentUTC() throws IOException {
final TestRunner runner = TestRunners.newTestRunner(new ParseCEF());
@ -288,7 +287,6 @@ public class TestParseCEF {
}
@Test
public void testCustomValidator() {
final TestRunner runner = TestRunners.newTestRunner(new ParseCEF());
@ -313,5 +311,36 @@ public class TestParseCEF {
}
}
}
@Test
public void testIncludeCustomExtensions() throws Exception {
String sample3 = "<159>Aug 09 08:56:28 8.8.8.8 CEF:0|x|Security|x.x.0|20|Transaction blocked|7| "
+ "act=blocked app=https dvc=8.8.8.8 dst=8.8.8.8 dhost=www.flynas.com dpt=443 src=8.8.8.8 "
+ "spt=53475 suser=x UserPath=LDAP://8.8.8.8 OU\\\\=1 - x x x x,OU\\\\=x x,DC\\\\=x,DC\\\\=com/x "
+ "destinationTranslatedPort=36436 rt=1628488588000 in=65412 out=546 requestMethod=GET "
+ "category=20 http_response=200 http_proxy_status_code=302 duration=13 "
+ "requestClientApplication=Mozilla/5.0 (Windows NT 10.0; WOW64; Trident/7.0;) like Gecko reason=- "
+ "cs1Label=Policy cs1=x x**x cs2Label=DynCat cs2=0 cs3Label=ContentType cs3=font/otf "
+ "cn1Label=DispositionCode cn1=1047 cn2Label=ScanDuration cn2=13 "
+ "request=https://www.flynas.com/css/fonts/GothamRounded-Book.otf URLRefer=https://www.flynas.com/en";
final TestRunner runner = TestRunners.newTestRunner(new ParseCEF());
runner.setProperty(ParseCEF.FIELDS_DESTINATION, ParseCEF.DESTINATION_CONTENT);
runner.setProperty(ParseCEF.TIME_REPRESENTATION, ParseCEF.UTC);
runner.setProperty(ParseCEF.INCLUDE_CUSTOM_EXTENSIONS, "true");
runner.enqueue(sample3.getBytes());
runner.run();
runner.assertAllFlowFilesTransferred(ParseCEF.REL_SUCCESS, 1);
final MockFlowFile mff = runner.getFlowFilesForRelationship(ParseCEF.REL_SUCCESS).get(0);
byte [] rawJson = mff.toByteArray();
JsonNode results = new ObjectMapper().readTree(rawJson);
JsonNode extension = results.get("extension");
Assert.assertEquals(200, extension.get("http_response").asInt());
}
}

4
nifi-nar-bundles/nifi-standard-bundle/pom.xml
View File

@ -315,8 +315,8 @@
</dependency>
<dependency>
<groupId>com.fluenda</groupId>
<artifactId>ParCEFone</artifactId>
<version>1.2.8</version>
<artifactId>parcefone</artifactId>
<version>2.0.0</version>
</dependency>
<dependency>
<groupId>com.github.wnameless.json</groupId>