NIFI-9231 - Add support for custom extensions in ParseCEF

This closes #5451

Signed-off-by: David Handermann <exceptionfactory@apache.org>

nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/pom.xml

@@ -344,7 +344,7 @@
         </dependency>
         <dependency>
             <groupId>com.fluenda</groupId>
-            <artifactId>ParCEFone</artifactId>
+            <artifactId>parcefone</artifactId>
         </dependency>
         <dependency>
             <groupId>com.github.wnameless.json</groupId>

nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/ParseCEF.java

@@ -136,6 +136,16 @@ public class ParseCEF extends AbstractProcessor {
             .defaultValue("true")
             .build();
 
+    public static final PropertyDescriptor INCLUDE_CUSTOM_EXTENSIONS = new PropertyDescriptor.Builder()
+            .name("INCLUDE_CUSTOM_EXTENSIONS")
+            .displayName("Include custom extensions")
+            .description("If set to true, custom extensions (not specified in the CEF specifications) will be "
+                    + "included in the generated data/attributes.")
+            .addValidator(StandardValidators.BOOLEAN_VALIDATOR)
+            .required(true)
+            .defaultValue("false")
+            .build();
+
     public static final String UTC = "UTC";
     public static final String LOCAL_TZ = "Local Timezone (system Default)";
     public static final PropertyDescriptor TIME_REPRESENTATION = new PropertyDescriptor.Builder()
@@ -176,6 +186,7 @@ public class ParseCEF extends AbstractProcessor {
         final List<PropertyDescriptor>properties = new ArrayList<>();
         properties.add(FIELDS_DESTINATION);
         properties.add(APPEND_RAW_MESSAGE_TO_JSON);
+        properties.add(INCLUDE_CUSTOM_EXTENSIONS);
         properties.add(TIME_REPRESENTATION);
         properties.add(DATETIME_REPRESENTATION);
         return properties;
@@ -258,6 +269,7 @@ public class ParseCEF extends AbstractProcessor {
 
         try {
             final String destination = context.getProperty(FIELDS_DESTINATION).getValue();
+            final boolean includeCustomExtensions = context.getProperty(INCLUDE_CUSTOM_EXTENSIONS).asBoolean();
 
             switch (destination) {
                 case DESTINATION_ATTRIBUTES:
@@ -270,7 +282,7 @@ public class ParseCEF extends AbstractProcessor {
                     }
 
                     // Process KVs composing the Extension field
-                    for (Map.Entry<String, Object> entry : event.getExtension(true).entrySet()) {
+                    for (Map.Entry<String, Object> entry : event.getExtension(true, includeCustomExtensions).entrySet()) {
                     attributes.put("cef.extension." + entry.getKey(), prettyResult(entry.getValue(), tzId));
 
                     flowFile = session.putAllAttributes(flowFile, attributes);
@@ -283,7 +295,7 @@ public class ParseCEF extends AbstractProcessor {
 
                     // Add two JSON objects containing one CEF field each
                     results.set("header", mapper.valueToTree(event.getHeader()));
-                    results.set("extension", mapper.valueToTree(event.getExtension(true)));
+                    results.set("extension", mapper.valueToTree(event.getExtension(true, includeCustomExtensions)));
 
                     // Add the original content to original CEF content
                     // to the resulting JSON

nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/TestParseCEF.java

@@ -206,7 +206,6 @@ public class TestParseCEF {
         Assert.assertEquals("chocolate!", inner.get("test_test_test").asText());
     }
 
-
     @Test
     public void testSuccessfulParseToContentUTC() throws IOException {
         final TestRunner runner = TestRunners.newTestRunner(new ParseCEF());
@@ -288,7 +287,6 @@ public class TestParseCEF {
 
     }
 
-
     @Test
     public void testCustomValidator() {
         final TestRunner runner = TestRunners.newTestRunner(new ParseCEF());
@@ -313,5 +311,36 @@ public class TestParseCEF {
             }
         }
     }
+
+    @Test
+    public void testIncludeCustomExtensions() throws Exception {
+        String sample3 = "<159>Aug 09 08:56:28 8.8.8.8 CEF:0|x|Security|x.x.0|20|Transaction blocked|7| "
+            + "act=blocked app=https dvc=8.8.8.8 dst=8.8.8.8 dhost=www.flynas.com dpt=443 src=8.8.8.8 "
+            + "spt=53475 suser=x UserPath=LDAP://8.8.8.8 OU\\\\=1 - x x x x,OU\\\\=x x,DC\\\\=x,DC\\\\=com/x "
+            + "destinationTranslatedPort=36436 rt=1628488588000 in=65412 out=546 requestMethod=GET  "
+            + "category=20 http_response=200 http_proxy_status_code=302 duration=13 "
+            + "requestClientApplication=Mozilla/5.0 (Windows NT 10.0; WOW64; Trident/7.0;) like Gecko reason=-  "
+            + "cs1Label=Policy cs1=x x**x cs2Label=DynCat cs2=0 cs3Label=ContentType cs3=font/otf "
+            + "cn1Label=DispositionCode cn1=1047 cn2Label=ScanDuration cn2=13 "
+            + "request=https://www.flynas.com/css/fonts/GothamRounded-Book.otf URLRefer=https://www.flynas.com/en";
+
+        final TestRunner runner = TestRunners.newTestRunner(new ParseCEF());
+        runner.setProperty(ParseCEF.FIELDS_DESTINATION, ParseCEF.DESTINATION_CONTENT);
+        runner.setProperty(ParseCEF.TIME_REPRESENTATION, ParseCEF.UTC);
+        runner.setProperty(ParseCEF.INCLUDE_CUSTOM_EXTENSIONS, "true");
+        runner.enqueue(sample3.getBytes());
+        runner.run();
+
+        runner.assertAllFlowFilesTransferred(ParseCEF.REL_SUCCESS, 1);
+        final MockFlowFile mff = runner.getFlowFilesForRelationship(ParseCEF.REL_SUCCESS).get(0);
+
+        byte [] rawJson = mff.toByteArray();
+
+        JsonNode results = new ObjectMapper().readTree(rawJson);
+
+        JsonNode extension = results.get("extension");
+        Assert.assertEquals(200, extension.get("http_response").asInt());
+    }
+
 }
 

nifi-nar-bundles/nifi-standard-bundle/pom.xml

@@ -315,8 +315,8 @@
             </dependency>
             <dependency>
                 <groupId>com.fluenda</groupId>
-                <artifactId>ParCEFone</artifactId>
+                <artifactId>parcefone</artifactId>
-                <version>1.2.8</version>
+                <version>2.0.0</version>
             </dependency>
             <dependency>
                 <groupId>com.github.wnameless.json</groupId>