From 9ad7802284e2688f8682869f62bbd3605dde357c Mon Sep 17 00:00:00 2001 From: Andre F de Miranda Date: Sat, 1 Apr 2017 01:57:50 +1100 Subject: [PATCH] NIFI-1939 - Correct issue where ParseSyslog was unable to parse RFC3164 messages containg an IPv6 address as source NIFI-1939 - Fix typo and adjust ListenSyslog as per PR feedback This closes #1639. Signed-off-by: Bryan Bende --- .../processors/standard/ListenSyslog.java | 2 +- .../nifi/processors/standard/ParseSyslog.java | 8 +++-- .../standard/syslog/SyslogParser.java | 2 +- .../processors/standard/TestParseSyslog.java | 36 +++++++++++++++++++ 4 files changed, 44 insertions(+), 4 deletions(-) diff --git a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/ListenSyslog.java b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/ListenSyslog.java index 76d5cbfcc3..54d516f903 100644 --- a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/ListenSyslog.java +++ b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/ListenSyslog.java @@ -91,7 +91,7 @@ import org.apache.nifi.ssl.SSLContextService; @WritesAttribute(attribute="syslog.facility", description="The facility of the Syslog message derived from the priority."), @WritesAttribute(attribute="syslog.version", description="The optional version from the Syslog message."), @WritesAttribute(attribute="syslog.timestamp", description="The timestamp of the Syslog message."), - @WritesAttribute(attribute="syslog.hostname", description="The hostname of the Syslog message."), + @WritesAttribute(attribute="syslog.hostname", description="The hostname or IP address of the Syslog message."), @WritesAttribute(attribute="syslog.sender", description="The hostname of the Syslog server that sent the message."), @WritesAttribute(attribute="syslog.body", description="The body of the Syslog message, everything after the hostname."), @WritesAttribute(attribute="syslog.valid", description="An indicator of whether this message matched the expected formats. " + diff --git a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/ParseSyslog.java b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/ParseSyslog.java index ae08b22d6f..31691ebf59 100644 --- a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/ParseSyslog.java +++ b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/ParseSyslog.java @@ -56,13 +56,17 @@ import org.apache.nifi.stream.io.StreamUtils; @SupportsBatching @InputRequirement(Requirement.INPUT_REQUIRED) @Tags({"logs", "syslog", "attributes", "system", "event", "message"}) -@CapabilityDescription("Parses the contents of a Syslog message and adds attributes to the FlowFile for each of the parts of the Syslog message") +@CapabilityDescription("Attempts to parses the contents of a Syslog message in accordance to RFC5424 and RFC3164 " + + "formats and adds attributes to the FlowFile for each of the parts of the Syslog message." + + "Note: Be mindfull that RFC3164 is informational and a wide range of different implementations are present in" + + " the wild. If messages fail parsing, considering using RFC5424 or using a generic parsing processors such as " + + "ExtractGrok.") @WritesAttributes({@WritesAttribute(attribute = "syslog.priority", description = "The priority of the Syslog message."), @WritesAttribute(attribute = "syslog.severity", description = "The severity of the Syslog message derived from the priority."), @WritesAttribute(attribute = "syslog.facility", description = "The facility of the Syslog message derived from the priority."), @WritesAttribute(attribute = "syslog.version", description = "The optional version from the Syslog message."), @WritesAttribute(attribute = "syslog.timestamp", description = "The timestamp of the Syslog message."), - @WritesAttribute(attribute = "syslog.hostname", description = "The hostname of the Syslog message."), + @WritesAttribute(attribute = "syslog.hostname", description = "The hostname or IP address of the Syslog message."), @WritesAttribute(attribute = "syslog.sender", description = "The hostname of the Syslog server that sent the message."), @WritesAttribute(attribute = "syslog.body", description = "The body of the Syslog message, everything after the hostname.")}) @SeeAlso({ListenSyslog.class, PutSyslog.class}) diff --git a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/syslog/SyslogParser.java b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/syslog/SyslogParser.java index 52caedba81..8235febca8 100644 --- a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/syslog/SyslogParser.java +++ b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/syslog/SyslogParser.java @@ -52,7 +52,7 @@ public class SyslogParser { // stamp MMM d HH:mm:ss, single digit date has two spaces "([A-Z][a-z][a-z]\\s{1,2}\\d{1,2}\\s\\d{2}[:]\\d{2}[:]\\d{2})" + "\\s" + // separator - "([\\w][\\w\\d\\.@-]*)" + // host + "([\\w][\\w\\d(\\.|\\:)@-]*)" + // host "\\s(.*)$"; // body public static final Collection MESSAGE_PATTERNS; diff --git a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/TestParseSyslog.java b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/TestParseSyslog.java index 9e84b376a0..ff6cc907e2 100644 --- a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/TestParseSyslog.java +++ b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/TestParseSyslog.java @@ -29,13 +29,18 @@ public class TestParseSyslog { static final String FAC = "4"; static final String TIME = "Oct 13 15:43:23"; static final String HOST = "localhost.home"; + static final String IPV6SRC = "fe80::216:3300:eeaa:eeaa"; + static final String IPV4SRC = "8.8.4.4"; static final String BODY = "some message"; static final String VALID_MESSAGE_RFC3164_0 = "<" + PRI + ">" + TIME + " " + HOST + " " + BODY + "\n"; + static final String VALID_MESSAGE_RFC3164_1 = "<" + PRI + ">" + TIME + " " + IPV6SRC + " " + BODY + "\n"; + static final String VALID_MESSAGE_RFC3164_2 = "<" + PRI + ">" + TIME + " " + IPV4SRC + " " + BODY + "\n"; @Test public void testSuccessfulParse3164() { final TestRunner runner = TestRunners.newTestRunner(new ParseSyslog()); + runner.enqueue(VALID_MESSAGE_RFC3164_0.getBytes()); runner.run(); @@ -49,6 +54,37 @@ public class TestParseSyslog { mff.assertAttributeEquals(SyslogAttributes.TIMESTAMP.key(), TIME); } + @Test + public void testValidIPv6Source() { + final TestRunner runner = TestRunners.newTestRunner(new ParseSyslog()); + runner.enqueue(VALID_MESSAGE_RFC3164_1.getBytes()); + runner.run(); + + runner.assertAllFlowFilesTransferred(ParseSyslog.REL_SUCCESS, 1); + final MockFlowFile mff = runner.getFlowFilesForRelationship(ParseSyslog.REL_SUCCESS).get(0); + mff.assertAttributeEquals(SyslogAttributes.BODY.key(), BODY); + mff.assertAttributeEquals(SyslogAttributes.FACILITY.key(), FAC); + mff.assertAttributeEquals(SyslogAttributes.HOSTNAME.key(), IPV6SRC); + mff.assertAttributeEquals(SyslogAttributes.PRIORITY.key(), PRI); + mff.assertAttributeEquals(SyslogAttributes.SEVERITY.key(), SEV); + mff.assertAttributeEquals(SyslogAttributes.TIMESTAMP.key(), TIME); + } + + @Test + public void testValidIPv4Source() { + final TestRunner runner = TestRunners.newTestRunner(new ParseSyslog()); + runner.enqueue(VALID_MESSAGE_RFC3164_2.getBytes()); + runner.run(); + + runner.assertAllFlowFilesTransferred(ParseSyslog.REL_SUCCESS, 1); + final MockFlowFile mff = runner.getFlowFilesForRelationship(ParseSyslog.REL_SUCCESS).get(0); + mff.assertAttributeEquals(SyslogAttributes.BODY.key(), BODY); + mff.assertAttributeEquals(SyslogAttributes.FACILITY.key(), FAC); + mff.assertAttributeEquals(SyslogAttributes.HOSTNAME.key(), IPV4SRC); + mff.assertAttributeEquals(SyslogAttributes.PRIORITY.key(), PRI); + mff.assertAttributeEquals(SyslogAttributes.SEVERITY.key(), SEV); + mff.assertAttributeEquals(SyslogAttributes.TIMESTAMP.key(), TIME); + } @Test public void testInvalidMessage() {