From 9e13cb59a2fe19b8ef3f772ec1d3d0b28d8b91b4 Mon Sep 17 00:00:00 2001 From: Martijn van Groningen Date: Mon, 19 Feb 2018 09:49:07 +0100 Subject: [PATCH] Moved Grok helper code to a separate Gradle module and let ingest-common module depend on it. --- libs/grok-common/build.gradle | 61 +++++++++++++++++++ .../licenses/jcodings-1.0.12.jar.sha1 | 0 .../licenses/jcodings-LICENSE.txt | 0 .../grok-common}/licenses/jcodings-NOTICE.txt | 0 .../grok-common}/licenses/joni-2.1.6.jar.sha1 | 0 .../grok-common}/licenses/joni-LICENSE.txt | 0 .../grok-common}/licenses/joni-NOTICE.txt | 0 .../java/org/elasticsearch/grok}/Grok.java | 45 +++++++++++++- .../elasticsearch/grok}/GrokMatchGroup.java | 2 +- .../src/main/resources/patterns/aws | 0 .../src/main/resources/patterns/bacula | 0 .../src/main/resources/patterns/bro | 0 .../src/main/resources/patterns/exim | 0 .../src/main/resources/patterns/firewalls | 0 .../src/main/resources/patterns/grok-patterns | 0 .../src/main/resources/patterns/haproxy | 0 .../src/main/resources/patterns/java | 0 .../src/main/resources/patterns/junos | 0 .../src/main/resources/patterns/linux-syslog | 0 .../resources/patterns/mcollective-patterns | 0 .../src/main/resources/patterns/mongodb | 0 .../src/main/resources/patterns/nagios | 0 .../src/main/resources/patterns/postgresql | 0 .../src/main/resources/patterns/rails | 0 .../src/main/resources/patterns/redis | 0 .../src/main/resources/patterns/ruby | 0 .../org/elasticsearch/grok}/GrokTests.java | 4 +- modules/ingest-common/build.gradle | 11 +--- .../ingest/common/GrokProcessor.java | 4 +- .../ingest/common/IngestCommonPlugin.java | 37 +---------- 30 files changed, 112 insertions(+), 52 deletions(-) create mode 100644 libs/grok-common/build.gradle rename {modules/ingest-common => libs/grok-common}/licenses/jcodings-1.0.12.jar.sha1 (100%) rename {modules/ingest-common => libs/grok-common}/licenses/jcodings-LICENSE.txt (100%) rename {modules/ingest-common => libs/grok-common}/licenses/jcodings-NOTICE.txt (100%) rename {modules/ingest-common => libs/grok-common}/licenses/joni-2.1.6.jar.sha1 (100%) rename {modules/ingest-common => libs/grok-common}/licenses/joni-LICENSE.txt (100%) rename {modules/ingest-common => libs/grok-common}/licenses/joni-NOTICE.txt (100%) rename {modules/ingest-common/src/main/java/org/elasticsearch/ingest/common => libs/grok-common/src/main/java/org/elasticsearch/grok}/Grok.java (80%) rename {modules/ingest-common/src/main/java/org/elasticsearch/ingest/common => libs/grok-common/src/main/java/org/elasticsearch/grok}/GrokMatchGroup.java (98%) rename {modules/ingest-common => libs/grok-common}/src/main/resources/patterns/aws (100%) rename {modules/ingest-common => libs/grok-common}/src/main/resources/patterns/bacula (100%) rename {modules/ingest-common => libs/grok-common}/src/main/resources/patterns/bro (100%) rename {modules/ingest-common => libs/grok-common}/src/main/resources/patterns/exim (100%) rename {modules/ingest-common => libs/grok-common}/src/main/resources/patterns/firewalls (100%) rename {modules/ingest-common => libs/grok-common}/src/main/resources/patterns/grok-patterns (100%) rename {modules/ingest-common => libs/grok-common}/src/main/resources/patterns/haproxy (100%) rename {modules/ingest-common => libs/grok-common}/src/main/resources/patterns/java (100%) rename {modules/ingest-common => libs/grok-common}/src/main/resources/patterns/junos (100%) rename {modules/ingest-common => libs/grok-common}/src/main/resources/patterns/linux-syslog (100%) rename {modules/ingest-common => libs/grok-common}/src/main/resources/patterns/mcollective-patterns (100%) rename {modules/ingest-common => libs/grok-common}/src/main/resources/patterns/mongodb (100%) rename {modules/ingest-common => libs/grok-common}/src/main/resources/patterns/nagios (100%) rename {modules/ingest-common => libs/grok-common}/src/main/resources/patterns/postgresql (100%) rename {modules/ingest-common => libs/grok-common}/src/main/resources/patterns/rails (100%) rename {modules/ingest-common => libs/grok-common}/src/main/resources/patterns/redis (100%) rename {modules/ingest-common => libs/grok-common}/src/main/resources/patterns/ruby (100%) rename {modules/ingest-common/src/test/java/org/elasticsearch/ingest/common => libs/grok-common/src/test/java/org/elasticsearch/grok}/GrokTests.java (99%) diff --git a/libs/grok-common/build.gradle b/libs/grok-common/build.gradle new file mode 100644 index 00000000000..64570ad3557 --- /dev/null +++ b/libs/grok-common/build.gradle @@ -0,0 +1,61 @@ +import org.elasticsearch.gradle.precommit.PrecommitTasks + +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +apply plugin: 'elasticsearch.build' + +dependencies { + compile 'org.jruby.joni:joni:2.1.6' + // joni dependencies: + compile 'org.jruby.jcodings:jcodings:1.0.12' + + testCompile "com.carrotsearch.randomizedtesting:randomizedtesting-runner:${versions.randomizedrunner}" + testCompile "junit:junit:${versions.junit}" + testCompile "org.hamcrest:hamcrest-all:${versions.hamcrest}" + if (isEclipse == false || project.path == ":libs:grok-common-tests") { + testCompile("org.elasticsearch.test:framework:${version}") { + exclude group: 'org.elasticsearch', module: 'grok-common' + } + } +} + +forbiddenApisMain { + signaturesURLs = [PrecommitTasks.getResource('/forbidden/jdk-signatures.txt')] +} + +if (isEclipse) { + // in eclipse the project is under a fake root, we need to change around the source sets + sourceSets { + if (project.path == ":libs:grok-common") { + main.java.srcDirs = ['java'] + main.resources.srcDirs = ['resources'] + } else { + test.java.srcDirs = ['java'] + test.resources.srcDirs = ['resources'] + } + } +} + +thirdPartyAudit.excludes = [ + // joni has AsmCompilerSupport, but that isn't being used: + 'org.objectweb.asm.ClassWriter', + 'org.objectweb.asm.MethodVisitor', + 'org.objectweb.asm.Opcodes', +] diff --git a/modules/ingest-common/licenses/jcodings-1.0.12.jar.sha1 b/libs/grok-common/licenses/jcodings-1.0.12.jar.sha1 similarity index 100% rename from modules/ingest-common/licenses/jcodings-1.0.12.jar.sha1 rename to libs/grok-common/licenses/jcodings-1.0.12.jar.sha1 diff --git a/modules/ingest-common/licenses/jcodings-LICENSE.txt b/libs/grok-common/licenses/jcodings-LICENSE.txt similarity index 100% rename from modules/ingest-common/licenses/jcodings-LICENSE.txt rename to libs/grok-common/licenses/jcodings-LICENSE.txt diff --git a/modules/ingest-common/licenses/jcodings-NOTICE.txt b/libs/grok-common/licenses/jcodings-NOTICE.txt similarity index 100% rename from modules/ingest-common/licenses/jcodings-NOTICE.txt rename to libs/grok-common/licenses/jcodings-NOTICE.txt diff --git a/modules/ingest-common/licenses/joni-2.1.6.jar.sha1 b/libs/grok-common/licenses/joni-2.1.6.jar.sha1 similarity index 100% rename from modules/ingest-common/licenses/joni-2.1.6.jar.sha1 rename to libs/grok-common/licenses/joni-2.1.6.jar.sha1 diff --git a/modules/ingest-common/licenses/joni-LICENSE.txt b/libs/grok-common/licenses/joni-LICENSE.txt similarity index 100% rename from modules/ingest-common/licenses/joni-LICENSE.txt rename to libs/grok-common/licenses/joni-LICENSE.txt diff --git a/modules/ingest-common/licenses/joni-NOTICE.txt b/libs/grok-common/licenses/joni-NOTICE.txt similarity index 100% rename from modules/ingest-common/licenses/joni-NOTICE.txt rename to libs/grok-common/licenses/joni-NOTICE.txt diff --git a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/Grok.java b/libs/grok-common/src/main/java/org/elasticsearch/grok/Grok.java similarity index 80% rename from modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/Grok.java rename to libs/grok-common/src/main/java/org/elasticsearch/grok/Grok.java index 576a3b85eb3..770f249b5a8 100644 --- a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/Grok.java +++ b/libs/grok-common/src/main/java/org/elasticsearch/grok/Grok.java @@ -17,7 +17,7 @@ * under the License. */ -package org.elasticsearch.ingest.common; +package org.elasticsearch.grok; import org.jcodings.specific.UTF8Encoding; import org.joni.Matcher; @@ -28,13 +28,18 @@ import org.joni.Region; import org.joni.Syntax; import org.joni.exception.ValueException; +import java.io.BufferedReader; +import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; import java.nio.charset.StandardCharsets; import java.util.HashMap; import java.util.Iterator; import java.util.Locale; import java.util.Map; +import java.util.Collections; -final class Grok { +public final class Grok { private static final String NAME_GROUP = "name"; private static final String SUBNAME_GROUP = "subname"; @@ -60,7 +65,7 @@ final class Grok { private final String expression; - Grok(Map patternBank, String grokPattern) { + public Grok(Map patternBank, String grokPattern) { this(patternBank, grokPattern, true); } @@ -176,5 +181,39 @@ final class Grok { } return null; } + + // Code for loading built-in grok patterns packaged with the jar file: + private static final String[] PATTERN_NAMES = new String[] { + "aws", "bacula", "bro", "exim", "firewalls", "grok-patterns", "haproxy", + "java", "junos", "linux-syslog", "mcollective-patterns", "mongodb", "nagios", + "postgresql", "rails", "redis", "ruby" + }; + + public static Map loadBuiltinPatterns() throws IOException { + Map builtinPatterns = new HashMap<>(); + for (String pattern : PATTERN_NAMES) { + try(InputStream is = Grok.class.getResourceAsStream("/patterns/" + pattern)) { + loadPatterns(builtinPatterns, is); + } + } + return Collections.unmodifiableMap(builtinPatterns); + } + + private static void loadPatterns(Map patternBank, InputStream inputStream) throws IOException { + String line; + BufferedReader br = new BufferedReader(new InputStreamReader(inputStream, StandardCharsets.UTF_8)); + while ((line = br.readLine()) != null) { + String trimmedLine = line.replaceAll("^\\s+", ""); + if (trimmedLine.startsWith("#") || trimmedLine.length() == 0) { + continue; + } + + String[] parts = trimmedLine.split("\\s+", 2); + if (parts.length == 2) { + patternBank.put(parts[0], parts[1]); + } + } + } + } diff --git a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/GrokMatchGroup.java b/libs/grok-common/src/main/java/org/elasticsearch/grok/GrokMatchGroup.java similarity index 98% rename from modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/GrokMatchGroup.java rename to libs/grok-common/src/main/java/org/elasticsearch/grok/GrokMatchGroup.java index 6ddb8d07e76..43bf16a18b7 100644 --- a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/GrokMatchGroup.java +++ b/libs/grok-common/src/main/java/org/elasticsearch/grok/GrokMatchGroup.java @@ -17,7 +17,7 @@ * under the License. */ -package org.elasticsearch.ingest.common; +package org.elasticsearch.grok; final class GrokMatchGroup { private static final String DEFAULT_TYPE = "string"; diff --git a/modules/ingest-common/src/main/resources/patterns/aws b/libs/grok-common/src/main/resources/patterns/aws similarity index 100% rename from modules/ingest-common/src/main/resources/patterns/aws rename to libs/grok-common/src/main/resources/patterns/aws diff --git a/modules/ingest-common/src/main/resources/patterns/bacula b/libs/grok-common/src/main/resources/patterns/bacula similarity index 100% rename from modules/ingest-common/src/main/resources/patterns/bacula rename to libs/grok-common/src/main/resources/patterns/bacula diff --git a/modules/ingest-common/src/main/resources/patterns/bro b/libs/grok-common/src/main/resources/patterns/bro similarity index 100% rename from modules/ingest-common/src/main/resources/patterns/bro rename to libs/grok-common/src/main/resources/patterns/bro diff --git a/modules/ingest-common/src/main/resources/patterns/exim b/libs/grok-common/src/main/resources/patterns/exim similarity index 100% rename from modules/ingest-common/src/main/resources/patterns/exim rename to libs/grok-common/src/main/resources/patterns/exim diff --git a/modules/ingest-common/src/main/resources/patterns/firewalls b/libs/grok-common/src/main/resources/patterns/firewalls similarity index 100% rename from modules/ingest-common/src/main/resources/patterns/firewalls rename to libs/grok-common/src/main/resources/patterns/firewalls diff --git a/modules/ingest-common/src/main/resources/patterns/grok-patterns b/libs/grok-common/src/main/resources/patterns/grok-patterns similarity index 100% rename from modules/ingest-common/src/main/resources/patterns/grok-patterns rename to libs/grok-common/src/main/resources/patterns/grok-patterns diff --git a/modules/ingest-common/src/main/resources/patterns/haproxy b/libs/grok-common/src/main/resources/patterns/haproxy similarity index 100% rename from modules/ingest-common/src/main/resources/patterns/haproxy rename to libs/grok-common/src/main/resources/patterns/haproxy diff --git a/modules/ingest-common/src/main/resources/patterns/java b/libs/grok-common/src/main/resources/patterns/java similarity index 100% rename from modules/ingest-common/src/main/resources/patterns/java rename to libs/grok-common/src/main/resources/patterns/java diff --git a/modules/ingest-common/src/main/resources/patterns/junos b/libs/grok-common/src/main/resources/patterns/junos similarity index 100% rename from modules/ingest-common/src/main/resources/patterns/junos rename to libs/grok-common/src/main/resources/patterns/junos diff --git a/modules/ingest-common/src/main/resources/patterns/linux-syslog b/libs/grok-common/src/main/resources/patterns/linux-syslog similarity index 100% rename from modules/ingest-common/src/main/resources/patterns/linux-syslog rename to libs/grok-common/src/main/resources/patterns/linux-syslog diff --git a/modules/ingest-common/src/main/resources/patterns/mcollective-patterns b/libs/grok-common/src/main/resources/patterns/mcollective-patterns similarity index 100% rename from modules/ingest-common/src/main/resources/patterns/mcollective-patterns rename to libs/grok-common/src/main/resources/patterns/mcollective-patterns diff --git a/modules/ingest-common/src/main/resources/patterns/mongodb b/libs/grok-common/src/main/resources/patterns/mongodb similarity index 100% rename from modules/ingest-common/src/main/resources/patterns/mongodb rename to libs/grok-common/src/main/resources/patterns/mongodb diff --git a/modules/ingest-common/src/main/resources/patterns/nagios b/libs/grok-common/src/main/resources/patterns/nagios similarity index 100% rename from modules/ingest-common/src/main/resources/patterns/nagios rename to libs/grok-common/src/main/resources/patterns/nagios diff --git a/modules/ingest-common/src/main/resources/patterns/postgresql b/libs/grok-common/src/main/resources/patterns/postgresql similarity index 100% rename from modules/ingest-common/src/main/resources/patterns/postgresql rename to libs/grok-common/src/main/resources/patterns/postgresql diff --git a/modules/ingest-common/src/main/resources/patterns/rails b/libs/grok-common/src/main/resources/patterns/rails similarity index 100% rename from modules/ingest-common/src/main/resources/patterns/rails rename to libs/grok-common/src/main/resources/patterns/rails diff --git a/modules/ingest-common/src/main/resources/patterns/redis b/libs/grok-common/src/main/resources/patterns/redis similarity index 100% rename from modules/ingest-common/src/main/resources/patterns/redis rename to libs/grok-common/src/main/resources/patterns/redis diff --git a/modules/ingest-common/src/main/resources/patterns/ruby b/libs/grok-common/src/main/resources/patterns/ruby similarity index 100% rename from modules/ingest-common/src/main/resources/patterns/ruby rename to libs/grok-common/src/main/resources/patterns/ruby diff --git a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/GrokTests.java b/libs/grok-common/src/test/java/org/elasticsearch/grok/GrokTests.java similarity index 99% rename from modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/GrokTests.java rename to libs/grok-common/src/test/java/org/elasticsearch/grok/GrokTests.java index 2c2c0f29f83..6504810713b 100644 --- a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/GrokTests.java +++ b/libs/grok-common/src/test/java/org/elasticsearch/grok/GrokTests.java @@ -17,7 +17,7 @@ * under the License. */ -package org.elasticsearch.ingest.common; +package org.elasticsearch.grok; import org.elasticsearch.test.ESTestCase; import org.junit.Before; @@ -39,7 +39,7 @@ public class GrokTests extends ESTestCase { @Before public void setup() throws IOException { - basePatterns = IngestCommonPlugin.loadBuiltinPatterns(); + basePatterns = Grok.loadBuiltinPatterns(); } public void testMatchWithoutCaptures() { diff --git a/modules/ingest-common/build.gradle b/modules/ingest-common/build.gradle index 6d93f663116..703f0e035de 100644 --- a/modules/ingest-common/build.gradle +++ b/modules/ingest-common/build.gradle @@ -23,17 +23,8 @@ esplugin { } dependencies { - compile 'org.jruby.joni:joni:2.1.6' - // joni dependencies: - compile 'org.jruby.jcodings:jcodings:1.0.12' + compile project(':libs:grok-common') } compileJava.options.compilerArgs << "-Xlint:-unchecked,-rawtypes" compileTestJava.options.compilerArgs << "-Xlint:-unchecked,-rawtypes" - -thirdPartyAudit.excludes = [ - // joni has AsmCompilerSupport, but that isn't being used: - 'org.objectweb.asm.ClassWriter', - 'org.objectweb.asm.MethodVisitor', - 'org.objectweb.asm.Opcodes', -] \ No newline at end of file diff --git a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/GrokProcessor.java b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/GrokProcessor.java index f8eb49b9239..8d1d2127e72 100644 --- a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/GrokProcessor.java +++ b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/GrokProcessor.java @@ -19,6 +19,7 @@ package org.elasticsearch.ingest.common; +import org.elasticsearch.grok.Grok; import org.elasticsearch.ingest.AbstractProcessor; import org.elasticsearch.ingest.ConfigurationUtils; import org.elasticsearch.ingest.IngestDocument; @@ -27,7 +28,6 @@ import org.elasticsearch.ingest.Processor; import java.util.HashMap; import java.util.List; import java.util.Map; -import java.util.Objects; import static org.elasticsearch.ingest.ConfigurationUtils.newConfigurationException; @@ -42,7 +42,7 @@ public final class GrokProcessor extends AbstractProcessor { private final boolean traceMatch; private final boolean ignoreMissing; - public GrokProcessor(String tag, Map patternBank, List matchPatterns, String matchField, + GrokProcessor(String tag, Map patternBank, List matchPatterns, String matchField, boolean traceMatch, boolean ignoreMissing) { super(tag); this.matchField = matchField; diff --git a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/IngestCommonPlugin.java b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/IngestCommonPlugin.java index 0182e290d72..8c7a1e76c3b 100644 --- a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/IngestCommonPlugin.java +++ b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/IngestCommonPlugin.java @@ -40,6 +40,7 @@ import org.elasticsearch.common.settings.ClusterSettings; import org.elasticsearch.common.settings.IndexScopedSettings; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.settings.SettingsFilter; +import org.elasticsearch.grok.Grok; import org.elasticsearch.ingest.Processor; import org.elasticsearch.plugins.ActionPlugin; import org.elasticsearch.plugins.IngestPlugin; @@ -49,22 +50,16 @@ import org.elasticsearch.rest.RestHandler; public class IngestCommonPlugin extends Plugin implements ActionPlugin, IngestPlugin { - // Code for loading built-in grok patterns packaged with the jar file: - private static final String[] PATTERN_NAMES = new String[] { - "aws", "bacula", "bro", "exim", "firewalls", "grok-patterns", "haproxy", - "java", "junos", "linux-syslog", "mcollective-patterns", "mongodb", "nagios", - "postgresql", "rails", "redis", "ruby" - }; static final Map GROK_PATTERNS; static { try { - GROK_PATTERNS = loadBuiltinPatterns(); + GROK_PATTERNS = Grok.loadBuiltinPatterns(); } catch (IOException e) { throw new UncheckedIOException("unable to load built-in grok patterns", e); } } - public IngestCommonPlugin() throws IOException { + public IngestCommonPlugin() { } @Override @@ -108,30 +103,4 @@ public class IngestCommonPlugin extends Plugin implements ActionPlugin, IngestPl return Arrays.asList(new GrokProcessorGetAction.RestAction(settings, restController)); } - - public static Map loadBuiltinPatterns() throws IOException { - Map builtinPatterns = new HashMap<>(); - for (String pattern : PATTERN_NAMES) { - try(InputStream is = IngestCommonPlugin.class.getResourceAsStream("/patterns/" + pattern)) { - loadPatterns(builtinPatterns, is); - } - } - return Collections.unmodifiableMap(builtinPatterns); - } - - private static void loadPatterns(Map patternBank, InputStream inputStream) throws IOException { - String line; - BufferedReader br = new BufferedReader(new InputStreamReader(inputStream, StandardCharsets.UTF_8)); - while ((line = br.readLine()) != null) { - String trimmedLine = line.replaceAll("^\\s+", ""); - if (trimmedLine.startsWith("#") || trimmedLine.length() == 0) { - continue; - } - - String[] parts = trimmedLine.split("\\s+", 2); - if (parts.length == 2) { - patternBank.put(parts[0], parts[1]); - } - } - } }