From 9e13cb59a2fe19b8ef3f772ec1d3d0b28d8b91b4 Mon Sep 17 00:00:00 2001 From: Martijn van Groningen Date: Mon, 19 Feb 2018 09:49:07 +0100 Subject: [PATCH 1/6] Moved Grok helper code to a separate Gradle module and let ingest-common module depend on it. --- libs/grok-common/build.gradle | 61 +++++++++++++++++++ .../licenses/jcodings-1.0.12.jar.sha1 | 0 .../licenses/jcodings-LICENSE.txt | 0 .../grok-common}/licenses/jcodings-NOTICE.txt | 0 .../grok-common}/licenses/joni-2.1.6.jar.sha1 | 0 .../grok-common}/licenses/joni-LICENSE.txt | 0 .../grok-common}/licenses/joni-NOTICE.txt | 0 .../java/org/elasticsearch/grok}/Grok.java | 45 +++++++++++++- .../elasticsearch/grok}/GrokMatchGroup.java | 2 +- .../src/main/resources/patterns/aws | 0 .../src/main/resources/patterns/bacula | 0 .../src/main/resources/patterns/bro | 0 .../src/main/resources/patterns/exim | 0 .../src/main/resources/patterns/firewalls | 0 .../src/main/resources/patterns/grok-patterns | 0 .../src/main/resources/patterns/haproxy | 0 .../src/main/resources/patterns/java | 0 .../src/main/resources/patterns/junos | 0 .../src/main/resources/patterns/linux-syslog | 0 .../resources/patterns/mcollective-patterns | 0 .../src/main/resources/patterns/mongodb | 0 .../src/main/resources/patterns/nagios | 0 .../src/main/resources/patterns/postgresql | 0 .../src/main/resources/patterns/rails | 0 .../src/main/resources/patterns/redis | 0 .../src/main/resources/patterns/ruby | 0 .../org/elasticsearch/grok}/GrokTests.java | 4 +- modules/ingest-common/build.gradle | 11 +--- .../ingest/common/GrokProcessor.java | 4 +- .../ingest/common/IngestCommonPlugin.java | 37 +---------- 30 files changed, 112 insertions(+), 52 deletions(-) create mode 100644 libs/grok-common/build.gradle rename {modules/ingest-common => libs/grok-common}/licenses/jcodings-1.0.12.jar.sha1 (100%) rename {modules/ingest-common => libs/grok-common}/licenses/jcodings-LICENSE.txt (100%) rename {modules/ingest-common => libs/grok-common}/licenses/jcodings-NOTICE.txt (100%) rename {modules/ingest-common => libs/grok-common}/licenses/joni-2.1.6.jar.sha1 (100%) rename {modules/ingest-common => libs/grok-common}/licenses/joni-LICENSE.txt (100%) rename {modules/ingest-common => libs/grok-common}/licenses/joni-NOTICE.txt (100%) rename {modules/ingest-common/src/main/java/org/elasticsearch/ingest/common => libs/grok-common/src/main/java/org/elasticsearch/grok}/Grok.java (80%) rename {modules/ingest-common/src/main/java/org/elasticsearch/ingest/common => libs/grok-common/src/main/java/org/elasticsearch/grok}/GrokMatchGroup.java (98%) rename {modules/ingest-common => libs/grok-common}/src/main/resources/patterns/aws (100%) rename {modules/ingest-common => libs/grok-common}/src/main/resources/patterns/bacula (100%) rename {modules/ingest-common => libs/grok-common}/src/main/resources/patterns/bro (100%) rename {modules/ingest-common => libs/grok-common}/src/main/resources/patterns/exim (100%) rename {modules/ingest-common => libs/grok-common}/src/main/resources/patterns/firewalls (100%) rename {modules/ingest-common => libs/grok-common}/src/main/resources/patterns/grok-patterns (100%) rename {modules/ingest-common => libs/grok-common}/src/main/resources/patterns/haproxy (100%) rename {modules/ingest-common => libs/grok-common}/src/main/resources/patterns/java (100%) rename {modules/ingest-common => libs/grok-common}/src/main/resources/patterns/junos (100%) rename {modules/ingest-common => libs/grok-common}/src/main/resources/patterns/linux-syslog (100%) rename {modules/ingest-common => libs/grok-common}/src/main/resources/patterns/mcollective-patterns (100%) rename {modules/ingest-common => libs/grok-common}/src/main/resources/patterns/mongodb (100%) rename {modules/ingest-common => libs/grok-common}/src/main/resources/patterns/nagios (100%) rename {modules/ingest-common => libs/grok-common}/src/main/resources/patterns/postgresql (100%) rename {modules/ingest-common => libs/grok-common}/src/main/resources/patterns/rails (100%) rename {modules/ingest-common => libs/grok-common}/src/main/resources/patterns/redis (100%) rename {modules/ingest-common => libs/grok-common}/src/main/resources/patterns/ruby (100%) rename {modules/ingest-common/src/test/java/org/elasticsearch/ingest/common => libs/grok-common/src/test/java/org/elasticsearch/grok}/GrokTests.java (99%) diff --git a/libs/grok-common/build.gradle b/libs/grok-common/build.gradle new file mode 100644 index 00000000000..64570ad3557 --- /dev/null +++ b/libs/grok-common/build.gradle @@ -0,0 +1,61 @@ +import org.elasticsearch.gradle.precommit.PrecommitTasks + +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +apply plugin: 'elasticsearch.build' + +dependencies { + compile 'org.jruby.joni:joni:2.1.6' + // joni dependencies: + compile 'org.jruby.jcodings:jcodings:1.0.12' + + testCompile "com.carrotsearch.randomizedtesting:randomizedtesting-runner:${versions.randomizedrunner}" + testCompile "junit:junit:${versions.junit}" + testCompile "org.hamcrest:hamcrest-all:${versions.hamcrest}" + if (isEclipse == false || project.path == ":libs:grok-common-tests") { + testCompile("org.elasticsearch.test:framework:${version}") { + exclude group: 'org.elasticsearch', module: 'grok-common' + } + } +} + +forbiddenApisMain { + signaturesURLs = [PrecommitTasks.getResource('/forbidden/jdk-signatures.txt')] +} + +if (isEclipse) { + // in eclipse the project is under a fake root, we need to change around the source sets + sourceSets { + if (project.path == ":libs:grok-common") { + main.java.srcDirs = ['java'] + main.resources.srcDirs = ['resources'] + } else { + test.java.srcDirs = ['java'] + test.resources.srcDirs = ['resources'] + } + } +} + +thirdPartyAudit.excludes = [ + // joni has AsmCompilerSupport, but that isn't being used: + 'org.objectweb.asm.ClassWriter', + 'org.objectweb.asm.MethodVisitor', + 'org.objectweb.asm.Opcodes', +] diff --git a/modules/ingest-common/licenses/jcodings-1.0.12.jar.sha1 b/libs/grok-common/licenses/jcodings-1.0.12.jar.sha1 similarity index 100% rename from modules/ingest-common/licenses/jcodings-1.0.12.jar.sha1 rename to libs/grok-common/licenses/jcodings-1.0.12.jar.sha1 diff --git a/modules/ingest-common/licenses/jcodings-LICENSE.txt b/libs/grok-common/licenses/jcodings-LICENSE.txt similarity index 100% rename from modules/ingest-common/licenses/jcodings-LICENSE.txt rename to libs/grok-common/licenses/jcodings-LICENSE.txt diff --git a/modules/ingest-common/licenses/jcodings-NOTICE.txt b/libs/grok-common/licenses/jcodings-NOTICE.txt similarity index 100% rename from modules/ingest-common/licenses/jcodings-NOTICE.txt rename to libs/grok-common/licenses/jcodings-NOTICE.txt diff --git a/modules/ingest-common/licenses/joni-2.1.6.jar.sha1 b/libs/grok-common/licenses/joni-2.1.6.jar.sha1 similarity index 100% rename from modules/ingest-common/licenses/joni-2.1.6.jar.sha1 rename to libs/grok-common/licenses/joni-2.1.6.jar.sha1 diff --git a/modules/ingest-common/licenses/joni-LICENSE.txt b/libs/grok-common/licenses/joni-LICENSE.txt similarity index 100% rename from modules/ingest-common/licenses/joni-LICENSE.txt rename to libs/grok-common/licenses/joni-LICENSE.txt diff --git a/modules/ingest-common/licenses/joni-NOTICE.txt b/libs/grok-common/licenses/joni-NOTICE.txt similarity index 100% rename from modules/ingest-common/licenses/joni-NOTICE.txt rename to libs/grok-common/licenses/joni-NOTICE.txt diff --git a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/Grok.java b/libs/grok-common/src/main/java/org/elasticsearch/grok/Grok.java similarity index 80% rename from modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/Grok.java rename to libs/grok-common/src/main/java/org/elasticsearch/grok/Grok.java index 576a3b85eb3..770f249b5a8 100644 --- a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/Grok.java +++ b/libs/grok-common/src/main/java/org/elasticsearch/grok/Grok.java @@ -17,7 +17,7 @@ * under the License. */ -package org.elasticsearch.ingest.common; +package org.elasticsearch.grok; import org.jcodings.specific.UTF8Encoding; import org.joni.Matcher; @@ -28,13 +28,18 @@ import org.joni.Region; import org.joni.Syntax; import org.joni.exception.ValueException; +import java.io.BufferedReader; +import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; import java.nio.charset.StandardCharsets; import java.util.HashMap; import java.util.Iterator; import java.util.Locale; import java.util.Map; +import java.util.Collections; -final class Grok { +public final class Grok { private static final String NAME_GROUP = "name"; private static final String SUBNAME_GROUP = "subname"; @@ -60,7 +65,7 @@ final class Grok { private final String expression; - Grok(Map patternBank, String grokPattern) { + public Grok(Map patternBank, String grokPattern) { this(patternBank, grokPattern, true); } @@ -176,5 +181,39 @@ final class Grok { } return null; } + + // Code for loading built-in grok patterns packaged with the jar file: + private static final String[] PATTERN_NAMES = new String[] { + "aws", "bacula", "bro", "exim", "firewalls", "grok-patterns", "haproxy", + "java", "junos", "linux-syslog", "mcollective-patterns", "mongodb", "nagios", + "postgresql", "rails", "redis", "ruby" + }; + + public static Map loadBuiltinPatterns() throws IOException { + Map builtinPatterns = new HashMap<>(); + for (String pattern : PATTERN_NAMES) { + try(InputStream is = Grok.class.getResourceAsStream("/patterns/" + pattern)) { + loadPatterns(builtinPatterns, is); + } + } + return Collections.unmodifiableMap(builtinPatterns); + } + + private static void loadPatterns(Map patternBank, InputStream inputStream) throws IOException { + String line; + BufferedReader br = new BufferedReader(new InputStreamReader(inputStream, StandardCharsets.UTF_8)); + while ((line = br.readLine()) != null) { + String trimmedLine = line.replaceAll("^\\s+", ""); + if (trimmedLine.startsWith("#") || trimmedLine.length() == 0) { + continue; + } + + String[] parts = trimmedLine.split("\\s+", 2); + if (parts.length == 2) { + patternBank.put(parts[0], parts[1]); + } + } + } + } diff --git a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/GrokMatchGroup.java b/libs/grok-common/src/main/java/org/elasticsearch/grok/GrokMatchGroup.java similarity index 98% rename from modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/GrokMatchGroup.java rename to libs/grok-common/src/main/java/org/elasticsearch/grok/GrokMatchGroup.java index 6ddb8d07e76..43bf16a18b7 100644 --- a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/GrokMatchGroup.java +++ b/libs/grok-common/src/main/java/org/elasticsearch/grok/GrokMatchGroup.java @@ -17,7 +17,7 @@ * under the License. */ -package org.elasticsearch.ingest.common; +package org.elasticsearch.grok; final class GrokMatchGroup { private static final String DEFAULT_TYPE = "string"; diff --git a/modules/ingest-common/src/main/resources/patterns/aws b/libs/grok-common/src/main/resources/patterns/aws similarity index 100% rename from modules/ingest-common/src/main/resources/patterns/aws rename to libs/grok-common/src/main/resources/patterns/aws diff --git a/modules/ingest-common/src/main/resources/patterns/bacula b/libs/grok-common/src/main/resources/patterns/bacula similarity index 100% rename from modules/ingest-common/src/main/resources/patterns/bacula rename to libs/grok-common/src/main/resources/patterns/bacula diff --git a/modules/ingest-common/src/main/resources/patterns/bro b/libs/grok-common/src/main/resources/patterns/bro similarity index 100% rename from modules/ingest-common/src/main/resources/patterns/bro rename to libs/grok-common/src/main/resources/patterns/bro diff --git a/modules/ingest-common/src/main/resources/patterns/exim b/libs/grok-common/src/main/resources/patterns/exim similarity index 100% rename from modules/ingest-common/src/main/resources/patterns/exim rename to libs/grok-common/src/main/resources/patterns/exim diff --git a/modules/ingest-common/src/main/resources/patterns/firewalls b/libs/grok-common/src/main/resources/patterns/firewalls similarity index 100% rename from modules/ingest-common/src/main/resources/patterns/firewalls rename to libs/grok-common/src/main/resources/patterns/firewalls diff --git a/modules/ingest-common/src/main/resources/patterns/grok-patterns b/libs/grok-common/src/main/resources/patterns/grok-patterns similarity index 100% rename from modules/ingest-common/src/main/resources/patterns/grok-patterns rename to libs/grok-common/src/main/resources/patterns/grok-patterns diff --git a/modules/ingest-common/src/main/resources/patterns/haproxy b/libs/grok-common/src/main/resources/patterns/haproxy similarity index 100% rename from modules/ingest-common/src/main/resources/patterns/haproxy rename to libs/grok-common/src/main/resources/patterns/haproxy diff --git a/modules/ingest-common/src/main/resources/patterns/java b/libs/grok-common/src/main/resources/patterns/java similarity index 100% rename from modules/ingest-common/src/main/resources/patterns/java rename to libs/grok-common/src/main/resources/patterns/java diff --git a/modules/ingest-common/src/main/resources/patterns/junos b/libs/grok-common/src/main/resources/patterns/junos similarity index 100% rename from modules/ingest-common/src/main/resources/patterns/junos rename to libs/grok-common/src/main/resources/patterns/junos diff --git a/modules/ingest-common/src/main/resources/patterns/linux-syslog b/libs/grok-common/src/main/resources/patterns/linux-syslog similarity index 100% rename from modules/ingest-common/src/main/resources/patterns/linux-syslog rename to libs/grok-common/src/main/resources/patterns/linux-syslog diff --git a/modules/ingest-common/src/main/resources/patterns/mcollective-patterns b/libs/grok-common/src/main/resources/patterns/mcollective-patterns similarity index 100% rename from modules/ingest-common/src/main/resources/patterns/mcollective-patterns rename to libs/grok-common/src/main/resources/patterns/mcollective-patterns diff --git a/modules/ingest-common/src/main/resources/patterns/mongodb b/libs/grok-common/src/main/resources/patterns/mongodb similarity index 100% rename from modules/ingest-common/src/main/resources/patterns/mongodb rename to libs/grok-common/src/main/resources/patterns/mongodb diff --git a/modules/ingest-common/src/main/resources/patterns/nagios b/libs/grok-common/src/main/resources/patterns/nagios similarity index 100% rename from modules/ingest-common/src/main/resources/patterns/nagios rename to libs/grok-common/src/main/resources/patterns/nagios diff --git a/modules/ingest-common/src/main/resources/patterns/postgresql b/libs/grok-common/src/main/resources/patterns/postgresql similarity index 100% rename from modules/ingest-common/src/main/resources/patterns/postgresql rename to libs/grok-common/src/main/resources/patterns/postgresql diff --git a/modules/ingest-common/src/main/resources/patterns/rails b/libs/grok-common/src/main/resources/patterns/rails similarity index 100% rename from modules/ingest-common/src/main/resources/patterns/rails rename to libs/grok-common/src/main/resources/patterns/rails diff --git a/modules/ingest-common/src/main/resources/patterns/redis b/libs/grok-common/src/main/resources/patterns/redis similarity index 100% rename from modules/ingest-common/src/main/resources/patterns/redis rename to libs/grok-common/src/main/resources/patterns/redis diff --git a/modules/ingest-common/src/main/resources/patterns/ruby b/libs/grok-common/src/main/resources/patterns/ruby similarity index 100% rename from modules/ingest-common/src/main/resources/patterns/ruby rename to libs/grok-common/src/main/resources/patterns/ruby diff --git a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/GrokTests.java b/libs/grok-common/src/test/java/org/elasticsearch/grok/GrokTests.java similarity index 99% rename from modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/GrokTests.java rename to libs/grok-common/src/test/java/org/elasticsearch/grok/GrokTests.java index 2c2c0f29f83..6504810713b 100644 --- a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/GrokTests.java +++ b/libs/grok-common/src/test/java/org/elasticsearch/grok/GrokTests.java @@ -17,7 +17,7 @@ * under the License. */ -package org.elasticsearch.ingest.common; +package org.elasticsearch.grok; import org.elasticsearch.test.ESTestCase; import org.junit.Before; @@ -39,7 +39,7 @@ public class GrokTests extends ESTestCase { @Before public void setup() throws IOException { - basePatterns = IngestCommonPlugin.loadBuiltinPatterns(); + basePatterns = Grok.loadBuiltinPatterns(); } public void testMatchWithoutCaptures() { diff --git a/modules/ingest-common/build.gradle b/modules/ingest-common/build.gradle index 6d93f663116..703f0e035de 100644 --- a/modules/ingest-common/build.gradle +++ b/modules/ingest-common/build.gradle @@ -23,17 +23,8 @@ esplugin { } dependencies { - compile 'org.jruby.joni:joni:2.1.6' - // joni dependencies: - compile 'org.jruby.jcodings:jcodings:1.0.12' + compile project(':libs:grok-common') } compileJava.options.compilerArgs << "-Xlint:-unchecked,-rawtypes" compileTestJava.options.compilerArgs << "-Xlint:-unchecked,-rawtypes" - -thirdPartyAudit.excludes = [ - // joni has AsmCompilerSupport, but that isn't being used: - 'org.objectweb.asm.ClassWriter', - 'org.objectweb.asm.MethodVisitor', - 'org.objectweb.asm.Opcodes', -] \ No newline at end of file diff --git a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/GrokProcessor.java b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/GrokProcessor.java index f8eb49b9239..8d1d2127e72 100644 --- a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/GrokProcessor.java +++ b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/GrokProcessor.java @@ -19,6 +19,7 @@ package org.elasticsearch.ingest.common; +import org.elasticsearch.grok.Grok; import org.elasticsearch.ingest.AbstractProcessor; import org.elasticsearch.ingest.ConfigurationUtils; import org.elasticsearch.ingest.IngestDocument; @@ -27,7 +28,6 @@ import org.elasticsearch.ingest.Processor; import java.util.HashMap; import java.util.List; import java.util.Map; -import java.util.Objects; import static org.elasticsearch.ingest.ConfigurationUtils.newConfigurationException; @@ -42,7 +42,7 @@ public final class GrokProcessor extends AbstractProcessor { private final boolean traceMatch; private final boolean ignoreMissing; - public GrokProcessor(String tag, Map patternBank, List matchPatterns, String matchField, + GrokProcessor(String tag, Map patternBank, List matchPatterns, String matchField, boolean traceMatch, boolean ignoreMissing) { super(tag); this.matchField = matchField; diff --git a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/IngestCommonPlugin.java b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/IngestCommonPlugin.java index 0182e290d72..8c7a1e76c3b 100644 --- a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/IngestCommonPlugin.java +++ b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/IngestCommonPlugin.java @@ -40,6 +40,7 @@ import org.elasticsearch.common.settings.ClusterSettings; import org.elasticsearch.common.settings.IndexScopedSettings; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.settings.SettingsFilter; +import org.elasticsearch.grok.Grok; import org.elasticsearch.ingest.Processor; import org.elasticsearch.plugins.ActionPlugin; import org.elasticsearch.plugins.IngestPlugin; @@ -49,22 +50,16 @@ import org.elasticsearch.rest.RestHandler; public class IngestCommonPlugin extends Plugin implements ActionPlugin, IngestPlugin { - // Code for loading built-in grok patterns packaged with the jar file: - private static final String[] PATTERN_NAMES = new String[] { - "aws", "bacula", "bro", "exim", "firewalls", "grok-patterns", "haproxy", - "java", "junos", "linux-syslog", "mcollective-patterns", "mongodb", "nagios", - "postgresql", "rails", "redis", "ruby" - }; static final Map GROK_PATTERNS; static { try { - GROK_PATTERNS = loadBuiltinPatterns(); + GROK_PATTERNS = Grok.loadBuiltinPatterns(); } catch (IOException e) { throw new UncheckedIOException("unable to load built-in grok patterns", e); } } - public IngestCommonPlugin() throws IOException { + public IngestCommonPlugin() { } @Override @@ -108,30 +103,4 @@ public class IngestCommonPlugin extends Plugin implements ActionPlugin, IngestPl return Arrays.asList(new GrokProcessorGetAction.RestAction(settings, restController)); } - - public static Map loadBuiltinPatterns() throws IOException { - Map builtinPatterns = new HashMap<>(); - for (String pattern : PATTERN_NAMES) { - try(InputStream is = IngestCommonPlugin.class.getResourceAsStream("/patterns/" + pattern)) { - loadPatterns(builtinPatterns, is); - } - } - return Collections.unmodifiableMap(builtinPatterns); - } - - private static void loadPatterns(Map patternBank, InputStream inputStream) throws IOException { - String line; - BufferedReader br = new BufferedReader(new InputStreamReader(inputStream, StandardCharsets.UTF_8)); - while ((line = br.readLine()) != null) { - String trimmedLine = line.replaceAll("^\\s+", ""); - if (trimmedLine.startsWith("#") || trimmedLine.length() == 0) { - continue; - } - - String[] parts = trimmedLine.split("\\s+", 2); - if (parts.length == 2) { - patternBank.put(parts[0], parts[1]); - } - } - } } From 2ca2585466a97fb7f3c5f03697cb0692621c06d3 Mon Sep 17 00:00:00 2001 From: Martijn van Groningen Date: Mon, 19 Feb 2018 10:00:54 +0100 Subject: [PATCH 2/6] removed unneeded test dependencies because these dependencies are already pulled in via test framework dependency. --- libs/grok-common/build.gradle | 3 --- 1 file changed, 3 deletions(-) diff --git a/libs/grok-common/build.gradle b/libs/grok-common/build.gradle index 64570ad3557..c9fd11202de 100644 --- a/libs/grok-common/build.gradle +++ b/libs/grok-common/build.gradle @@ -26,9 +26,6 @@ dependencies { // joni dependencies: compile 'org.jruby.jcodings:jcodings:1.0.12' - testCompile "com.carrotsearch.randomizedtesting:randomizedtesting-runner:${versions.randomizedrunner}" - testCompile "junit:junit:${versions.junit}" - testCompile "org.hamcrest:hamcrest-all:${versions.hamcrest}" if (isEclipse == false || project.path == ":libs:grok-common-tests") { testCompile("org.elasticsearch.test:framework:${version}") { exclude group: 'org.elasticsearch', module: 'grok-common' From 3fad16e76cc18c581493e52a0d08e7553ac17f51 Mon Sep 17 00:00:00 2001 From: Martijn van Groningen Date: Tue, 20 Feb 2018 08:02:02 +0100 Subject: [PATCH 3/6] renamed module --- libs/{grok-common => grok}/build.gradle | 8 +++++--- .../licenses/jcodings-1.0.12.jar.sha1 | 0 libs/{grok-common => grok}/licenses/jcodings-LICENSE.txt | 0 libs/{grok-common => grok}/licenses/jcodings-NOTICE.txt | 0 libs/{grok-common => grok}/licenses/joni-2.1.6.jar.sha1 | 0 libs/{grok-common => grok}/licenses/joni-LICENSE.txt | 0 libs/{grok-common => grok}/licenses/joni-NOTICE.txt | 0 .../src/main/java/org/elasticsearch/grok/Grok.java | 0 .../main/java/org/elasticsearch/grok/GrokMatchGroup.java | 0 .../{grok-common => grok}/src/main/resources/patterns/aws | 0 .../src/main/resources/patterns/bacula | 0 .../{grok-common => grok}/src/main/resources/patterns/bro | 0 .../src/main/resources/patterns/exim | 0 .../src/main/resources/patterns/firewalls | 0 .../src/main/resources/patterns/grok-patterns | 0 .../src/main/resources/patterns/haproxy | 0 .../src/main/resources/patterns/java | 0 .../src/main/resources/patterns/junos | 0 .../src/main/resources/patterns/linux-syslog | 0 .../src/main/resources/patterns/mcollective-patterns | 0 .../src/main/resources/patterns/mongodb | 0 .../src/main/resources/patterns/nagios | 0 .../src/main/resources/patterns/postgresql | 0 .../src/main/resources/patterns/rails | 0 .../src/main/resources/patterns/redis | 0 .../src/main/resources/patterns/ruby | 0 .../src/test/java/org/elasticsearch/grok/GrokTests.java | 0 modules/ingest-common/build.gradle | 2 +- 28 files changed, 6 insertions(+), 4 deletions(-) rename libs/{grok-common => grok}/build.gradle (89%) rename libs/{grok-common => grok}/licenses/jcodings-1.0.12.jar.sha1 (100%) rename libs/{grok-common => grok}/licenses/jcodings-LICENSE.txt (100%) rename libs/{grok-common => grok}/licenses/jcodings-NOTICE.txt (100%) rename libs/{grok-common => grok}/licenses/joni-2.1.6.jar.sha1 (100%) rename libs/{grok-common => grok}/licenses/joni-LICENSE.txt (100%) rename libs/{grok-common => grok}/licenses/joni-NOTICE.txt (100%) rename libs/{grok-common => grok}/src/main/java/org/elasticsearch/grok/Grok.java (100%) rename libs/{grok-common => grok}/src/main/java/org/elasticsearch/grok/GrokMatchGroup.java (100%) rename libs/{grok-common => grok}/src/main/resources/patterns/aws (100%) rename libs/{grok-common => grok}/src/main/resources/patterns/bacula (100%) rename libs/{grok-common => grok}/src/main/resources/patterns/bro (100%) rename libs/{grok-common => grok}/src/main/resources/patterns/exim (100%) rename libs/{grok-common => grok}/src/main/resources/patterns/firewalls (100%) rename libs/{grok-common => grok}/src/main/resources/patterns/grok-patterns (100%) rename libs/{grok-common => grok}/src/main/resources/patterns/haproxy (100%) rename libs/{grok-common => grok}/src/main/resources/patterns/java (100%) rename libs/{grok-common => grok}/src/main/resources/patterns/junos (100%) rename libs/{grok-common => grok}/src/main/resources/patterns/linux-syslog (100%) rename libs/{grok-common => grok}/src/main/resources/patterns/mcollective-patterns (100%) rename libs/{grok-common => grok}/src/main/resources/patterns/mongodb (100%) rename libs/{grok-common => grok}/src/main/resources/patterns/nagios (100%) rename libs/{grok-common => grok}/src/main/resources/patterns/postgresql (100%) rename libs/{grok-common => grok}/src/main/resources/patterns/rails (100%) rename libs/{grok-common => grok}/src/main/resources/patterns/redis (100%) rename libs/{grok-common => grok}/src/main/resources/patterns/ruby (100%) rename libs/{grok-common => grok}/src/test/java/org/elasticsearch/grok/GrokTests.java (100%) diff --git a/libs/grok-common/build.gradle b/libs/grok/build.gradle similarity index 89% rename from libs/grok-common/build.gradle rename to libs/grok/build.gradle index c9fd11202de..c91312cf9e8 100644 --- a/libs/grok-common/build.gradle +++ b/libs/grok/build.gradle @@ -21,14 +21,16 @@ import org.elasticsearch.gradle.precommit.PrecommitTasks apply plugin: 'elasticsearch.build' +archivesBaseName = 'elasticsearch-grok' + dependencies { compile 'org.jruby.joni:joni:2.1.6' // joni dependencies: compile 'org.jruby.jcodings:jcodings:1.0.12' - if (isEclipse == false || project.path == ":libs:grok-common-tests") { + if (isEclipse == false || project.path == ":libs:grok-tests") { testCompile("org.elasticsearch.test:framework:${version}") { - exclude group: 'org.elasticsearch', module: 'grok-common' + exclude group: 'org.elasticsearch', module: 'grok' } } } @@ -40,7 +42,7 @@ forbiddenApisMain { if (isEclipse) { // in eclipse the project is under a fake root, we need to change around the source sets sourceSets { - if (project.path == ":libs:grok-common") { + if (project.path == ":libs:grok") { main.java.srcDirs = ['java'] main.resources.srcDirs = ['resources'] } else { diff --git a/libs/grok-common/licenses/jcodings-1.0.12.jar.sha1 b/libs/grok/licenses/jcodings-1.0.12.jar.sha1 similarity index 100% rename from libs/grok-common/licenses/jcodings-1.0.12.jar.sha1 rename to libs/grok/licenses/jcodings-1.0.12.jar.sha1 diff --git a/libs/grok-common/licenses/jcodings-LICENSE.txt b/libs/grok/licenses/jcodings-LICENSE.txt similarity index 100% rename from libs/grok-common/licenses/jcodings-LICENSE.txt rename to libs/grok/licenses/jcodings-LICENSE.txt diff --git a/libs/grok-common/licenses/jcodings-NOTICE.txt b/libs/grok/licenses/jcodings-NOTICE.txt similarity index 100% rename from libs/grok-common/licenses/jcodings-NOTICE.txt rename to libs/grok/licenses/jcodings-NOTICE.txt diff --git a/libs/grok-common/licenses/joni-2.1.6.jar.sha1 b/libs/grok/licenses/joni-2.1.6.jar.sha1 similarity index 100% rename from libs/grok-common/licenses/joni-2.1.6.jar.sha1 rename to libs/grok/licenses/joni-2.1.6.jar.sha1 diff --git a/libs/grok-common/licenses/joni-LICENSE.txt b/libs/grok/licenses/joni-LICENSE.txt similarity index 100% rename from libs/grok-common/licenses/joni-LICENSE.txt rename to libs/grok/licenses/joni-LICENSE.txt diff --git a/libs/grok-common/licenses/joni-NOTICE.txt b/libs/grok/licenses/joni-NOTICE.txt similarity index 100% rename from libs/grok-common/licenses/joni-NOTICE.txt rename to libs/grok/licenses/joni-NOTICE.txt diff --git a/libs/grok-common/src/main/java/org/elasticsearch/grok/Grok.java b/libs/grok/src/main/java/org/elasticsearch/grok/Grok.java similarity index 100% rename from libs/grok-common/src/main/java/org/elasticsearch/grok/Grok.java rename to libs/grok/src/main/java/org/elasticsearch/grok/Grok.java diff --git a/libs/grok-common/src/main/java/org/elasticsearch/grok/GrokMatchGroup.java b/libs/grok/src/main/java/org/elasticsearch/grok/GrokMatchGroup.java similarity index 100% rename from libs/grok-common/src/main/java/org/elasticsearch/grok/GrokMatchGroup.java rename to libs/grok/src/main/java/org/elasticsearch/grok/GrokMatchGroup.java diff --git a/libs/grok-common/src/main/resources/patterns/aws b/libs/grok/src/main/resources/patterns/aws similarity index 100% rename from libs/grok-common/src/main/resources/patterns/aws rename to libs/grok/src/main/resources/patterns/aws diff --git a/libs/grok-common/src/main/resources/patterns/bacula b/libs/grok/src/main/resources/patterns/bacula similarity index 100% rename from libs/grok-common/src/main/resources/patterns/bacula rename to libs/grok/src/main/resources/patterns/bacula diff --git a/libs/grok-common/src/main/resources/patterns/bro b/libs/grok/src/main/resources/patterns/bro similarity index 100% rename from libs/grok-common/src/main/resources/patterns/bro rename to libs/grok/src/main/resources/patterns/bro diff --git a/libs/grok-common/src/main/resources/patterns/exim b/libs/grok/src/main/resources/patterns/exim similarity index 100% rename from libs/grok-common/src/main/resources/patterns/exim rename to libs/grok/src/main/resources/patterns/exim diff --git a/libs/grok-common/src/main/resources/patterns/firewalls b/libs/grok/src/main/resources/patterns/firewalls similarity index 100% rename from libs/grok-common/src/main/resources/patterns/firewalls rename to libs/grok/src/main/resources/patterns/firewalls diff --git a/libs/grok-common/src/main/resources/patterns/grok-patterns b/libs/grok/src/main/resources/patterns/grok-patterns similarity index 100% rename from libs/grok-common/src/main/resources/patterns/grok-patterns rename to libs/grok/src/main/resources/patterns/grok-patterns diff --git a/libs/grok-common/src/main/resources/patterns/haproxy b/libs/grok/src/main/resources/patterns/haproxy similarity index 100% rename from libs/grok-common/src/main/resources/patterns/haproxy rename to libs/grok/src/main/resources/patterns/haproxy diff --git a/libs/grok-common/src/main/resources/patterns/java b/libs/grok/src/main/resources/patterns/java similarity index 100% rename from libs/grok-common/src/main/resources/patterns/java rename to libs/grok/src/main/resources/patterns/java diff --git a/libs/grok-common/src/main/resources/patterns/junos b/libs/grok/src/main/resources/patterns/junos similarity index 100% rename from libs/grok-common/src/main/resources/patterns/junos rename to libs/grok/src/main/resources/patterns/junos diff --git a/libs/grok-common/src/main/resources/patterns/linux-syslog b/libs/grok/src/main/resources/patterns/linux-syslog similarity index 100% rename from libs/grok-common/src/main/resources/patterns/linux-syslog rename to libs/grok/src/main/resources/patterns/linux-syslog diff --git a/libs/grok-common/src/main/resources/patterns/mcollective-patterns b/libs/grok/src/main/resources/patterns/mcollective-patterns similarity index 100% rename from libs/grok-common/src/main/resources/patterns/mcollective-patterns rename to libs/grok/src/main/resources/patterns/mcollective-patterns diff --git a/libs/grok-common/src/main/resources/patterns/mongodb b/libs/grok/src/main/resources/patterns/mongodb similarity index 100% rename from libs/grok-common/src/main/resources/patterns/mongodb rename to libs/grok/src/main/resources/patterns/mongodb diff --git a/libs/grok-common/src/main/resources/patterns/nagios b/libs/grok/src/main/resources/patterns/nagios similarity index 100% rename from libs/grok-common/src/main/resources/patterns/nagios rename to libs/grok/src/main/resources/patterns/nagios diff --git a/libs/grok-common/src/main/resources/patterns/postgresql b/libs/grok/src/main/resources/patterns/postgresql similarity index 100% rename from libs/grok-common/src/main/resources/patterns/postgresql rename to libs/grok/src/main/resources/patterns/postgresql diff --git a/libs/grok-common/src/main/resources/patterns/rails b/libs/grok/src/main/resources/patterns/rails similarity index 100% rename from libs/grok-common/src/main/resources/patterns/rails rename to libs/grok/src/main/resources/patterns/rails diff --git a/libs/grok-common/src/main/resources/patterns/redis b/libs/grok/src/main/resources/patterns/redis similarity index 100% rename from libs/grok-common/src/main/resources/patterns/redis rename to libs/grok/src/main/resources/patterns/redis diff --git a/libs/grok-common/src/main/resources/patterns/ruby b/libs/grok/src/main/resources/patterns/ruby similarity index 100% rename from libs/grok-common/src/main/resources/patterns/ruby rename to libs/grok/src/main/resources/patterns/ruby diff --git a/libs/grok-common/src/test/java/org/elasticsearch/grok/GrokTests.java b/libs/grok/src/test/java/org/elasticsearch/grok/GrokTests.java similarity index 100% rename from libs/grok-common/src/test/java/org/elasticsearch/grok/GrokTests.java rename to libs/grok/src/test/java/org/elasticsearch/grok/GrokTests.java diff --git a/modules/ingest-common/build.gradle b/modules/ingest-common/build.gradle index 703f0e035de..424c1197da3 100644 --- a/modules/ingest-common/build.gradle +++ b/modules/ingest-common/build.gradle @@ -23,7 +23,7 @@ esplugin { } dependencies { - compile project(':libs:grok-common') + compile project(':libs:grok') } compileJava.options.compilerArgs << "-Xlint:-unchecked,-rawtypes" From 9c405e8595aa15045f12031127ef9230f33e5a08 Mon Sep 17 00:00:00 2001 From: Martijn van Groningen Date: Tue, 20 Feb 2018 08:09:24 +0100 Subject: [PATCH 4/6] made load method private and add another static getter that users of Grok can use to get the builtin patterns. --- .../java/org/elasticsearch/grok/Grok.java | 33 ++++++++++++------- .../org/elasticsearch/grok/GrokTests.java | 4 +-- .../ingest/common/IngestCommonPlugin.java | 9 +---- 3 files changed, 25 insertions(+), 21 deletions(-) diff --git a/libs/grok/src/main/java/org/elasticsearch/grok/Grok.java b/libs/grok/src/main/java/org/elasticsearch/grok/Grok.java index 770f249b5a8..36a06777c4f 100644 --- a/libs/grok/src/main/java/org/elasticsearch/grok/Grok.java +++ b/libs/grok/src/main/java/org/elasticsearch/grok/Grok.java @@ -28,10 +28,7 @@ import org.joni.Region; import org.joni.Syntax; import org.joni.exception.ValueException; -import java.io.BufferedReader; -import java.io.IOException; -import java.io.InputStream; -import java.io.InputStreamReader; +import java.io.*; import java.nio.charset.StandardCharsets; import java.util.HashMap; import java.util.Iterator; @@ -59,6 +56,17 @@ public final class Grok { ")?" + "\\}"; private static final Regex GROK_PATTERN_REGEX = new Regex(GROK_PATTERN.getBytes(StandardCharsets.UTF_8), 0, GROK_PATTERN.getBytes(StandardCharsets.UTF_8).length, Option.NONE, UTF8Encoding.INSTANCE, Syntax.DEFAULT); + + private static final Map builtinPatterns; + + static { + try { + builtinPatterns = loadBuiltinPatterns(); + } catch (IOException e) { + throw new UncheckedIOException("unable to load built-in grok patterns", e); + } + } + private final Map patternBank; private final boolean namedCaptures; private final Regex compiledExpression; @@ -182,14 +190,17 @@ public final class Grok { return null; } - // Code for loading built-in grok patterns packaged with the jar file: - private static final String[] PATTERN_NAMES = new String[] { - "aws", "bacula", "bro", "exim", "firewalls", "grok-patterns", "haproxy", - "java", "junos", "linux-syslog", "mcollective-patterns", "mongodb", "nagios", - "postgresql", "rails", "redis", "ruby" - }; + public static Map getBuiltinPatterns() { + return builtinPatterns; + } - public static Map loadBuiltinPatterns() throws IOException { + private static Map loadBuiltinPatterns() throws IOException { + // Code for loading built-in grok patterns packaged with the jar file: + String[] PATTERN_NAMES = new String[] { + "aws", "bacula", "bro", "exim", "firewalls", "grok-patterns", "haproxy", + "java", "junos", "linux-syslog", "mcollective-patterns", "mongodb", "nagios", + "postgresql", "rails", "redis", "ruby" + }; Map builtinPatterns = new HashMap<>(); for (String pattern : PATTERN_NAMES) { try(InputStream is = Grok.class.getResourceAsStream("/patterns/" + pattern)) { diff --git a/libs/grok/src/test/java/org/elasticsearch/grok/GrokTests.java b/libs/grok/src/test/java/org/elasticsearch/grok/GrokTests.java index 6504810713b..931842d9f24 100644 --- a/libs/grok/src/test/java/org/elasticsearch/grok/GrokTests.java +++ b/libs/grok/src/test/java/org/elasticsearch/grok/GrokTests.java @@ -38,8 +38,8 @@ public class GrokTests extends ESTestCase { private Map basePatterns; @Before - public void setup() throws IOException { - basePatterns = Grok.loadBuiltinPatterns(); + public void setup() { + basePatterns = Grok.getBuiltinPatterns(); } public void testMatchWithoutCaptures() { diff --git a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/IngestCommonPlugin.java b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/IngestCommonPlugin.java index 8c7a1e76c3b..79c6dadb4fc 100644 --- a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/IngestCommonPlugin.java +++ b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/IngestCommonPlugin.java @@ -50,14 +50,7 @@ import org.elasticsearch.rest.RestHandler; public class IngestCommonPlugin extends Plugin implements ActionPlugin, IngestPlugin { - static final Map GROK_PATTERNS; - static { - try { - GROK_PATTERNS = Grok.loadBuiltinPatterns(); - } catch (IOException e) { - throw new UncheckedIOException("unable to load built-in grok patterns", e); - } - } + static final Map GROK_PATTERNS = Grok.getBuiltinPatterns(); public IngestCommonPlugin() { } From 72de14115b9303d6a7b7351eca246cdd0fa3673a Mon Sep 17 00:00:00 2001 From: Martijn van Groningen Date: Tue, 20 Feb 2018 08:46:57 +0100 Subject: [PATCH 5/6] fixed codestyle violation --- libs/grok/src/main/java/org/elasticsearch/grok/Grok.java | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/libs/grok/src/main/java/org/elasticsearch/grok/Grok.java b/libs/grok/src/main/java/org/elasticsearch/grok/Grok.java index 36a06777c4f..4cbeb848060 100644 --- a/libs/grok/src/main/java/org/elasticsearch/grok/Grok.java +++ b/libs/grok/src/main/java/org/elasticsearch/grok/Grok.java @@ -28,7 +28,11 @@ import org.joni.Region; import org.joni.Syntax; import org.joni.exception.ValueException; -import java.io.*; +import java.io.BufferedReader; +import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.io.UncheckedIOException; import java.nio.charset.StandardCharsets; import java.util.HashMap; import java.util.Iterator; From e55ce1474dc3f4f0aa5f4ea976bc00a0c233dece Mon Sep 17 00:00:00 2001 From: Martijn van Groningen Date: Tue, 20 Feb 2018 13:49:57 +0100 Subject: [PATCH 6/6] Applied @colings86 changes to the build in order to make new module work in Eclipse too. --- libs/grok/src/main/eclipse-build.gradle | 3 +++ libs/grok/src/test/eclipse-build.gradle | 7 +++++++ settings.gradle | 5 +++++ 3 files changed, 15 insertions(+) create mode 100644 libs/grok/src/main/eclipse-build.gradle create mode 100644 libs/grok/src/test/eclipse-build.gradle diff --git a/libs/grok/src/main/eclipse-build.gradle b/libs/grok/src/main/eclipse-build.gradle new file mode 100644 index 00000000000..3188c7aff01 --- /dev/null +++ b/libs/grok/src/main/eclipse-build.gradle @@ -0,0 +1,3 @@ + +// this is just shell gradle file for eclipse to have separate projects for grok src and tests +apply from: '../../build.gradle' diff --git a/libs/grok/src/test/eclipse-build.gradle b/libs/grok/src/test/eclipse-build.gradle new file mode 100644 index 00000000000..c5d791c1663 --- /dev/null +++ b/libs/grok/src/test/eclipse-build.gradle @@ -0,0 +1,7 @@ + +// this is just shell gradle file for eclipse to have separate projects for grok src and tests +apply from: '../../build.gradle' + +dependencies { + testCompile project(':libs:grok') +} diff --git a/settings.gradle b/settings.gradle index f083612de10..420b4104d62 100644 --- a/settings.gradle +++ b/settings.gradle @@ -81,6 +81,7 @@ if (isEclipse) { projects << 'libs:elasticsearch-core-tests' projects << 'libs:elasticsearch-nio-tests' projects << 'libs:secure-sm-tests' + projects << 'libs:grok-tests' } include projects.toArray(new String[0]) @@ -104,6 +105,10 @@ if (isEclipse) { project(":libs:secure-sm").buildFileName = 'eclipse-build.gradle' project(":libs:secure-sm-tests").projectDir = new File(rootProject.projectDir, 'libs/secure-sm/src/test') project(":libs:secure-sm-tests").buildFileName = 'eclipse-build.gradle' + project(":libs:grok").projectDir = new File(rootProject.projectDir, 'libs/grok/src/main') + project(":libs:grok").buildFileName = 'eclipse-build.gradle' + project(":libs:grok-tests").projectDir = new File(rootProject.projectDir, 'libs/grok/src/test') + project(":libs:grok-tests").buildFileName = 'eclipse-build.gradle' } // look for extra plugins for elasticsearch