diff --git a/libs/grok/build.gradle b/libs/grok/build.gradle new file mode 100644 index 00000000000..c91312cf9e8 --- /dev/null +++ b/libs/grok/build.gradle @@ -0,0 +1,60 @@ +import org.elasticsearch.gradle.precommit.PrecommitTasks + +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +apply plugin: 'elasticsearch.build' + +archivesBaseName = 'elasticsearch-grok' + +dependencies { + compile 'org.jruby.joni:joni:2.1.6' + // joni dependencies: + compile 'org.jruby.jcodings:jcodings:1.0.12' + + if (isEclipse == false || project.path == ":libs:grok-tests") { + testCompile("org.elasticsearch.test:framework:${version}") { + exclude group: 'org.elasticsearch', module: 'grok' + } + } +} + +forbiddenApisMain { + signaturesURLs = [PrecommitTasks.getResource('/forbidden/jdk-signatures.txt')] +} + +if (isEclipse) { + // in eclipse the project is under a fake root, we need to change around the source sets + sourceSets { + if (project.path == ":libs:grok") { + main.java.srcDirs = ['java'] + main.resources.srcDirs = ['resources'] + } else { + test.java.srcDirs = ['java'] + test.resources.srcDirs = ['resources'] + } + } +} + +thirdPartyAudit.excludes = [ + // joni has AsmCompilerSupport, but that isn't being used: + 'org.objectweb.asm.ClassWriter', + 'org.objectweb.asm.MethodVisitor', + 'org.objectweb.asm.Opcodes', +] diff --git a/modules/ingest-common/licenses/jcodings-1.0.12.jar.sha1 b/libs/grok/licenses/jcodings-1.0.12.jar.sha1 similarity index 100% rename from modules/ingest-common/licenses/jcodings-1.0.12.jar.sha1 rename to libs/grok/licenses/jcodings-1.0.12.jar.sha1 diff --git a/modules/ingest-common/licenses/jcodings-LICENSE.txt b/libs/grok/licenses/jcodings-LICENSE.txt similarity index 100% rename from modules/ingest-common/licenses/jcodings-LICENSE.txt rename to libs/grok/licenses/jcodings-LICENSE.txt diff --git a/modules/ingest-common/licenses/jcodings-NOTICE.txt b/libs/grok/licenses/jcodings-NOTICE.txt similarity index 100% rename from modules/ingest-common/licenses/jcodings-NOTICE.txt rename to libs/grok/licenses/jcodings-NOTICE.txt diff --git a/modules/ingest-common/licenses/joni-2.1.6.jar.sha1 b/libs/grok/licenses/joni-2.1.6.jar.sha1 similarity index 100% rename from modules/ingest-common/licenses/joni-2.1.6.jar.sha1 rename to libs/grok/licenses/joni-2.1.6.jar.sha1 diff --git a/modules/ingest-common/licenses/joni-LICENSE.txt b/libs/grok/licenses/joni-LICENSE.txt similarity index 100% rename from modules/ingest-common/licenses/joni-LICENSE.txt rename to libs/grok/licenses/joni-LICENSE.txt diff --git a/modules/ingest-common/licenses/joni-NOTICE.txt b/libs/grok/licenses/joni-NOTICE.txt similarity index 100% rename from modules/ingest-common/licenses/joni-NOTICE.txt rename to libs/grok/licenses/joni-NOTICE.txt diff --git a/libs/grok/src/main/eclipse-build.gradle b/libs/grok/src/main/eclipse-build.gradle new file mode 100644 index 00000000000..3188c7aff01 --- /dev/null +++ b/libs/grok/src/main/eclipse-build.gradle @@ -0,0 +1,3 @@ + +// this is just shell gradle file for eclipse to have separate projects for grok src and tests +apply from: '../../build.gradle' diff --git a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/Grok.java b/libs/grok/src/main/java/org/elasticsearch/grok/Grok.java similarity index 77% rename from modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/Grok.java rename to libs/grok/src/main/java/org/elasticsearch/grok/Grok.java index 576a3b85eb3..4cbeb848060 100644 --- a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/Grok.java +++ b/libs/grok/src/main/java/org/elasticsearch/grok/Grok.java @@ -17,7 +17,7 @@ * under the License. */ -package org.elasticsearch.ingest.common; +package org.elasticsearch.grok; import org.jcodings.specific.UTF8Encoding; import org.joni.Matcher; @@ -28,13 +28,19 @@ import org.joni.Region; import org.joni.Syntax; import org.joni.exception.ValueException; +import java.io.BufferedReader; +import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.io.UncheckedIOException; import java.nio.charset.StandardCharsets; import java.util.HashMap; import java.util.Iterator; import java.util.Locale; import java.util.Map; +import java.util.Collections; -final class Grok { +public final class Grok { private static final String NAME_GROUP = "name"; private static final String SUBNAME_GROUP = "subname"; @@ -54,13 +60,24 @@ final class Grok { ")?" + "\\}"; private static final Regex GROK_PATTERN_REGEX = new Regex(GROK_PATTERN.getBytes(StandardCharsets.UTF_8), 0, GROK_PATTERN.getBytes(StandardCharsets.UTF_8).length, Option.NONE, UTF8Encoding.INSTANCE, Syntax.DEFAULT); + + private static final Map builtinPatterns; + + static { + try { + builtinPatterns = loadBuiltinPatterns(); + } catch (IOException e) { + throw new UncheckedIOException("unable to load built-in grok patterns", e); + } + } + private final Map patternBank; private final boolean namedCaptures; private final Regex compiledExpression; private final String expression; - Grok(Map patternBank, String grokPattern) { + public Grok(Map patternBank, String grokPattern) { this(patternBank, grokPattern, true); } @@ -176,5 +193,42 @@ final class Grok { } return null; } + + public static Map getBuiltinPatterns() { + return builtinPatterns; + } + + private static Map loadBuiltinPatterns() throws IOException { + // Code for loading built-in grok patterns packaged with the jar file: + String[] PATTERN_NAMES = new String[] { + "aws", "bacula", "bro", "exim", "firewalls", "grok-patterns", "haproxy", + "java", "junos", "linux-syslog", "mcollective-patterns", "mongodb", "nagios", + "postgresql", "rails", "redis", "ruby" + }; + Map builtinPatterns = new HashMap<>(); + for (String pattern : PATTERN_NAMES) { + try(InputStream is = Grok.class.getResourceAsStream("/patterns/" + pattern)) { + loadPatterns(builtinPatterns, is); + } + } + return Collections.unmodifiableMap(builtinPatterns); + } + + private static void loadPatterns(Map patternBank, InputStream inputStream) throws IOException { + String line; + BufferedReader br = new BufferedReader(new InputStreamReader(inputStream, StandardCharsets.UTF_8)); + while ((line = br.readLine()) != null) { + String trimmedLine = line.replaceAll("^\\s+", ""); + if (trimmedLine.startsWith("#") || trimmedLine.length() == 0) { + continue; + } + + String[] parts = trimmedLine.split("\\s+", 2); + if (parts.length == 2) { + patternBank.put(parts[0], parts[1]); + } + } + } + } diff --git a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/GrokMatchGroup.java b/libs/grok/src/main/java/org/elasticsearch/grok/GrokMatchGroup.java similarity index 98% rename from modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/GrokMatchGroup.java rename to libs/grok/src/main/java/org/elasticsearch/grok/GrokMatchGroup.java index 6ddb8d07e76..43bf16a18b7 100644 --- a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/GrokMatchGroup.java +++ b/libs/grok/src/main/java/org/elasticsearch/grok/GrokMatchGroup.java @@ -17,7 +17,7 @@ * under the License. */ -package org.elasticsearch.ingest.common; +package org.elasticsearch.grok; final class GrokMatchGroup { private static final String DEFAULT_TYPE = "string"; diff --git a/modules/ingest-common/src/main/resources/patterns/aws b/libs/grok/src/main/resources/patterns/aws similarity index 100% rename from modules/ingest-common/src/main/resources/patterns/aws rename to libs/grok/src/main/resources/patterns/aws diff --git a/modules/ingest-common/src/main/resources/patterns/bacula b/libs/grok/src/main/resources/patterns/bacula similarity index 100% rename from modules/ingest-common/src/main/resources/patterns/bacula rename to libs/grok/src/main/resources/patterns/bacula diff --git a/modules/ingest-common/src/main/resources/patterns/bro b/libs/grok/src/main/resources/patterns/bro similarity index 100% rename from modules/ingest-common/src/main/resources/patterns/bro rename to libs/grok/src/main/resources/patterns/bro diff --git a/modules/ingest-common/src/main/resources/patterns/exim b/libs/grok/src/main/resources/patterns/exim similarity index 100% rename from modules/ingest-common/src/main/resources/patterns/exim rename to libs/grok/src/main/resources/patterns/exim diff --git a/modules/ingest-common/src/main/resources/patterns/firewalls b/libs/grok/src/main/resources/patterns/firewalls similarity index 100% rename from modules/ingest-common/src/main/resources/patterns/firewalls rename to libs/grok/src/main/resources/patterns/firewalls diff --git a/modules/ingest-common/src/main/resources/patterns/grok-patterns b/libs/grok/src/main/resources/patterns/grok-patterns similarity index 100% rename from modules/ingest-common/src/main/resources/patterns/grok-patterns rename to libs/grok/src/main/resources/patterns/grok-patterns diff --git a/modules/ingest-common/src/main/resources/patterns/haproxy b/libs/grok/src/main/resources/patterns/haproxy similarity index 100% rename from modules/ingest-common/src/main/resources/patterns/haproxy rename to libs/grok/src/main/resources/patterns/haproxy diff --git a/modules/ingest-common/src/main/resources/patterns/java b/libs/grok/src/main/resources/patterns/java similarity index 100% rename from modules/ingest-common/src/main/resources/patterns/java rename to libs/grok/src/main/resources/patterns/java diff --git a/modules/ingest-common/src/main/resources/patterns/junos b/libs/grok/src/main/resources/patterns/junos similarity index 100% rename from modules/ingest-common/src/main/resources/patterns/junos rename to libs/grok/src/main/resources/patterns/junos diff --git a/modules/ingest-common/src/main/resources/patterns/linux-syslog b/libs/grok/src/main/resources/patterns/linux-syslog similarity index 100% rename from modules/ingest-common/src/main/resources/patterns/linux-syslog rename to libs/grok/src/main/resources/patterns/linux-syslog diff --git a/modules/ingest-common/src/main/resources/patterns/mcollective-patterns b/libs/grok/src/main/resources/patterns/mcollective-patterns similarity index 100% rename from modules/ingest-common/src/main/resources/patterns/mcollective-patterns rename to libs/grok/src/main/resources/patterns/mcollective-patterns diff --git a/modules/ingest-common/src/main/resources/patterns/mongodb b/libs/grok/src/main/resources/patterns/mongodb similarity index 100% rename from modules/ingest-common/src/main/resources/patterns/mongodb rename to libs/grok/src/main/resources/patterns/mongodb diff --git a/modules/ingest-common/src/main/resources/patterns/nagios b/libs/grok/src/main/resources/patterns/nagios similarity index 100% rename from modules/ingest-common/src/main/resources/patterns/nagios rename to libs/grok/src/main/resources/patterns/nagios diff --git a/modules/ingest-common/src/main/resources/patterns/postgresql b/libs/grok/src/main/resources/patterns/postgresql similarity index 100% rename from modules/ingest-common/src/main/resources/patterns/postgresql rename to libs/grok/src/main/resources/patterns/postgresql diff --git a/modules/ingest-common/src/main/resources/patterns/rails b/libs/grok/src/main/resources/patterns/rails similarity index 100% rename from modules/ingest-common/src/main/resources/patterns/rails rename to libs/grok/src/main/resources/patterns/rails diff --git a/modules/ingest-common/src/main/resources/patterns/redis b/libs/grok/src/main/resources/patterns/redis similarity index 100% rename from modules/ingest-common/src/main/resources/patterns/redis rename to libs/grok/src/main/resources/patterns/redis diff --git a/modules/ingest-common/src/main/resources/patterns/ruby b/libs/grok/src/main/resources/patterns/ruby similarity index 100% rename from modules/ingest-common/src/main/resources/patterns/ruby rename to libs/grok/src/main/resources/patterns/ruby diff --git a/libs/grok/src/test/eclipse-build.gradle b/libs/grok/src/test/eclipse-build.gradle new file mode 100644 index 00000000000..c5d791c1663 --- /dev/null +++ b/libs/grok/src/test/eclipse-build.gradle @@ -0,0 +1,7 @@ + +// this is just shell gradle file for eclipse to have separate projects for grok src and tests +apply from: '../../build.gradle' + +dependencies { + testCompile project(':libs:grok') +} diff --git a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/GrokTests.java b/libs/grok/src/test/java/org/elasticsearch/grok/GrokTests.java similarity index 99% rename from modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/GrokTests.java rename to libs/grok/src/test/java/org/elasticsearch/grok/GrokTests.java index 2c2c0f29f83..931842d9f24 100644 --- a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/GrokTests.java +++ b/libs/grok/src/test/java/org/elasticsearch/grok/GrokTests.java @@ -17,7 +17,7 @@ * under the License. */ -package org.elasticsearch.ingest.common; +package org.elasticsearch.grok; import org.elasticsearch.test.ESTestCase; import org.junit.Before; @@ -38,8 +38,8 @@ public class GrokTests extends ESTestCase { private Map basePatterns; @Before - public void setup() throws IOException { - basePatterns = IngestCommonPlugin.loadBuiltinPatterns(); + public void setup() { + basePatterns = Grok.getBuiltinPatterns(); } public void testMatchWithoutCaptures() { diff --git a/modules/ingest-common/build.gradle b/modules/ingest-common/build.gradle index 6d93f663116..424c1197da3 100644 --- a/modules/ingest-common/build.gradle +++ b/modules/ingest-common/build.gradle @@ -23,17 +23,8 @@ esplugin { } dependencies { - compile 'org.jruby.joni:joni:2.1.6' - // joni dependencies: - compile 'org.jruby.jcodings:jcodings:1.0.12' + compile project(':libs:grok') } compileJava.options.compilerArgs << "-Xlint:-unchecked,-rawtypes" compileTestJava.options.compilerArgs << "-Xlint:-unchecked,-rawtypes" - -thirdPartyAudit.excludes = [ - // joni has AsmCompilerSupport, but that isn't being used: - 'org.objectweb.asm.ClassWriter', - 'org.objectweb.asm.MethodVisitor', - 'org.objectweb.asm.Opcodes', -] \ No newline at end of file diff --git a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/GrokProcessor.java b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/GrokProcessor.java index f8eb49b9239..8d1d2127e72 100644 --- a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/GrokProcessor.java +++ b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/GrokProcessor.java @@ -19,6 +19,7 @@ package org.elasticsearch.ingest.common; +import org.elasticsearch.grok.Grok; import org.elasticsearch.ingest.AbstractProcessor; import org.elasticsearch.ingest.ConfigurationUtils; import org.elasticsearch.ingest.IngestDocument; @@ -27,7 +28,6 @@ import org.elasticsearch.ingest.Processor; import java.util.HashMap; import java.util.List; import java.util.Map; -import java.util.Objects; import static org.elasticsearch.ingest.ConfigurationUtils.newConfigurationException; @@ -42,7 +42,7 @@ public final class GrokProcessor extends AbstractProcessor { private final boolean traceMatch; private final boolean ignoreMissing; - public GrokProcessor(String tag, Map patternBank, List matchPatterns, String matchField, + GrokProcessor(String tag, Map patternBank, List matchPatterns, String matchField, boolean traceMatch, boolean ignoreMissing) { super(tag); this.matchField = matchField; diff --git a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/IngestCommonPlugin.java b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/IngestCommonPlugin.java index 0182e290d72..79c6dadb4fc 100644 --- a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/IngestCommonPlugin.java +++ b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/IngestCommonPlugin.java @@ -40,6 +40,7 @@ import org.elasticsearch.common.settings.ClusterSettings; import org.elasticsearch.common.settings.IndexScopedSettings; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.settings.SettingsFilter; +import org.elasticsearch.grok.Grok; import org.elasticsearch.ingest.Processor; import org.elasticsearch.plugins.ActionPlugin; import org.elasticsearch.plugins.IngestPlugin; @@ -49,22 +50,9 @@ import org.elasticsearch.rest.RestHandler; public class IngestCommonPlugin extends Plugin implements ActionPlugin, IngestPlugin { - // Code for loading built-in grok patterns packaged with the jar file: - private static final String[] PATTERN_NAMES = new String[] { - "aws", "bacula", "bro", "exim", "firewalls", "grok-patterns", "haproxy", - "java", "junos", "linux-syslog", "mcollective-patterns", "mongodb", "nagios", - "postgresql", "rails", "redis", "ruby" - }; - static final Map GROK_PATTERNS; - static { - try { - GROK_PATTERNS = loadBuiltinPatterns(); - } catch (IOException e) { - throw new UncheckedIOException("unable to load built-in grok patterns", e); - } - } + static final Map GROK_PATTERNS = Grok.getBuiltinPatterns(); - public IngestCommonPlugin() throws IOException { + public IngestCommonPlugin() { } @Override @@ -108,30 +96,4 @@ public class IngestCommonPlugin extends Plugin implements ActionPlugin, IngestPl return Arrays.asList(new GrokProcessorGetAction.RestAction(settings, restController)); } - - public static Map loadBuiltinPatterns() throws IOException { - Map builtinPatterns = new HashMap<>(); - for (String pattern : PATTERN_NAMES) { - try(InputStream is = IngestCommonPlugin.class.getResourceAsStream("/patterns/" + pattern)) { - loadPatterns(builtinPatterns, is); - } - } - return Collections.unmodifiableMap(builtinPatterns); - } - - private static void loadPatterns(Map patternBank, InputStream inputStream) throws IOException { - String line; - BufferedReader br = new BufferedReader(new InputStreamReader(inputStream, StandardCharsets.UTF_8)); - while ((line = br.readLine()) != null) { - String trimmedLine = line.replaceAll("^\\s+", ""); - if (trimmedLine.startsWith("#") || trimmedLine.length() == 0) { - continue; - } - - String[] parts = trimmedLine.split("\\s+", 2); - if (parts.length == 2) { - patternBank.put(parts[0], parts[1]); - } - } - } } diff --git a/settings.gradle b/settings.gradle index f083612de10..420b4104d62 100644 --- a/settings.gradle +++ b/settings.gradle @@ -81,6 +81,7 @@ if (isEclipse) { projects << 'libs:elasticsearch-core-tests' projects << 'libs:elasticsearch-nio-tests' projects << 'libs:secure-sm-tests' + projects << 'libs:grok-tests' } include projects.toArray(new String[0]) @@ -104,6 +105,10 @@ if (isEclipse) { project(":libs:secure-sm").buildFileName = 'eclipse-build.gradle' project(":libs:secure-sm-tests").projectDir = new File(rootProject.projectDir, 'libs/secure-sm/src/test') project(":libs:secure-sm-tests").buildFileName = 'eclipse-build.gradle' + project(":libs:grok").projectDir = new File(rootProject.projectDir, 'libs/grok/src/main') + project(":libs:grok").buildFileName = 'eclipse-build.gradle' + project(":libs:grok-tests").projectDir = new File(rootProject.projectDir, 'libs/grok/src/test') + project(":libs:grok-tests").buildFileName = 'eclipse-build.gradle' } // look for extra plugins for elasticsearch