Moved Grok helper code to a separate Gradle module and let ingest-common module depend on it.

libs/grok/build.gradle

@@ -0,0 +1,60 @@
+import org.elasticsearch.gradle.precommit.PrecommitTasks
+
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+apply plugin: 'elasticsearch.build'
+
+archivesBaseName = 'elasticsearch-grok'
+
+dependencies {
+    compile 'org.jruby.joni:joni:2.1.6'
+    // joni dependencies:
+    compile 'org.jruby.jcodings:jcodings:1.0.12'
+
+    if (isEclipse == false || project.path == ":libs:grok-tests") {
+        testCompile("org.elasticsearch.test:framework:${version}") {
+            exclude group: 'org.elasticsearch', module: 'grok'
+        }
+    }
+}
+
+forbiddenApisMain {
+    signaturesURLs = [PrecommitTasks.getResource('/forbidden/jdk-signatures.txt')]
+}
+
+if (isEclipse) {
+    // in eclipse the project is under a fake root, we need to change around the source sets
+    sourceSets {
+        if (project.path == ":libs:grok") {
+            main.java.srcDirs = ['java']
+            main.resources.srcDirs = ['resources']
+        } else {
+            test.java.srcDirs = ['java']
+            test.resources.srcDirs = ['resources']
+        }
+    }
+}
+
+thirdPartyAudit.excludes = [
+        // joni has AsmCompilerSupport, but that isn't being used:
+        'org.objectweb.asm.ClassWriter',
+        'org.objectweb.asm.MethodVisitor',
+        'org.objectweb.asm.Opcodes',
+]

libs/grok/src/main/eclipse-build.gradle

@@ -0,0 +1,3 @@
+
+// this is just shell gradle file for eclipse to have separate projects for grok src and tests
+apply from: '../../build.gradle'

libs/grok/src/main/java/org/elasticsearch/grok/Grok.java

@@ -17,7 +17,7 @@
  * under the License.
  */
 
-package org.elasticsearch.ingest.common;
+package org.elasticsearch.grok;
 
 import org.jcodings.specific.UTF8Encoding;
 import org.joni.Matcher;
@@ -28,13 +28,19 @@ import org.joni.Region;
 import org.joni.Syntax;
 import org.joni.exception.ValueException;
 
+import java.io.BufferedReader;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.InputStreamReader;
+import java.io.UncheckedIOException;
 import java.nio.charset.StandardCharsets;
 import java.util.HashMap;
 import java.util.Iterator;
 import java.util.Locale;
 import java.util.Map;
+import java.util.Collections;
 
-final class Grok {
+public final class Grok {
 
     private static final String NAME_GROUP = "name";
     private static final String SUBNAME_GROUP = "subname";
@@ -54,13 +60,24 @@ final class Grok {
             ")?" + "\\}";
     private static final Regex GROK_PATTERN_REGEX = new Regex(GROK_PATTERN.getBytes(StandardCharsets.UTF_8), 0,
             GROK_PATTERN.getBytes(StandardCharsets.UTF_8).length, Option.NONE, UTF8Encoding.INSTANCE, Syntax.DEFAULT);
+
+    private static final Map<String, String> builtinPatterns;
+
+    static {
+        try {
+            builtinPatterns = loadBuiltinPatterns();
+        } catch (IOException e) {
+            throw new UncheckedIOException("unable to load built-in grok patterns", e);
+        }
+    }
+
     private final Map<String, String> patternBank;
     private final boolean namedCaptures;
     private final Regex compiledExpression;
     private final String expression;
 
 
-    Grok(Map<String, String> patternBank, String grokPattern) {
+    public Grok(Map<String, String> patternBank, String grokPattern) {
         this(patternBank, grokPattern, true);
     }
 
@@ -176,5 +193,42 @@ final class Grok {
         }
         return null;
     }
+
+    public static Map<String, String> getBuiltinPatterns() {
+        return builtinPatterns;
+    }
+
+    private static Map<String, String> loadBuiltinPatterns() throws IOException {
+        // Code for loading built-in grok patterns packaged with the jar file:
+        String[] PATTERN_NAMES = new String[] {
+            "aws", "bacula", "bro", "exim", "firewalls", "grok-patterns", "haproxy",
+            "java", "junos", "linux-syslog", "mcollective-patterns", "mongodb", "nagios",
+            "postgresql", "rails", "redis", "ruby"
+        };
+        Map<String, String> builtinPatterns = new HashMap<>();
+        for (String pattern : PATTERN_NAMES) {
+            try(InputStream is = Grok.class.getResourceAsStream("/patterns/" + pattern)) {
+                loadPatterns(builtinPatterns, is);
+            }
+        }
+        return Collections.unmodifiableMap(builtinPatterns);
+    }
+
+    private static void loadPatterns(Map<String, String> patternBank, InputStream inputStream) throws IOException {
+        String line;
+        BufferedReader br = new BufferedReader(new InputStreamReader(inputStream, StandardCharsets.UTF_8));
+        while ((line = br.readLine()) != null) {
+            String trimmedLine = line.replaceAll("^\\s+", "");
+            if (trimmedLine.startsWith("#") || trimmedLine.length() == 0) {
+                continue;
+            }
+
+            String[] parts = trimmedLine.split("\\s+", 2);
+            if (parts.length == 2) {
+                patternBank.put(parts[0], parts[1]);
+            }
+        }
+    }
+
 }
 

libs/grok/src/main/java/org/elasticsearch/grok/GrokMatchGroup.java

@@ -17,7 +17,7 @@
  * under the License.
  */
 
-package org.elasticsearch.ingest.common;
+package org.elasticsearch.grok;
 
 final class GrokMatchGroup {
     private static final String DEFAULT_TYPE = "string";

libs/grok/src/test/eclipse-build.gradle

@@ -0,0 +1,7 @@
+
+// this is just shell gradle file for eclipse to have separate projects for grok src and tests
+apply from: '../../build.gradle'
+
+dependencies {
+  testCompile project(':libs:grok')
+}

libs/grok/src/test/java/org/elasticsearch/grok/GrokTests.java

@@ -17,7 +17,7 @@
  * under the License.
  */
 
-package org.elasticsearch.ingest.common;
+package org.elasticsearch.grok;
 
 import org.elasticsearch.test.ESTestCase;
 import org.junit.Before;
@@ -38,8 +38,8 @@ public class GrokTests extends ESTestCase {
     private Map<String, String> basePatterns;
 
     @Before
-    public void setup() throws IOException {
+    public void setup() {
-        basePatterns = IngestCommonPlugin.loadBuiltinPatterns();
+        basePatterns = Grok.getBuiltinPatterns();
     }
 
     public void testMatchWithoutCaptures() {

modules/ingest-common/build.gradle

@@ -23,17 +23,8 @@ esplugin {
 }
 
 dependencies {
-    compile 'org.jruby.joni:joni:2.1.6'
+    compile project(':libs:grok')
-    // joni dependencies:
-    compile 'org.jruby.jcodings:jcodings:1.0.12'
 }
 
 compileJava.options.compilerArgs << "-Xlint:-unchecked,-rawtypes"
 compileTestJava.options.compilerArgs << "-Xlint:-unchecked,-rawtypes"
-
-thirdPartyAudit.excludes = [
-        // joni has AsmCompilerSupport, but that isn't being used:
-        'org.objectweb.asm.ClassWriter',
-        'org.objectweb.asm.MethodVisitor',
-        'org.objectweb.asm.Opcodes',
-]

modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/GrokProcessor.java

@@ -19,6 +19,7 @@
 
 package org.elasticsearch.ingest.common;
 
+import org.elasticsearch.grok.Grok;
 import org.elasticsearch.ingest.AbstractProcessor;
 import org.elasticsearch.ingest.ConfigurationUtils;
 import org.elasticsearch.ingest.IngestDocument;
@@ -27,7 +28,6 @@ import org.elasticsearch.ingest.Processor;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
-import java.util.Objects;
 
 import static org.elasticsearch.ingest.ConfigurationUtils.newConfigurationException;
 
@@ -42,7 +42,7 @@ public final class GrokProcessor extends AbstractProcessor {
     private final boolean traceMatch;
     private final boolean ignoreMissing;
 
-    public GrokProcessor(String tag, Map<String, String> patternBank, List<String> matchPatterns, String matchField,
+    GrokProcessor(String tag, Map<String, String> patternBank, List<String> matchPatterns, String matchField,
                          boolean traceMatch, boolean ignoreMissing) {
         super(tag);
         this.matchField = matchField;

modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/IngestCommonPlugin.java

@@ -40,6 +40,7 @@ import org.elasticsearch.common.settings.ClusterSettings;
 import org.elasticsearch.common.settings.IndexScopedSettings;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.settings.SettingsFilter;
+import org.elasticsearch.grok.Grok;
 import org.elasticsearch.ingest.Processor;
 import org.elasticsearch.plugins.ActionPlugin;
 import org.elasticsearch.plugins.IngestPlugin;
@@ -49,22 +50,9 @@ import org.elasticsearch.rest.RestHandler;
 
 public class IngestCommonPlugin extends Plugin implements ActionPlugin, IngestPlugin {
 
-    // Code for loading built-in grok patterns packaged with the jar file:
+    static final Map<String, String> GROK_PATTERNS = Grok.getBuiltinPatterns();
-    private static final String[] PATTERN_NAMES = new String[] {
-        "aws", "bacula", "bro", "exim", "firewalls", "grok-patterns", "haproxy",
-        "java", "junos", "linux-syslog", "mcollective-patterns", "mongodb", "nagios",
-        "postgresql", "rails", "redis", "ruby"
-    };
-    static final Map<String, String> GROK_PATTERNS;
-    static {
-        try {
-            GROK_PATTERNS = loadBuiltinPatterns();
-        } catch (IOException e) {
-            throw new UncheckedIOException("unable to load built-in grok patterns", e);
-        }
-    }
 
-    public IngestCommonPlugin() throws IOException {
+    public IngestCommonPlugin() {
     }
 
     @Override
@@ -108,30 +96,4 @@ public class IngestCommonPlugin extends Plugin implements ActionPlugin, IngestPl
         return Arrays.asList(new GrokProcessorGetAction.RestAction(settings, restController));
     }
 
-
-    public static Map<String, String> loadBuiltinPatterns() throws IOException {
-        Map<String, String> builtinPatterns = new HashMap<>();
-        for (String pattern : PATTERN_NAMES) {
-            try(InputStream is = IngestCommonPlugin.class.getResourceAsStream("/patterns/" + pattern)) {
-                loadPatterns(builtinPatterns, is);
-            }
-        }
-        return Collections.unmodifiableMap(builtinPatterns);
-    }
-
-    private static void loadPatterns(Map<String, String> patternBank, InputStream inputStream) throws IOException {
-        String line;
-        BufferedReader br = new BufferedReader(new InputStreamReader(inputStream, StandardCharsets.UTF_8));
-        while ((line = br.readLine()) != null) {
-            String trimmedLine = line.replaceAll("^\\s+", "");
-            if (trimmedLine.startsWith("#") || trimmedLine.length() == 0) {
-                continue;
-            }
-
-            String[] parts = trimmedLine.split("\\s+", 2);
-            if (parts.length == 2) {
-                patternBank.put(parts[0], parts[1]);
-            }
-        }
-    }
 }

settings.gradle

@@ -81,6 +81,7 @@ if (isEclipse) {
   projects << 'libs:elasticsearch-core-tests'
   projects << 'libs:elasticsearch-nio-tests'
   projects << 'libs:secure-sm-tests'
+  projects << 'libs:grok-tests'
 }
 
 include projects.toArray(new String[0])
@@ -104,6 +105,10 @@ if (isEclipse) {
   project(":libs:secure-sm").buildFileName = 'eclipse-build.gradle'
   project(":libs:secure-sm-tests").projectDir = new File(rootProject.projectDir, 'libs/secure-sm/src/test')
   project(":libs:secure-sm-tests").buildFileName = 'eclipse-build.gradle'
+  project(":libs:grok").projectDir = new File(rootProject.projectDir, 'libs/grok/src/main')
+  project(":libs:grok").buildFileName = 'eclipse-build.gradle'
+  project(":libs:grok-tests").projectDir = new File(rootProject.projectDir, 'libs/grok/src/test')
+  project(":libs:grok-tests").buildFileName = 'eclipse-build.gradle'
 }
 
 // look for extra plugins for elasticsearch