Moved the grok processor to its own module, so that it will available out-of-the-box, while its dependencies are isolated

modules/ingest-grok/build.gradle

@@ -0,0 +1,39 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+esplugin {
+    description 'Ingest processor that uses grok patterns to split text'
+    classname 'org.elasticsearch.ingest.grok.IngestGrokPlugin'
+}
+
+dependencies {
+    compile 'org.jruby.joni:joni:2.1.6'
+    // joni dependencies:
+    compile 'org.jruby.jcodings:jcodings:1.0.12'
+}
+
+compileJava.options.compilerArgs << "-Xlint:-rawtypes,-unchecked,-serial"
+compileTestJava.options.compilerArgs << "-Xlint:-rawtypes,-unchecked"
+
+thirdPartyAudit.excludes = [
+        // joni has AsmCompilerSupport, but that isn't being used:
+        'org.objectweb.asm.ClassWriter',
+        'org.objectweb.asm.MethodVisitor',
+        'org.objectweb.asm.Opcodes',
+]

modules/ingest-grok/licenses/jcodings-1.0.12.jar.sha1

@@ -0,0 +1 @@
+6bc17079fcaa8823ea8cd0d4c66516335b558db8

modules/ingest-grok/licenses/joni-2.1.6.jar.sha1

@@ -0,0 +1 @@
+0f23c95a06eaecbc8c74c7458a8bfd13e4fd2d3a

modules/ingest-grok/src/main/java/org/elasticsearch/ingest/grok/Grok.java

@@ -17,7 +17,7 @@
  * under the License.
  */
 
-package org.elasticsearch.ingest.processor;
+package org.elasticsearch.ingest.grok;
 
 import org.jcodings.specific.UTF8Encoding;
 import org.joni.Matcher;

modules/ingest-grok/src/main/java/org/elasticsearch/ingest/grok/GrokMatchGroup.java

@@ -17,7 +17,7 @@
  * under the License.
  */
 
-package org.elasticsearch.ingest.processor;
+package org.elasticsearch.ingest.grok;
 
 final class GrokMatchGroup {
     private static final String DEFAULT_TYPE = "string";

modules/ingest-grok/src/main/java/org/elasticsearch/ingest/grok/GrokProcessor.java

@@ -17,7 +17,7 @@
  * under the License.
  */
 
-package org.elasticsearch.ingest.processor;
+package org.elasticsearch.ingest.grok;
 
 import org.elasticsearch.ingest.core.IngestDocument;
 import org.elasticsearch.ingest.core.ConfigurationUtils;
@@ -28,10 +28,7 @@ import java.io.IOException;
 import java.io.InputStream;
 import java.io.InputStreamReader;
 import java.nio.charset.StandardCharsets;
-import java.nio.file.DirectoryStream;
-import java.nio.file.Files;
-import java.nio.file.Path;
-import java.nio.file.StandardOpenOption;
+import java.util.Collections;
 import java.util.HashMap;
 import java.util.Map;
 
@@ -71,12 +68,23 @@ public final class GrokProcessor implements Processor {
         return grok;
     }
 
-    public static class Factory implements Processor.Factory<GrokProcessor> {
+    public final static class Factory implements Processor.Factory<GrokProcessor> {
 
-        private final Path grokConfigDirectory;
+        private final static String[] PATTERN_NAMES = new String[] {
+            "aws", "bacula", "bro", "exim", "firewalls", "grok-patterns", "haproxy",
+            "java", "junos", "linux-syslog", "mcollective-patterns", "mongodb", "nagios",
+            "postgresql", "rails", "redis", "ruby"
+        };
+        private final Map<String, String> builtinPatternBank;
 
-        public Factory(Path configDirectory) {
-            this.grokConfigDirectory = configDirectory.resolve("ingest").resolve("grok");
+        public Factory() throws IOException {
+            Map<String, String> builtinPatterns = new HashMap<>();
+            for (String pattern : PATTERN_NAMES) {
+                try(InputStream is = getClass().getResourceAsStream("/patterns/" + pattern)) {
+                    loadBankFromStream(builtinPatterns, is);
+                }
+            }
+            this.builtinPatternBank = Collections.unmodifiableMap(builtinPatterns);
         }
 
         static void loadBankFromStream(Map<String, String> patternBank, InputStream inputStream) throws IOException {
@@ -99,20 +107,7 @@ public final class GrokProcessor implements Processor {
             String matchField = ConfigurationUtils.readStringProperty(config, "field");
             String matchPattern = ConfigurationUtils.readStringProperty(config, "pattern");
             Map<String, String> customPatternBank = ConfigurationUtils.readOptionalMap(config, "pattern_definitions");
-
-            Map<String, String> patternBank = new HashMap<>();
-
-            Path patternsDirectory = grokConfigDirectory.resolve("patterns");
-            try (DirectoryStream<Path> stream = Files.newDirectoryStream(patternsDirectory)) {
-                for (Path patternFilePath : stream) {
-                    if (Files.isRegularFile(patternFilePath)) {
-                        try(InputStream is = Files.newInputStream(patternFilePath, StandardOpenOption.READ)) {
-                            loadBankFromStream(patternBank, is);
-                        }
-                    }
-                }
-            }
-
+            Map<String, String> patternBank = new HashMap<>(builtinPatternBank);
             if (customPatternBank != null) {
                 patternBank.putAll(customPatternBank);
             }

modules/ingest-grok/src/main/java/org/elasticsearch/ingest/grok/IngestGrokPlugin.java

@@ -0,0 +1,48 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.ingest.grok;
+
+import org.elasticsearch.ingest.IngestModule;
+import org.elasticsearch.plugins.Plugin;
+
+import java.io.IOException;
+
+public class IngestGrokPlugin extends Plugin {
+
+    @Override
+    public String name() {
+        return "ingest-grok";
+    }
+
+    @Override
+    public String description() {
+        return "Ingest processor that uses grok patterns to split text";
+    }
+
+    public void onModule(IngestModule ingestModule) {
+        ingestModule.registerProcessor(GrokProcessor.TYPE, (environment, templateService) -> {
+            try {
+                return new GrokProcessor.Factory();
+            } catch (IOException e) {
+                throw new RuntimeException(e);
+            }
+        });
+    }
+}

modules/ingest-grok/src/test/java/org/elasticsearch/ingest/grok/GrokProcessorFactoryTests.java

@@ -17,14 +17,11 @@
  * under the License.
  */
 
-package org.elasticsearch.ingest.processor;
+package org.elasticsearch.ingest.grok;
 
-import org.elasticsearch.ingest.processor.GrokProcessor;
+import org.elasticsearch.ingest.grok.GrokProcessor;
 import org.elasticsearch.test.ESTestCase;
-import org.junit.Before;
 
-import java.nio.file.Files;
-import java.nio.file.Path;
 import java.util.Collections;
 import java.util.HashMap;
 import java.util.Map;
@@ -34,18 +31,8 @@ import static org.hamcrest.Matchers.notNullValue;
 
 public class GrokProcessorFactoryTests extends ESTestCase {
 
-    private Path configDir;
-
-    @Before
-    public void prepareConfigDirectory() throws Exception {
-        this.configDir = createTempDir();
-        Path grokDir = configDir.resolve("ingest").resolve("grok");
-        Path patternsDir = grokDir.resolve("patterns");
-        Files.createDirectories(patternsDir);
-    }
-
     public void testBuild() throws Exception {
-        GrokProcessor.Factory factory = new GrokProcessor.Factory(configDir);
+        GrokProcessor.Factory factory = new GrokProcessor.Factory();
 
         Map<String, Object> config = new HashMap<>();
         config.put("field", "_field");
@@ -56,7 +43,7 @@ public class GrokProcessorFactoryTests extends ESTestCase {
     }
 
     public void testCreateWithCustomPatterns() throws Exception {
-        GrokProcessor.Factory factory = new GrokProcessor.Factory(configDir);
+        GrokProcessor.Factory factory = new GrokProcessor.Factory();
 
         Map<String, Object> config = new HashMap<>();
         config.put("field", "_field");

modules/ingest-grok/src/test/java/org/elasticsearch/ingest/grok/GrokProcessorTests.java

@@ -17,10 +17,12 @@
  * under the License.
  */
 
-package org.elasticsearch.ingest.processor;
+package org.elasticsearch.ingest.grok;
 
-import org.elasticsearch.ingest.core.IngestDocument;
 import org.elasticsearch.ingest.RandomDocumentPicks;
+import org.elasticsearch.ingest.core.IngestDocument;
+import org.elasticsearch.ingest.grok.Grok;
+import org.elasticsearch.ingest.grok.GrokProcessor;
 import org.elasticsearch.test.ESTestCase;
 
 import java.util.Collections;

modules/ingest-grok/src/test/java/org/elasticsearch/ingest/grok/GrokTests.java

@@ -17,10 +17,10 @@
  * under the License.
  */
 
-package org.elasticsearch.ingest.processor;
+package org.elasticsearch.ingest.grok;
 
-import org.elasticsearch.ingest.processor.Grok;
-import org.elasticsearch.ingest.processor.GrokProcessor;
+import org.elasticsearch.ingest.grok.Grok;
+import org.elasticsearch.ingest.grok.GrokProcessor;
 import org.elasticsearch.test.ESTestCase;
 import org.junit.Before;
 
@@ -52,8 +52,8 @@ public class GrokTests extends ESTestCase {
     @Before
     public void setup() throws IOException {
         basePatterns = newBankFromStreams(
-                getClass().getResourceAsStream("/grok/patterns/grok-patterns"),
-                getClass().getResourceAsStream("/grok/patterns/linux-syslog")
+                getClass().getResourceAsStream("/patterns/grok-patterns"),
+                getClass().getResourceAsStream("/patterns/linux-syslog")
         );
     }
 

modules/ingest-grok/src/test/java/org/elasticsearch/ingest/grok/IngestGrokRestIT.java

@@ -0,0 +1,49 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.ingest.grok;
+
+import com.carrotsearch.randomizedtesting.annotations.Name;
+import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;
+import org.elasticsearch.ingest.grok.IngestGrokPlugin;
+import org.elasticsearch.plugins.Plugin;
+import org.elasticsearch.test.rest.ESRestTestCase;
+import org.elasticsearch.test.rest.RestTestCandidate;
+import org.elasticsearch.test.rest.parser.RestTestParseException;
+
+import java.io.IOException;
+import java.util.Collection;
+
+public class IngestGrokRestIT extends ESRestTestCase {
+
+    @Override
+    protected Collection<Class<? extends Plugin>> nodePlugins() {
+        return pluginList(IngestGrokPlugin.class);
+    }
+
+    public IngestGrokRestIT(@Name("yaml") RestTestCandidate testCandidate) {
+        super(testCandidate);
+    }
+
+    @ParametersFactory
+    public static Iterable<Object[]> parameters() throws IOException, RestTestParseException {
+        return ESRestTestCase.createParameters(0, 1);
+    }
+}
+

modules/ingest-grok/src/test/resources/rest-api-spec/test/ingest_grok/10_basic.yaml

@@ -0,0 +1,12 @@
+"Ingest grok installed":
+    - do:
+        cluster.state: {}
+
+    # Get master node id
+    - set: { master_node: master }
+
+    - do:
+        nodes.info: {}
+
+    - match:  { nodes.$master.modules.0.name: ingest-grok  }
+    - match:  { nodes.$master.modules.0.jvm: true  }

plugins/ingest/build.gradle

@@ -23,10 +23,6 @@ esplugin {
 }
 
 dependencies {
-  compile 'org.jruby.joni:joni:2.1.6'
-  // joni dependencies:
-  compile 'org.jruby.jcodings:jcodings:1.0.12'
-
   compile ('com.maxmind.geoip2:geoip2:2.4.0')
   // geoip2 dependencies:
   compile('com.fasterxml.jackson.core:jackson-annotations:2.5.0')
@@ -72,8 +68,4 @@ thirdPartyAudit.excludes = [
   'com.google.api.client.http.HttpResponseException',
   'com.google.api.client.http.javanet.NetHttpTransport',
   'com.google.api.client.http.javanet.NetHttpTransport',
-  // joni has AsmCompilerSupport, but that isn't being used:
-  'org.objectweb.asm.ClassWriter',
-  'org.objectweb.asm.MethodVisitor',
-  'org.objectweb.asm.Opcodes',
 ]

plugins/ingest/licenses/jcodings-1.0.12.jar.sha1

@@ -1 +0,0 @@
-6bc17079fcaa8823ea8cd0d4c66516335b558db8

plugins/ingest/licenses/joni-2.1.6.jar.sha1

@@ -1 +0,0 @@
-0f23c95a06eaecbc8c74c7458a8bfd13e4fd2d3a

plugins/ingest/src/main/java/org/elasticsearch/plugin/ingest/IngestPlugin.java

@@ -21,7 +21,6 @@ package org.elasticsearch.plugin.ingest;
 
 import org.elasticsearch.ingest.IngestModule;
 import org.elasticsearch.ingest.processor.GeoIpProcessor;
-import org.elasticsearch.ingest.processor.GrokProcessor;
 import org.elasticsearch.plugins.Plugin;
 
 public class IngestPlugin extends Plugin {
@@ -40,6 +39,5 @@ public class IngestPlugin extends Plugin {
 
     public void onModule(IngestModule ingestModule) {
         ingestModule.registerProcessor(GeoIpProcessor.TYPE, (environment, templateService) -> new GeoIpProcessor.Factory(environment.configFile()));
-        ingestModule.registerProcessor(GrokProcessor.TYPE, (environment, templateService) -> new GrokProcessor.Factory(environment.configFile()));
     }
 }

settings.gradle

@@ -14,6 +14,7 @@ List projects = [
   'modules:lang-expression',
   'modules:lang-groovy',
   'modules:lang-mustache',
+  'modules:ingest-grok',
   'plugins:analysis-icu',
   'plugins:analysis-kuromoji',
   'plugins:analysis-phonetic',