Add Ingest-Processor specific Rest Endpoints & Add Grok endpoint (#25059)

This PR enables Ingest plugins to leverage processor-scoped REST
endpoints. First of which being the Grok endpoint that retrieves
Grok Patterns for users to retrieve all the built-in patterns.
Example usage: Kibana Grok Autocomplete!

docs/reference/ingest/ingest-node.asciidoc

@@ -1454,6 +1454,32 @@ second (index starts at zero) pattern in `patterns` to match.
 This trace metadata enables debugging which of the patterns matched. This information is stored in the ingest
 metadata and will not be indexed.
 
+[[grok-processor-rest-get]]
+==== Retrieving patterns from REST endpoint
+
+The Grok Processor comes packaged with its own REST endpoint for retrieving which patterns the processor is packaged with.
+
+[source,js]
+--------------------------------------------------
+GET _ingest/processor/grok
+--------------------------------------------------
+// CONSOLE
+
+The above request will return a response body containing a key-value representation of the built-in patterns dictionary.
+
+[source,js]
+--------------------------------------------------
+{
+  "patterns" : {
+    "BACULA_CAPACITY" : "%{INT}{1,3}(,%{INT}{3})*",
+    "PATH" : "(?:%{UNIXPATH}|%{WINPATH})",
+    ...
+}
+--------------------------------------------------
+// NOTCONSOLE
+
+This can be useful to reference as the built-in patterns change across versions.
+
 [[gsub-processor]]
 === Gsub Processor
 Converts a string field by applying a regular expression and a replacement.

modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/GrokProcessorGetAction.java

@@ -0,0 +1,162 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.ingest.common;
+
+import org.elasticsearch.action.Action;
+import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.action.ActionRequest;
+import org.elasticsearch.action.ActionRequestBuilder;
+import org.elasticsearch.action.ActionRequestValidationException;
+import org.elasticsearch.action.support.ActionFilters;
+import org.elasticsearch.action.support.HandledTransportAction;
+import org.elasticsearch.action.support.master.AcknowledgedResponse;
+import org.elasticsearch.client.ElasticsearchClient;
+import org.elasticsearch.client.node.NodeClient;
+import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
+import org.elasticsearch.common.inject.Inject;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.xcontent.ToXContent;
+import org.elasticsearch.common.xcontent.ToXContentObject;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.rest.BaseRestHandler;
+import org.elasticsearch.rest.BytesRestResponse;
+import org.elasticsearch.rest.RestController;
+import org.elasticsearch.rest.RestRequest;
+import org.elasticsearch.rest.RestResponse;
+import org.elasticsearch.rest.action.RestBuilderListener;
+import org.elasticsearch.threadpool.ThreadPool;
+import org.elasticsearch.transport.TransportService;
+
+import java.io.IOException;
+import java.util.Map;
+
+import static org.elasticsearch.ingest.common.IngestCommonPlugin.GROK_PATTERNS;
+import static org.elasticsearch.rest.RestRequest.Method.GET;
+import static org.elasticsearch.rest.RestStatus.OK;
+
+public class GrokProcessorGetAction extends Action<GrokProcessorGetAction.Request,
+    GrokProcessorGetAction.Response, GrokProcessorGetAction.RequestBuilder> {
+
+    public static final GrokProcessorGetAction INSTANCE = new GrokProcessorGetAction();
+    public static final String NAME = "cluster:admin/ingest/processor/grok/get";
+
+    private GrokProcessorGetAction() {
+        super(NAME);
+    }
+
+    @Override
+    public RequestBuilder newRequestBuilder(ElasticsearchClient client) {
+        return new RequestBuilder(client);
+    }
+
+    @Override
+    public Response newResponse() {
+        return new Response(null);
+    }
+
+    public static class Request extends ActionRequest {
+        @Override
+        public ActionRequestValidationException validate() {
+            return null;
+        }
+    }
+
+    public static class RequestBuilder extends ActionRequestBuilder<Request, Response, RequestBuilder> {
+        public RequestBuilder(ElasticsearchClient client) {
+            super(client, GrokProcessorGetAction.INSTANCE, new Request());
+        }
+    }
+
+    public static class Response extends AcknowledgedResponse implements ToXContentObject {
+        private Map<String, String> grokPatterns;
+
+        public Response(Map<String, String> grokPatterns) {
+            this.grokPatterns = grokPatterns;
+        }
+
+        public Map<String, String> getGrokPatterns() {
+            return grokPatterns;
+        }
+
+        @Override
+        public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+            builder.startObject();
+            builder.field("patterns");
+            builder.map(grokPatterns);
+            builder.endObject();
+            return builder;
+        }
+
+        @Override
+        public void readFrom(StreamInput in) throws IOException {
+            super.readFrom(in);
+            grokPatterns = in.readMap(StreamInput::readString, StreamInput::readString);
+        }
+
+        @Override
+        public void writeTo(StreamOutput out) throws IOException {
+            super.writeTo(out);
+            out.writeMap(grokPatterns, StreamOutput::writeString, StreamOutput::writeString);
+        }
+    }
+
+    public static class TransportAction extends HandledTransportAction<Request, Response> {
+
+        @Inject
+        public TransportAction(Settings settings, ThreadPool threadPool, TransportService transportService,
+                               ActionFilters actionFilters, IndexNameExpressionResolver indexNameExpressionResolver) {
+            super(settings, NAME, threadPool, transportService, actionFilters,
+                indexNameExpressionResolver, Request::new);
+        }
+
+        @Override
+        protected void doExecute(Request request, ActionListener<Response> listener) {
+            try {
+                listener.onResponse(new Response(GROK_PATTERNS));
+            } catch (Exception e) {
+                listener.onFailure(e);
+            }
+        }
+    }
+
+    public static class RestAction extends BaseRestHandler {
+        public RestAction(Settings settings, RestController controller) {
+            super(settings);
+            controller.registerHandler(GET, "/_ingest/processor/grok", this);
+        }
+
+        @Override
+        public String getName() {
+            return "ingest_processor_grok_get";
+        }
+
+        @Override
+        protected RestChannelConsumer prepareRequest(RestRequest request, NodeClient client) throws IOException {
+            return channel -> client.executeLocally(INSTANCE, new Request(), new RestBuilderListener<Response>(channel) {
+                @Override
+                public RestResponse buildResponse(Response response, XContentBuilder builder) throws Exception {
+                    response.toXContent(builder, ToXContent.EMPTY_PARAMS);
+                    return new BytesRestResponse(OK, builder);
+                }
+            });
+        }
+    }
+}

modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/IngestCommonPlugin.java

@@ -23,21 +23,48 @@ import java.io.BufferedReader;
 import java.io.IOException;
 import java.io.InputStream;
 import java.io.InputStreamReader;
+import java.io.UncheckedIOException;
 import java.nio.charset.StandardCharsets;
+import java.util.Arrays;
 import java.util.Collections;
 import java.util.HashMap;
+import java.util.List;
 import java.util.Map;
+import java.util.function.Supplier;
 
+import org.elasticsearch.action.ActionRequest;
+import org.elasticsearch.action.ActionResponse;
+import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
+import org.elasticsearch.cluster.node.DiscoveryNodes;
+import org.elasticsearch.common.settings.ClusterSettings;
+import org.elasticsearch.common.settings.IndexScopedSettings;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.settings.SettingsFilter;
 import org.elasticsearch.ingest.Processor;
+import org.elasticsearch.plugins.ActionPlugin;
 import org.elasticsearch.plugins.IngestPlugin;
 import org.elasticsearch.plugins.Plugin;
+import org.elasticsearch.rest.RestController;
+import org.elasticsearch.rest.RestHandler;
 
-public class IngestCommonPlugin extends Plugin implements IngestPlugin {
+public class IngestCommonPlugin extends Plugin implements ActionPlugin, IngestPlugin {
 
-    private final Map<String, String> builtinPatterns;
+    // Code for loading built-in grok patterns packaged with the jar file:
+    private static final String[] PATTERN_NAMES = new String[] {
+        "aws", "bacula", "bro", "exim", "firewalls", "grok-patterns", "haproxy",
+        "java", "junos", "linux-syslog", "mcollective-patterns", "mongodb", "nagios",
+        "postgresql", "rails", "redis", "ruby"
+    };
+    static final Map<String, String> GROK_PATTERNS;
+    static {
+        try {
+            GROK_PATTERNS = loadBuiltinPatterns();
+        } catch (IOException e) {
+            throw new UncheckedIOException("unable to load built-in grok patterns", e);
+        }
+    }
 
     public IngestCommonPlugin() throws IOException {
-        this.builtinPatterns = loadBuiltinPatterns();
     }
 
     @Override
@@ -59,7 +86,7 @@ public class IngestCommonPlugin extends Plugin implements IngestPlugin {
         processors.put(ForEachProcessor.TYPE, new ForEachProcessor.Factory());
         processors.put(DateIndexNameProcessor.TYPE, new DateIndexNameProcessor.Factory());
         processors.put(SortProcessor.TYPE, new SortProcessor.Factory());
-        processors.put(GrokProcessor.TYPE, new GrokProcessor.Factory(builtinPatterns));
+        processors.put(GrokProcessor.TYPE, new GrokProcessor.Factory(GROK_PATTERNS));
         processors.put(ScriptProcessor.TYPE, new ScriptProcessor.Factory(parameters.scriptService));
         processors.put(DotExpanderProcessor.TYPE, new DotExpanderProcessor.Factory());
         processors.put(JsonProcessor.TYPE, new JsonProcessor.Factory());
@@ -67,13 +94,19 @@ public class IngestCommonPlugin extends Plugin implements IngestPlugin {
         return Collections.unmodifiableMap(processors);
     }
 
-    // Code for loading built-in grok patterns packaged with the jar file:
+    @Override
+    public List<ActionHandler<? extends ActionRequest, ? extends ActionResponse>> getActions() {
+        return Arrays.asList(new ActionHandler<>(GrokProcessorGetAction.INSTANCE, GrokProcessorGetAction.TransportAction.class));
+    }
+
+    @Override
+    public List<RestHandler> getRestHandlers(Settings settings, RestController restController, ClusterSettings clusterSettings,
+                                             IndexScopedSettings indexScopedSettings, SettingsFilter settingsFilter,
+                                             IndexNameExpressionResolver indexNameExpressionResolver,
+                                             Supplier<DiscoveryNodes> nodesInCluster) {
+        return Arrays.asList(new GrokProcessorGetAction.RestAction(settings, restController));
+    }
 
-    private static final String[] PATTERN_NAMES = new String[] {
-            "aws", "bacula", "bro", "exim", "firewalls", "grok-patterns", "haproxy",
-            "java", "junos", "linux-syslog", "mcollective-patterns", "mongodb", "nagios",
-            "postgresql", "rails", "redis", "ruby"
-    };
 
     public static Map<String, String> loadBuiltinPatterns() throws IOException {
         Map<String, String> builtinPatterns = new HashMap<>();

modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/GrokProcessorGetActionTests.java

@@ -0,0 +1,72 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.ingest.common;
+
+import org.elasticsearch.common.io.stream.BytesStreamOutput;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.xcontent.ToXContent;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentHelper;
+import org.elasticsearch.common.xcontent.json.JsonXContent;
+import org.elasticsearch.test.ESTestCase;
+
+import java.util.Collections;
+import java.util.Map;
+
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.core.IsNull.nullValue;
+
+
+public class GrokProcessorGetActionTests extends ESTestCase {
+    private static final Map<String, String> TEST_PATTERNS = Collections.singletonMap("PATTERN", "foo");
+
+    public void testRequest() throws Exception {
+        GrokProcessorGetAction.Request request = new GrokProcessorGetAction.Request();
+        BytesStreamOutput out = new BytesStreamOutput();
+        request.writeTo(out);
+        StreamInput streamInput = out.bytes().streamInput();
+        GrokProcessorGetAction.Request otherRequest = new GrokProcessorGetAction.Request();
+        otherRequest.readFrom(streamInput);
+        assertThat(otherRequest.validate(), nullValue());
+    }
+
+    public void testResponseSerialization() throws Exception {
+        GrokProcessorGetAction.Response response = new GrokProcessorGetAction.Response(TEST_PATTERNS);
+        BytesStreamOutput out = new BytesStreamOutput();
+        response.writeTo(out);
+        StreamInput streamInput = out.bytes().streamInput();
+        GrokProcessorGetAction.Response otherResponse = new GrokProcessorGetAction.Response(null);
+        otherResponse.readFrom(streamInput);
+        assertThat(response.getGrokPatterns(), equalTo(TEST_PATTERNS));
+        assertThat(response.getGrokPatterns(), equalTo(otherResponse.getGrokPatterns()));
+    }
+
+    @SuppressWarnings("unchecked")
+    public void testResponseToXContent() throws Exception {
+        GrokProcessorGetAction.Response response = new GrokProcessorGetAction.Response(TEST_PATTERNS);
+        try (XContentBuilder builder = JsonXContent.contentBuilder()) {
+            response.toXContent(builder, ToXContent.EMPTY_PARAMS);
+            Map<String, Object> converted = XContentHelper.convertToMap(builder.bytes(), false, builder.contentType()).v2();
+            Map<String, String> patterns = (Map<String, String>) converted.get("patterns");
+            assertThat(patterns.size(), equalTo(1));
+            assertThat(patterns.get("PATTERN"), equalTo("foo"));
+        }
+    }
+}

modules/ingest-common/src/test/resources/rest-api-spec/test/ingest/120_grok.yml

@@ -154,3 +154,10 @@ teardown:
   - length: { docs.0.doc._ingest: 2 }
   - match: { docs.0.doc._ingest._grok_match_index: "1" }
   - is_true: docs.0.doc._ingest.timestamp
+
+---
+"Test Grok Patterns Retrieval":
+  - do:
+      ingest.processor.grok: {}
+  - length: { patterns: 303 }
+  - match: { patterns.PATH: "(?:%{UNIXPATH}|%{WINPATH})" }

rest-api-spec/src/main/resources/rest-api-spec/api/ingest.processor.grok.json

@@ -0,0 +1,15 @@
+{
+  "ingest.processor.grok": {
+    "documentation": "https://www.elastic.co/guide/en/elasticsearch/plugins/master/ingest.html",
+    "methods": [ "GET" ],
+    "url": {
+      "path": "/_ingest/processor/grok",
+      "paths": ["/_ingest/processor/grok"],
+      "parts": {
+      },
+      "params": {
+      }
+    },
+    "body": null
+  }
+}