From a771912a22bd46d9598232a74ed991d71f0f1cfc Mon Sep 17 00:00:00 2001 From: Tal Levy Date: Thu, 8 Jun 2017 15:24:35 -0700 Subject: [PATCH] Add Ingest-Processor specific Rest Endpoints & Add Grok endpoint (#25059) This PR enables Ingest plugins to leverage processor-scoped REST endpoints. First of which being the Grok endpoint that retrieves Grok Patterns for users to retrieve all the built-in patterns. Example usage: Kibana Grok Autocomplete! --- docs/reference/ingest/ingest-node.asciidoc | 26 +++ .../ingest/common/GrokProcessorGetAction.java | 162 ++++++++++++++++++ .../ingest/common/IngestCommonPlugin.java | 53 ++++-- .../common/GrokProcessorGetActionTests.java | 72 ++++++++ .../rest-api-spec/test/ingest/120_grok.yml | 7 + .../api/ingest.processor.grok.json | 15 ++ 6 files changed, 325 insertions(+), 10 deletions(-) create mode 100644 modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/GrokProcessorGetAction.java create mode 100644 modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/GrokProcessorGetActionTests.java create mode 100644 rest-api-spec/src/main/resources/rest-api-spec/api/ingest.processor.grok.json diff --git a/docs/reference/ingest/ingest-node.asciidoc b/docs/reference/ingest/ingest-node.asciidoc index 076545cdd25..4c96ee9c6d6 100644 --- a/docs/reference/ingest/ingest-node.asciidoc +++ b/docs/reference/ingest/ingest-node.asciidoc @@ -1454,6 +1454,32 @@ second (index starts at zero) pattern in `patterns` to match. This trace metadata enables debugging which of the patterns matched. This information is stored in the ingest metadata and will not be indexed. +[[grok-processor-rest-get]] +==== Retrieving patterns from REST endpoint + +The Grok Processor comes packaged with its own REST endpoint for retrieving which patterns the processor is packaged with. + +[source,js] +-------------------------------------------------- +GET _ingest/processor/grok +-------------------------------------------------- +// CONSOLE + +The above request will return a response body containing a key-value representation of the built-in patterns dictionary. + +[source,js] +-------------------------------------------------- +{ + "patterns" : { + "BACULA_CAPACITY" : "%{INT}{1,3}(,%{INT}{3})*", + "PATH" : "(?:%{UNIXPATH}|%{WINPATH})", + ... +} +-------------------------------------------------- +// NOTCONSOLE + +This can be useful to reference as the built-in patterns change across versions. + [[gsub-processor]] === Gsub Processor Converts a string field by applying a regular expression and a replacement. diff --git a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/GrokProcessorGetAction.java b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/GrokProcessorGetAction.java new file mode 100644 index 00000000000..45ffcc53a4d --- /dev/null +++ b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/GrokProcessorGetAction.java @@ -0,0 +1,162 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.elasticsearch.ingest.common; + +import org.elasticsearch.action.Action; +import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.ActionRequest; +import org.elasticsearch.action.ActionRequestBuilder; +import org.elasticsearch.action.ActionRequestValidationException; +import org.elasticsearch.action.support.ActionFilters; +import org.elasticsearch.action.support.HandledTransportAction; +import org.elasticsearch.action.support.master.AcknowledgedResponse; +import org.elasticsearch.client.ElasticsearchClient; +import org.elasticsearch.client.node.NodeClient; +import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; +import org.elasticsearch.common.inject.Inject; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.xcontent.ToXContent; +import org.elasticsearch.common.xcontent.ToXContentObject; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.rest.BaseRestHandler; +import org.elasticsearch.rest.BytesRestResponse; +import org.elasticsearch.rest.RestController; +import org.elasticsearch.rest.RestRequest; +import org.elasticsearch.rest.RestResponse; +import org.elasticsearch.rest.action.RestBuilderListener; +import org.elasticsearch.threadpool.ThreadPool; +import org.elasticsearch.transport.TransportService; + +import java.io.IOException; +import java.util.Map; + +import static org.elasticsearch.ingest.common.IngestCommonPlugin.GROK_PATTERNS; +import static org.elasticsearch.rest.RestRequest.Method.GET; +import static org.elasticsearch.rest.RestStatus.OK; + +public class GrokProcessorGetAction extends Action { + + public static final GrokProcessorGetAction INSTANCE = new GrokProcessorGetAction(); + public static final String NAME = "cluster:admin/ingest/processor/grok/get"; + + private GrokProcessorGetAction() { + super(NAME); + } + + @Override + public RequestBuilder newRequestBuilder(ElasticsearchClient client) { + return new RequestBuilder(client); + } + + @Override + public Response newResponse() { + return new Response(null); + } + + public static class Request extends ActionRequest { + @Override + public ActionRequestValidationException validate() { + return null; + } + } + + public static class RequestBuilder extends ActionRequestBuilder { + public RequestBuilder(ElasticsearchClient client) { + super(client, GrokProcessorGetAction.INSTANCE, new Request()); + } + } + + public static class Response extends AcknowledgedResponse implements ToXContentObject { + private Map grokPatterns; + + public Response(Map grokPatterns) { + this.grokPatterns = grokPatterns; + } + + public Map getGrokPatterns() { + return grokPatterns; + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject(); + builder.field("patterns"); + builder.map(grokPatterns); + builder.endObject(); + return builder; + } + + @Override + public void readFrom(StreamInput in) throws IOException { + super.readFrom(in); + grokPatterns = in.readMap(StreamInput::readString, StreamInput::readString); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + super.writeTo(out); + out.writeMap(grokPatterns, StreamOutput::writeString, StreamOutput::writeString); + } + } + + public static class TransportAction extends HandledTransportAction { + + @Inject + public TransportAction(Settings settings, ThreadPool threadPool, TransportService transportService, + ActionFilters actionFilters, IndexNameExpressionResolver indexNameExpressionResolver) { + super(settings, NAME, threadPool, transportService, actionFilters, + indexNameExpressionResolver, Request::new); + } + + @Override + protected void doExecute(Request request, ActionListener listener) { + try { + listener.onResponse(new Response(GROK_PATTERNS)); + } catch (Exception e) { + listener.onFailure(e); + } + } + } + + public static class RestAction extends BaseRestHandler { + public RestAction(Settings settings, RestController controller) { + super(settings); + controller.registerHandler(GET, "/_ingest/processor/grok", this); + } + + @Override + public String getName() { + return "ingest_processor_grok_get"; + } + + @Override + protected RestChannelConsumer prepareRequest(RestRequest request, NodeClient client) throws IOException { + return channel -> client.executeLocally(INSTANCE, new Request(), new RestBuilderListener(channel) { + @Override + public RestResponse buildResponse(Response response, XContentBuilder builder) throws Exception { + response.toXContent(builder, ToXContent.EMPTY_PARAMS); + return new BytesRestResponse(OK, builder); + } + }); + } + } +} diff --git a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/IngestCommonPlugin.java b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/IngestCommonPlugin.java index 8c6ca8e919e..76f2b8c2f16 100644 --- a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/IngestCommonPlugin.java +++ b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/IngestCommonPlugin.java @@ -23,21 +23,48 @@ import java.io.BufferedReader; import java.io.IOException; import java.io.InputStream; import java.io.InputStreamReader; +import java.io.UncheckedIOException; import java.nio.charset.StandardCharsets; +import java.util.Arrays; import java.util.Collections; import java.util.HashMap; +import java.util.List; import java.util.Map; +import java.util.function.Supplier; +import org.elasticsearch.action.ActionRequest; +import org.elasticsearch.action.ActionResponse; +import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; +import org.elasticsearch.cluster.node.DiscoveryNodes; +import org.elasticsearch.common.settings.ClusterSettings; +import org.elasticsearch.common.settings.IndexScopedSettings; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.settings.SettingsFilter; import org.elasticsearch.ingest.Processor; +import org.elasticsearch.plugins.ActionPlugin; import org.elasticsearch.plugins.IngestPlugin; import org.elasticsearch.plugins.Plugin; +import org.elasticsearch.rest.RestController; +import org.elasticsearch.rest.RestHandler; -public class IngestCommonPlugin extends Plugin implements IngestPlugin { +public class IngestCommonPlugin extends Plugin implements ActionPlugin, IngestPlugin { - private final Map builtinPatterns; + // Code for loading built-in grok patterns packaged with the jar file: + private static final String[] PATTERN_NAMES = new String[] { + "aws", "bacula", "bro", "exim", "firewalls", "grok-patterns", "haproxy", + "java", "junos", "linux-syslog", "mcollective-patterns", "mongodb", "nagios", + "postgresql", "rails", "redis", "ruby" + }; + static final Map GROK_PATTERNS; + static { + try { + GROK_PATTERNS = loadBuiltinPatterns(); + } catch (IOException e) { + throw new UncheckedIOException("unable to load built-in grok patterns", e); + } + } public IngestCommonPlugin() throws IOException { - this.builtinPatterns = loadBuiltinPatterns(); } @Override @@ -59,7 +86,7 @@ public class IngestCommonPlugin extends Plugin implements IngestPlugin { processors.put(ForEachProcessor.TYPE, new ForEachProcessor.Factory()); processors.put(DateIndexNameProcessor.TYPE, new DateIndexNameProcessor.Factory()); processors.put(SortProcessor.TYPE, new SortProcessor.Factory()); - processors.put(GrokProcessor.TYPE, new GrokProcessor.Factory(builtinPatterns)); + processors.put(GrokProcessor.TYPE, new GrokProcessor.Factory(GROK_PATTERNS)); processors.put(ScriptProcessor.TYPE, new ScriptProcessor.Factory(parameters.scriptService)); processors.put(DotExpanderProcessor.TYPE, new DotExpanderProcessor.Factory()); processors.put(JsonProcessor.TYPE, new JsonProcessor.Factory()); @@ -67,13 +94,19 @@ public class IngestCommonPlugin extends Plugin implements IngestPlugin { return Collections.unmodifiableMap(processors); } - // Code for loading built-in grok patterns packaged with the jar file: + @Override + public List> getActions() { + return Arrays.asList(new ActionHandler<>(GrokProcessorGetAction.INSTANCE, GrokProcessorGetAction.TransportAction.class)); + } + + @Override + public List getRestHandlers(Settings settings, RestController restController, ClusterSettings clusterSettings, + IndexScopedSettings indexScopedSettings, SettingsFilter settingsFilter, + IndexNameExpressionResolver indexNameExpressionResolver, + Supplier nodesInCluster) { + return Arrays.asList(new GrokProcessorGetAction.RestAction(settings, restController)); + } - private static final String[] PATTERN_NAMES = new String[] { - "aws", "bacula", "bro", "exim", "firewalls", "grok-patterns", "haproxy", - "java", "junos", "linux-syslog", "mcollective-patterns", "mongodb", "nagios", - "postgresql", "rails", "redis", "ruby" - }; public static Map loadBuiltinPatterns() throws IOException { Map builtinPatterns = new HashMap<>(); diff --git a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/GrokProcessorGetActionTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/GrokProcessorGetActionTests.java new file mode 100644 index 00000000000..aa54044f845 --- /dev/null +++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/GrokProcessorGetActionTests.java @@ -0,0 +1,72 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.ingest.common; + +import org.elasticsearch.common.io.stream.BytesStreamOutput; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.xcontent.ToXContent; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentHelper; +import org.elasticsearch.common.xcontent.json.JsonXContent; +import org.elasticsearch.test.ESTestCase; + +import java.util.Collections; +import java.util.Map; + +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.core.IsNull.nullValue; + + +public class GrokProcessorGetActionTests extends ESTestCase { + private static final Map TEST_PATTERNS = Collections.singletonMap("PATTERN", "foo"); + + public void testRequest() throws Exception { + GrokProcessorGetAction.Request request = new GrokProcessorGetAction.Request(); + BytesStreamOutput out = new BytesStreamOutput(); + request.writeTo(out); + StreamInput streamInput = out.bytes().streamInput(); + GrokProcessorGetAction.Request otherRequest = new GrokProcessorGetAction.Request(); + otherRequest.readFrom(streamInput); + assertThat(otherRequest.validate(), nullValue()); + } + + public void testResponseSerialization() throws Exception { + GrokProcessorGetAction.Response response = new GrokProcessorGetAction.Response(TEST_PATTERNS); + BytesStreamOutput out = new BytesStreamOutput(); + response.writeTo(out); + StreamInput streamInput = out.bytes().streamInput(); + GrokProcessorGetAction.Response otherResponse = new GrokProcessorGetAction.Response(null); + otherResponse.readFrom(streamInput); + assertThat(response.getGrokPatterns(), equalTo(TEST_PATTERNS)); + assertThat(response.getGrokPatterns(), equalTo(otherResponse.getGrokPatterns())); + } + + @SuppressWarnings("unchecked") + public void testResponseToXContent() throws Exception { + GrokProcessorGetAction.Response response = new GrokProcessorGetAction.Response(TEST_PATTERNS); + try (XContentBuilder builder = JsonXContent.contentBuilder()) { + response.toXContent(builder, ToXContent.EMPTY_PARAMS); + Map converted = XContentHelper.convertToMap(builder.bytes(), false, builder.contentType()).v2(); + Map patterns = (Map) converted.get("patterns"); + assertThat(patterns.size(), equalTo(1)); + assertThat(patterns.get("PATTERN"), equalTo("foo")); + } + } +} diff --git a/modules/ingest-common/src/test/resources/rest-api-spec/test/ingest/120_grok.yml b/modules/ingest-common/src/test/resources/rest-api-spec/test/ingest/120_grok.yml index 0d77ccce61c..353365df95a 100644 --- a/modules/ingest-common/src/test/resources/rest-api-spec/test/ingest/120_grok.yml +++ b/modules/ingest-common/src/test/resources/rest-api-spec/test/ingest/120_grok.yml @@ -154,3 +154,10 @@ teardown: - length: { docs.0.doc._ingest: 2 } - match: { docs.0.doc._ingest._grok_match_index: "1" } - is_true: docs.0.doc._ingest.timestamp + +--- +"Test Grok Patterns Retrieval": + - do: + ingest.processor.grok: {} + - length: { patterns: 303 } + - match: { patterns.PATH: "(?:%{UNIXPATH}|%{WINPATH})" } diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/ingest.processor.grok.json b/rest-api-spec/src/main/resources/rest-api-spec/api/ingest.processor.grok.json new file mode 100644 index 00000000000..1bd0c3ceec4 --- /dev/null +++ b/rest-api-spec/src/main/resources/rest-api-spec/api/ingest.processor.grok.json @@ -0,0 +1,15 @@ +{ + "ingest.processor.grok": { + "documentation": "https://www.elastic.co/guide/en/elasticsearch/plugins/master/ingest.html", + "methods": [ "GET" ], + "url": { + "path": "/_ingest/processor/grok", + "paths": ["/_ingest/processor/grok"], + "parts": { + }, + "params": { + } + }, + "body": null + } +}