Add Ingest-Processor specific Rest Endpoints & Add Grok endpoint (#25059)
This PR enables Ingest plugins to leverage processor-scoped REST endpoints. First of which being the Grok endpoint that retrieves Grok Patterns for users to retrieve all the built-in patterns. Example usage: Kibana Grok Autocomplete!
This commit is contained in:
parent
340909582f
commit
a771912a22
|
@ -1454,6 +1454,32 @@ second (index starts at zero) pattern in `patterns` to match.
|
|||
This trace metadata enables debugging which of the patterns matched. This information is stored in the ingest
|
||||
metadata and will not be indexed.
|
||||
|
||||
[[grok-processor-rest-get]]
|
||||
==== Retrieving patterns from REST endpoint
|
||||
|
||||
The Grok Processor comes packaged with its own REST endpoint for retrieving which patterns the processor is packaged with.
|
||||
|
||||
[source,js]
|
||||
--------------------------------------------------
|
||||
GET _ingest/processor/grok
|
||||
--------------------------------------------------
|
||||
// CONSOLE
|
||||
|
||||
The above request will return a response body containing a key-value representation of the built-in patterns dictionary.
|
||||
|
||||
[source,js]
|
||||
--------------------------------------------------
|
||||
{
|
||||
"patterns" : {
|
||||
"BACULA_CAPACITY" : "%{INT}{1,3}(,%{INT}{3})*",
|
||||
"PATH" : "(?:%{UNIXPATH}|%{WINPATH})",
|
||||
...
|
||||
}
|
||||
--------------------------------------------------
|
||||
// NOTCONSOLE
|
||||
|
||||
This can be useful to reference as the built-in patterns change across versions.
|
||||
|
||||
[[gsub-processor]]
|
||||
=== Gsub Processor
|
||||
Converts a string field by applying a regular expression and a replacement.
|
||||
|
|
|
@ -0,0 +1,162 @@
|
|||
/*
|
||||
* Licensed to Elasticsearch under one or more contributor
|
||||
* license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright
|
||||
* ownership. Elasticsearch licenses this file to you under
|
||||
* the Apache License, Version 2.0 (the "License"); you may
|
||||
* not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing,
|
||||
* software distributed under the License is distributed on an
|
||||
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
* KIND, either express or implied. See the License for the
|
||||
* specific language governing permissions and limitations
|
||||
* under the License.
|
||||
*/
|
||||
package org.elasticsearch.ingest.common;
|
||||
|
||||
import org.elasticsearch.action.Action;
|
||||
import org.elasticsearch.action.ActionListener;
|
||||
import org.elasticsearch.action.ActionRequest;
|
||||
import org.elasticsearch.action.ActionRequestBuilder;
|
||||
import org.elasticsearch.action.ActionRequestValidationException;
|
||||
import org.elasticsearch.action.support.ActionFilters;
|
||||
import org.elasticsearch.action.support.HandledTransportAction;
|
||||
import org.elasticsearch.action.support.master.AcknowledgedResponse;
|
||||
import org.elasticsearch.client.ElasticsearchClient;
|
||||
import org.elasticsearch.client.node.NodeClient;
|
||||
import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
|
||||
import org.elasticsearch.common.inject.Inject;
|
||||
import org.elasticsearch.common.io.stream.StreamInput;
|
||||
import org.elasticsearch.common.io.stream.StreamOutput;
|
||||
import org.elasticsearch.common.settings.Settings;
|
||||
import org.elasticsearch.common.xcontent.ToXContent;
|
||||
import org.elasticsearch.common.xcontent.ToXContentObject;
|
||||
import org.elasticsearch.common.xcontent.XContentBuilder;
|
||||
import org.elasticsearch.rest.BaseRestHandler;
|
||||
import org.elasticsearch.rest.BytesRestResponse;
|
||||
import org.elasticsearch.rest.RestController;
|
||||
import org.elasticsearch.rest.RestRequest;
|
||||
import org.elasticsearch.rest.RestResponse;
|
||||
import org.elasticsearch.rest.action.RestBuilderListener;
|
||||
import org.elasticsearch.threadpool.ThreadPool;
|
||||
import org.elasticsearch.transport.TransportService;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.util.Map;
|
||||
|
||||
import static org.elasticsearch.ingest.common.IngestCommonPlugin.GROK_PATTERNS;
|
||||
import static org.elasticsearch.rest.RestRequest.Method.GET;
|
||||
import static org.elasticsearch.rest.RestStatus.OK;
|
||||
|
||||
public class GrokProcessorGetAction extends Action<GrokProcessorGetAction.Request,
|
||||
GrokProcessorGetAction.Response, GrokProcessorGetAction.RequestBuilder> {
|
||||
|
||||
public static final GrokProcessorGetAction INSTANCE = new GrokProcessorGetAction();
|
||||
public static final String NAME = "cluster:admin/ingest/processor/grok/get";
|
||||
|
||||
private GrokProcessorGetAction() {
|
||||
super(NAME);
|
||||
}
|
||||
|
||||
@Override
|
||||
public RequestBuilder newRequestBuilder(ElasticsearchClient client) {
|
||||
return new RequestBuilder(client);
|
||||
}
|
||||
|
||||
@Override
|
||||
public Response newResponse() {
|
||||
return new Response(null);
|
||||
}
|
||||
|
||||
public static class Request extends ActionRequest {
|
||||
@Override
|
||||
public ActionRequestValidationException validate() {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
public static class RequestBuilder extends ActionRequestBuilder<Request, Response, RequestBuilder> {
|
||||
public RequestBuilder(ElasticsearchClient client) {
|
||||
super(client, GrokProcessorGetAction.INSTANCE, new Request());
|
||||
}
|
||||
}
|
||||
|
||||
public static class Response extends AcknowledgedResponse implements ToXContentObject {
|
||||
private Map<String, String> grokPatterns;
|
||||
|
||||
public Response(Map<String, String> grokPatterns) {
|
||||
this.grokPatterns = grokPatterns;
|
||||
}
|
||||
|
||||
public Map<String, String> getGrokPatterns() {
|
||||
return grokPatterns;
|
||||
}
|
||||
|
||||
@Override
|
||||
public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
|
||||
builder.startObject();
|
||||
builder.field("patterns");
|
||||
builder.map(grokPatterns);
|
||||
builder.endObject();
|
||||
return builder;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void readFrom(StreamInput in) throws IOException {
|
||||
super.readFrom(in);
|
||||
grokPatterns = in.readMap(StreamInput::readString, StreamInput::readString);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void writeTo(StreamOutput out) throws IOException {
|
||||
super.writeTo(out);
|
||||
out.writeMap(grokPatterns, StreamOutput::writeString, StreamOutput::writeString);
|
||||
}
|
||||
}
|
||||
|
||||
public static class TransportAction extends HandledTransportAction<Request, Response> {
|
||||
|
||||
@Inject
|
||||
public TransportAction(Settings settings, ThreadPool threadPool, TransportService transportService,
|
||||
ActionFilters actionFilters, IndexNameExpressionResolver indexNameExpressionResolver) {
|
||||
super(settings, NAME, threadPool, transportService, actionFilters,
|
||||
indexNameExpressionResolver, Request::new);
|
||||
}
|
||||
|
||||
@Override
|
||||
protected void doExecute(Request request, ActionListener<Response> listener) {
|
||||
try {
|
||||
listener.onResponse(new Response(GROK_PATTERNS));
|
||||
} catch (Exception e) {
|
||||
listener.onFailure(e);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public static class RestAction extends BaseRestHandler {
|
||||
public RestAction(Settings settings, RestController controller) {
|
||||
super(settings);
|
||||
controller.registerHandler(GET, "/_ingest/processor/grok", this);
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getName() {
|
||||
return "ingest_processor_grok_get";
|
||||
}
|
||||
|
||||
@Override
|
||||
protected RestChannelConsumer prepareRequest(RestRequest request, NodeClient client) throws IOException {
|
||||
return channel -> client.executeLocally(INSTANCE, new Request(), new RestBuilderListener<Response>(channel) {
|
||||
@Override
|
||||
public RestResponse buildResponse(Response response, XContentBuilder builder) throws Exception {
|
||||
response.toXContent(builder, ToXContent.EMPTY_PARAMS);
|
||||
return new BytesRestResponse(OK, builder);
|
||||
}
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
|
@ -23,21 +23,48 @@ import java.io.BufferedReader;
|
|||
import java.io.IOException;
|
||||
import java.io.InputStream;
|
||||
import java.io.InputStreamReader;
|
||||
import java.io.UncheckedIOException;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.util.Arrays;
|
||||
import java.util.Collections;
|
||||
import java.util.HashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.function.Supplier;
|
||||
|
||||
import org.elasticsearch.action.ActionRequest;
|
||||
import org.elasticsearch.action.ActionResponse;
|
||||
import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
|
||||
import org.elasticsearch.cluster.node.DiscoveryNodes;
|
||||
import org.elasticsearch.common.settings.ClusterSettings;
|
||||
import org.elasticsearch.common.settings.IndexScopedSettings;
|
||||
import org.elasticsearch.common.settings.Settings;
|
||||
import org.elasticsearch.common.settings.SettingsFilter;
|
||||
import org.elasticsearch.ingest.Processor;
|
||||
import org.elasticsearch.plugins.ActionPlugin;
|
||||
import org.elasticsearch.plugins.IngestPlugin;
|
||||
import org.elasticsearch.plugins.Plugin;
|
||||
import org.elasticsearch.rest.RestController;
|
||||
import org.elasticsearch.rest.RestHandler;
|
||||
|
||||
public class IngestCommonPlugin extends Plugin implements IngestPlugin {
|
||||
public class IngestCommonPlugin extends Plugin implements ActionPlugin, IngestPlugin {
|
||||
|
||||
private final Map<String, String> builtinPatterns;
|
||||
// Code for loading built-in grok patterns packaged with the jar file:
|
||||
private static final String[] PATTERN_NAMES = new String[] {
|
||||
"aws", "bacula", "bro", "exim", "firewalls", "grok-patterns", "haproxy",
|
||||
"java", "junos", "linux-syslog", "mcollective-patterns", "mongodb", "nagios",
|
||||
"postgresql", "rails", "redis", "ruby"
|
||||
};
|
||||
static final Map<String, String> GROK_PATTERNS;
|
||||
static {
|
||||
try {
|
||||
GROK_PATTERNS = loadBuiltinPatterns();
|
||||
} catch (IOException e) {
|
||||
throw new UncheckedIOException("unable to load built-in grok patterns", e);
|
||||
}
|
||||
}
|
||||
|
||||
public IngestCommonPlugin() throws IOException {
|
||||
this.builtinPatterns = loadBuiltinPatterns();
|
||||
}
|
||||
|
||||
@Override
|
||||
|
@ -59,7 +86,7 @@ public class IngestCommonPlugin extends Plugin implements IngestPlugin {
|
|||
processors.put(ForEachProcessor.TYPE, new ForEachProcessor.Factory());
|
||||
processors.put(DateIndexNameProcessor.TYPE, new DateIndexNameProcessor.Factory());
|
||||
processors.put(SortProcessor.TYPE, new SortProcessor.Factory());
|
||||
processors.put(GrokProcessor.TYPE, new GrokProcessor.Factory(builtinPatterns));
|
||||
processors.put(GrokProcessor.TYPE, new GrokProcessor.Factory(GROK_PATTERNS));
|
||||
processors.put(ScriptProcessor.TYPE, new ScriptProcessor.Factory(parameters.scriptService));
|
||||
processors.put(DotExpanderProcessor.TYPE, new DotExpanderProcessor.Factory());
|
||||
processors.put(JsonProcessor.TYPE, new JsonProcessor.Factory());
|
||||
|
@ -67,13 +94,19 @@ public class IngestCommonPlugin extends Plugin implements IngestPlugin {
|
|||
return Collections.unmodifiableMap(processors);
|
||||
}
|
||||
|
||||
// Code for loading built-in grok patterns packaged with the jar file:
|
||||
@Override
|
||||
public List<ActionHandler<? extends ActionRequest, ? extends ActionResponse>> getActions() {
|
||||
return Arrays.asList(new ActionHandler<>(GrokProcessorGetAction.INSTANCE, GrokProcessorGetAction.TransportAction.class));
|
||||
}
|
||||
|
||||
@Override
|
||||
public List<RestHandler> getRestHandlers(Settings settings, RestController restController, ClusterSettings clusterSettings,
|
||||
IndexScopedSettings indexScopedSettings, SettingsFilter settingsFilter,
|
||||
IndexNameExpressionResolver indexNameExpressionResolver,
|
||||
Supplier<DiscoveryNodes> nodesInCluster) {
|
||||
return Arrays.asList(new GrokProcessorGetAction.RestAction(settings, restController));
|
||||
}
|
||||
|
||||
private static final String[] PATTERN_NAMES = new String[] {
|
||||
"aws", "bacula", "bro", "exim", "firewalls", "grok-patterns", "haproxy",
|
||||
"java", "junos", "linux-syslog", "mcollective-patterns", "mongodb", "nagios",
|
||||
"postgresql", "rails", "redis", "ruby"
|
||||
};
|
||||
|
||||
public static Map<String, String> loadBuiltinPatterns() throws IOException {
|
||||
Map<String, String> builtinPatterns = new HashMap<>();
|
||||
|
|
|
@ -0,0 +1,72 @@
|
|||
/*
|
||||
* Licensed to Elasticsearch under one or more contributor
|
||||
* license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright
|
||||
* ownership. Elasticsearch licenses this file to you under
|
||||
* the Apache License, Version 2.0 (the "License"); you may
|
||||
* not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing,
|
||||
* software distributed under the License is distributed on an
|
||||
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
* KIND, either express or implied. See the License for the
|
||||
* specific language governing permissions and limitations
|
||||
* under the License.
|
||||
*/
|
||||
|
||||
package org.elasticsearch.ingest.common;
|
||||
|
||||
import org.elasticsearch.common.io.stream.BytesStreamOutput;
|
||||
import org.elasticsearch.common.io.stream.StreamInput;
|
||||
import org.elasticsearch.common.xcontent.ToXContent;
|
||||
import org.elasticsearch.common.xcontent.XContentBuilder;
|
||||
import org.elasticsearch.common.xcontent.XContentHelper;
|
||||
import org.elasticsearch.common.xcontent.json.JsonXContent;
|
||||
import org.elasticsearch.test.ESTestCase;
|
||||
|
||||
import java.util.Collections;
|
||||
import java.util.Map;
|
||||
|
||||
import static org.hamcrest.Matchers.equalTo;
|
||||
import static org.hamcrest.core.IsNull.nullValue;
|
||||
|
||||
|
||||
public class GrokProcessorGetActionTests extends ESTestCase {
|
||||
private static final Map<String, String> TEST_PATTERNS = Collections.singletonMap("PATTERN", "foo");
|
||||
|
||||
public void testRequest() throws Exception {
|
||||
GrokProcessorGetAction.Request request = new GrokProcessorGetAction.Request();
|
||||
BytesStreamOutput out = new BytesStreamOutput();
|
||||
request.writeTo(out);
|
||||
StreamInput streamInput = out.bytes().streamInput();
|
||||
GrokProcessorGetAction.Request otherRequest = new GrokProcessorGetAction.Request();
|
||||
otherRequest.readFrom(streamInput);
|
||||
assertThat(otherRequest.validate(), nullValue());
|
||||
}
|
||||
|
||||
public void testResponseSerialization() throws Exception {
|
||||
GrokProcessorGetAction.Response response = new GrokProcessorGetAction.Response(TEST_PATTERNS);
|
||||
BytesStreamOutput out = new BytesStreamOutput();
|
||||
response.writeTo(out);
|
||||
StreamInput streamInput = out.bytes().streamInput();
|
||||
GrokProcessorGetAction.Response otherResponse = new GrokProcessorGetAction.Response(null);
|
||||
otherResponse.readFrom(streamInput);
|
||||
assertThat(response.getGrokPatterns(), equalTo(TEST_PATTERNS));
|
||||
assertThat(response.getGrokPatterns(), equalTo(otherResponse.getGrokPatterns()));
|
||||
}
|
||||
|
||||
@SuppressWarnings("unchecked")
|
||||
public void testResponseToXContent() throws Exception {
|
||||
GrokProcessorGetAction.Response response = new GrokProcessorGetAction.Response(TEST_PATTERNS);
|
||||
try (XContentBuilder builder = JsonXContent.contentBuilder()) {
|
||||
response.toXContent(builder, ToXContent.EMPTY_PARAMS);
|
||||
Map<String, Object> converted = XContentHelper.convertToMap(builder.bytes(), false, builder.contentType()).v2();
|
||||
Map<String, String> patterns = (Map<String, String>) converted.get("patterns");
|
||||
assertThat(patterns.size(), equalTo(1));
|
||||
assertThat(patterns.get("PATTERN"), equalTo("foo"));
|
||||
}
|
||||
}
|
||||
}
|
|
@ -154,3 +154,10 @@ teardown:
|
|||
- length: { docs.0.doc._ingest: 2 }
|
||||
- match: { docs.0.doc._ingest._grok_match_index: "1" }
|
||||
- is_true: docs.0.doc._ingest.timestamp
|
||||
|
||||
---
|
||||
"Test Grok Patterns Retrieval":
|
||||
- do:
|
||||
ingest.processor.grok: {}
|
||||
- length: { patterns: 303 }
|
||||
- match: { patterns.PATH: "(?:%{UNIXPATH}|%{WINPATH})" }
|
||||
|
|
|
@ -0,0 +1,15 @@
|
|||
{
|
||||
"ingest.processor.grok": {
|
||||
"documentation": "https://www.elastic.co/guide/en/elasticsearch/plugins/master/ingest.html",
|
||||
"methods": [ "GET" ],
|
||||
"url": {
|
||||
"path": "/_ingest/processor/grok",
|
||||
"paths": ["/_ingest/processor/grok"],
|
||||
"parts": {
|
||||
},
|
||||
"params": {
|
||||
}
|
||||
},
|
||||
"body": null
|
||||
}
|
||||
}
|
Loading…
Reference in New Issue