From 0553ba9151dd5882391d076216b9363679a64949 Mon Sep 17 00:00:00 2001 From: Chris Earle Date: Tue, 26 Jul 2016 13:14:27 -0400 Subject: [PATCH] [Ingest] Add REST _ingest/pipeline to get all pipelines This adds an extra REST handler for "_ingest/pipeline" so that users do not need to supply "_ingest/pipeline/*" to get all of them. - Also adds a teardown section to related REST-tests for ingest. --- .../action/ingest/GetPipelineRequest.java | 9 ++-- .../elasticsearch/ingest/PipelineStore.java | 5 ++ .../action/ingest/RestGetPipelineAction.java | 1 + .../ingest/PipelineStoreTests.java | 16 ++++-- .../ingest/100_date_index_name_processor.yaml | 7 +++ .../rest-api-spec/test/ingest/110_sort.yaml | 7 +++ .../rest-api-spec/test/ingest/120_grok.yaml | 7 +++ .../rest-api-spec/test/ingest/20_crud.yaml | 52 +++++++++++++++++++ .../test/ingest/30_date_processor.yaml | 7 +++ .../rest-api-spec/test/ingest/40_mutate.yaml | 7 +++ .../test/ingest/50_on_failure.yaml | 7 +++ .../rest-api-spec/test/ingest/60_fail.yaml | 7 +++ .../rest-api-spec/test/ingest/70_bulk.yaml | 11 ++++ .../rest-api-spec/test/ingest/80_foreach.yaml | 7 +++ .../test/ingest/90_simulate.yaml | 7 +++ .../api/ingest.get_pipeline.json | 2 +- 16 files changed, 150 insertions(+), 9 deletions(-) diff --git a/core/src/main/java/org/elasticsearch/action/ingest/GetPipelineRequest.java b/core/src/main/java/org/elasticsearch/action/ingest/GetPipelineRequest.java index 6525c26c6c6..1ba22bce805 100644 --- a/core/src/main/java/org/elasticsearch/action/ingest/GetPipelineRequest.java +++ b/core/src/main/java/org/elasticsearch/action/ingest/GetPipelineRequest.java @@ -21,26 +21,25 @@ package org.elasticsearch.action.ingest; import org.elasticsearch.action.ActionRequestValidationException; import org.elasticsearch.action.support.master.MasterNodeReadRequest; +import org.elasticsearch.common.Strings; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import java.io.IOException; -import java.util.Objects; - -import static org.elasticsearch.action.ValidateActions.addValidationError; public class GetPipelineRequest extends MasterNodeReadRequest { private String[] ids; public GetPipelineRequest(String... ids) { - if (ids == null || ids.length == 0) { - throw new IllegalArgumentException("No ids specified"); + if (ids == null) { + throw new IllegalArgumentException("ids cannot be null"); } this.ids = ids; } GetPipelineRequest() { + this.ids = Strings.EMPTY_ARRAY; } public String[] getIds() { diff --git a/core/src/main/java/org/elasticsearch/ingest/PipelineStore.java b/core/src/main/java/org/elasticsearch/ingest/PipelineStore.java index 50ae19a2ca1..713001c233e 100644 --- a/core/src/main/java/org/elasticsearch/ingest/PipelineStore.java +++ b/core/src/main/java/org/elasticsearch/ingest/PipelineStore.java @@ -207,6 +207,11 @@ public class PipelineStore extends AbstractComponent implements ClusterStateList return Collections.emptyList(); } + // if we didn't ask for _any_ ID, then we get them all (this is the same as if they ask for '*') + if (ids.length == 0) { + return new ArrayList<>(ingestMetadata.getPipelines().values()); + } + List result = new ArrayList<>(ids.length); for (String id : ids) { if (Regex.isSimpleMatchPattern(id)) { diff --git a/core/src/main/java/org/elasticsearch/rest/action/ingest/RestGetPipelineAction.java b/core/src/main/java/org/elasticsearch/rest/action/ingest/RestGetPipelineAction.java index 55f14cfaa84..2b7adedcacf 100644 --- a/core/src/main/java/org/elasticsearch/rest/action/ingest/RestGetPipelineAction.java +++ b/core/src/main/java/org/elasticsearch/rest/action/ingest/RestGetPipelineAction.java @@ -35,6 +35,7 @@ public class RestGetPipelineAction extends BaseRestHandler { @Inject public RestGetPipelineAction(Settings settings, RestController controller) { super(settings); + controller.registerHandler(RestRequest.Method.GET, "/_ingest/pipeline", this); controller.registerHandler(RestRequest.Method.GET, "/_ingest/pipeline/{id}", this); } diff --git a/core/src/test/java/org/elasticsearch/ingest/PipelineStoreTests.java b/core/src/test/java/org/elasticsearch/ingest/PipelineStoreTests.java index 1510d25b695..29032ae4326 100644 --- a/core/src/test/java/org/elasticsearch/ingest/PipelineStoreTests.java +++ b/core/src/test/java/org/elasticsearch/ingest/PipelineStoreTests.java @@ -28,11 +28,9 @@ import org.elasticsearch.cluster.ClusterName; import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.metadata.MetaData; import org.elasticsearch.cluster.node.DiscoveryNode; -import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.bytes.BytesArray; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.transport.LocalTransportAddress; -import org.elasticsearch.script.ScriptService; import org.elasticsearch.test.ESTestCase; import org.junit.Before; @@ -48,7 +46,6 @@ import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.is; import static org.hamcrest.Matchers.notNullValue; import static org.hamcrest.Matchers.nullValue; -import static org.mockito.Mockito.mock; public class PipelineStoreTests extends ESTestCase { @@ -216,6 +213,19 @@ public class PipelineStoreTests extends ESTestCase { assertThat(pipelines.size(), equalTo(2)); assertThat(pipelines.get(0).getId(), equalTo("_id1")); assertThat(pipelines.get(1).getId(), equalTo("_id2")); + + // get all variants: (no IDs or '*') + pipelines = store.innerGetPipelines(ingestMetadata); + pipelines.sort((o1, o2) -> o1.getId().compareTo(o2.getId())); + assertThat(pipelines.size(), equalTo(2)); + assertThat(pipelines.get(0).getId(), equalTo("_id1")); + assertThat(pipelines.get(1).getId(), equalTo("_id2")); + + pipelines = store.innerGetPipelines(ingestMetadata, "*"); + pipelines.sort((o1, o2) -> o1.getId().compareTo(o2.getId())); + assertThat(pipelines.size(), equalTo(2)); + assertThat(pipelines.get(0).getId(), equalTo("_id1")); + assertThat(pipelines.get(1).getId(), equalTo("_id2")); } public void testCrud() throws Exception { diff --git a/modules/ingest-common/src/test/resources/rest-api-spec/test/ingest/100_date_index_name_processor.yaml b/modules/ingest-common/src/test/resources/rest-api-spec/test/ingest/100_date_index_name_processor.yaml index c99c1025c10..ccf83cc96bf 100644 --- a/modules/ingest-common/src/test/resources/rest-api-spec/test/ingest/100_date_index_name_processor.yaml +++ b/modules/ingest-common/src/test/resources/rest-api-spec/test/ingest/100_date_index_name_processor.yaml @@ -1,3 +1,10 @@ +--- +teardown: + - do: + ingest.delete_pipeline: + id: "1" + ignore: 404 + --- "Test date index name processor with defaults": - do: diff --git a/modules/ingest-common/src/test/resources/rest-api-spec/test/ingest/110_sort.yaml b/modules/ingest-common/src/test/resources/rest-api-spec/test/ingest/110_sort.yaml index 096e57fa4e1..e18a0dcd252 100644 --- a/modules/ingest-common/src/test/resources/rest-api-spec/test/ingest/110_sort.yaml +++ b/modules/ingest-common/src/test/resources/rest-api-spec/test/ingest/110_sort.yaml @@ -1,3 +1,10 @@ +--- +teardown: + - do: + ingest.delete_pipeline: + id: "my_pipeline" + ignore: 404 + --- "Test sort Processor": - do: diff --git a/modules/ingest-common/src/test/resources/rest-api-spec/test/ingest/120_grok.yaml b/modules/ingest-common/src/test/resources/rest-api-spec/test/ingest/120_grok.yaml index a51f5a102b0..0d77ccce61c 100644 --- a/modules/ingest-common/src/test/resources/rest-api-spec/test/ingest/120_grok.yaml +++ b/modules/ingest-common/src/test/resources/rest-api-spec/test/ingest/120_grok.yaml @@ -1,3 +1,10 @@ +--- +teardown: + - do: + ingest.delete_pipeline: + id: "my_pipeline" + ignore: 404 + --- "Test Grok Pipeline": - do: diff --git a/modules/ingest-common/src/test/resources/rest-api-spec/test/ingest/20_crud.yaml b/modules/ingest-common/src/test/resources/rest-api-spec/test/ingest/20_crud.yaml index b494161aff1..2fb6ac230ff 100644 --- a/modules/ingest-common/src/test/resources/rest-api-spec/test/ingest/20_crud.yaml +++ b/modules/ingest-common/src/test/resources/rest-api-spec/test/ingest/20_crud.yaml @@ -1,3 +1,18 @@ +--- +teardown: + - do: + ingest.delete_pipeline: + id: "my_pipeline" + ignore: 404 + - do: + ingest.delete_pipeline: + id: "first_pipeline" + ignore: 404 + - do: + ingest.delete_pipeline: + id: "second_pipeline" + ignore: 404 + --- "Test basic pipeline crud": - do: @@ -23,6 +38,11 @@ - match: { pipelines.0.id: "my_pipeline" } - match: { pipelines.0.config.description: "_description" } + - do: + ingest.get_pipeline: {} + - match: { pipelines.0.id: "my_pipeline" } + - match: { pipelines.0.config.description: "_description" } + - do: ingest.delete_pipeline: id: "my_pipeline" @@ -33,6 +53,38 @@ ingest.get_pipeline: id: "my_pipeline" +--- +"Test Get All Pipelines (unordered)": + - do: + ingest.put_pipeline: + id: "first_pipeline" + body: > + { + "description": "first", + "processors": [ + { + "set" : { + "field" : "field1", + "value": "_value" + } + } + ] + } + - do: + ingest.put_pipeline: + id: "second_pipeline" + body: > + { + "description": "second", + "processors": [] + } + + # Order is not guaranteed by the response, so we check for length instead; above tests that we get appropriate values + - do: + ingest.get_pipeline: {} + - length: { pipelines: 2 } + + --- "Test invalid config": - do: diff --git a/modules/ingest-common/src/test/resources/rest-api-spec/test/ingest/30_date_processor.yaml b/modules/ingest-common/src/test/resources/rest-api-spec/test/ingest/30_date_processor.yaml index 33e25e4989b..93dfa52196f 100644 --- a/modules/ingest-common/src/test/resources/rest-api-spec/test/ingest/30_date_processor.yaml +++ b/modules/ingest-common/src/test/resources/rest-api-spec/test/ingest/30_date_processor.yaml @@ -1,3 +1,10 @@ +--- +teardown: + - do: + ingest.delete_pipeline: + id: "my_pipeline" + ignore: 404 + --- "Test date processor": - do: diff --git a/modules/ingest-common/src/test/resources/rest-api-spec/test/ingest/40_mutate.yaml b/modules/ingest-common/src/test/resources/rest-api-spec/test/ingest/40_mutate.yaml index c5d27f720cb..8150891ebd0 100644 --- a/modules/ingest-common/src/test/resources/rest-api-spec/test/ingest/40_mutate.yaml +++ b/modules/ingest-common/src/test/resources/rest-api-spec/test/ingest/40_mutate.yaml @@ -1,3 +1,10 @@ +--- +teardown: + - do: + ingest.delete_pipeline: + id: "my_pipeline" + ignore: 404 + --- "Test mutate processors": - do: diff --git a/modules/ingest-common/src/test/resources/rest-api-spec/test/ingest/50_on_failure.yaml b/modules/ingest-common/src/test/resources/rest-api-spec/test/ingest/50_on_failure.yaml index a4395427e04..53c1a9a7923 100644 --- a/modules/ingest-common/src/test/resources/rest-api-spec/test/ingest/50_on_failure.yaml +++ b/modules/ingest-common/src/test/resources/rest-api-spec/test/ingest/50_on_failure.yaml @@ -1,3 +1,10 @@ +--- +teardown: + - do: + ingest.delete_pipeline: + id: "my_pipeline" + ignore: 404 + --- "Test Pipeline With On Failure Block": - do: diff --git a/modules/ingest-common/src/test/resources/rest-api-spec/test/ingest/60_fail.yaml b/modules/ingest-common/src/test/resources/rest-api-spec/test/ingest/60_fail.yaml index 019c229ae38..e080991e13c 100644 --- a/modules/ingest-common/src/test/resources/rest-api-spec/test/ingest/60_fail.yaml +++ b/modules/ingest-common/src/test/resources/rest-api-spec/test/ingest/60_fail.yaml @@ -1,3 +1,10 @@ +--- +teardown: + - do: + ingest.delete_pipeline: + id: "my_pipeline" + ignore: 404 + --- "Test Fail Processor": - do: diff --git a/modules/ingest-common/src/test/resources/rest-api-spec/test/ingest/70_bulk.yaml b/modules/ingest-common/src/test/resources/rest-api-spec/test/ingest/70_bulk.yaml index d834d86b2ea..cf428b32524 100644 --- a/modules/ingest-common/src/test/resources/rest-api-spec/test/ingest/70_bulk.yaml +++ b/modules/ingest-common/src/test/resources/rest-api-spec/test/ingest/70_bulk.yaml @@ -31,6 +31,17 @@ setup: ] } +--- +teardown: + - do: + ingest.delete_pipeline: + id: "pipeline1" + ignore: 404 + - do: + ingest.delete_pipeline: + id: "pipeline2" + ignore: 404 + --- "Test bulk request without default pipeline": diff --git a/modules/ingest-common/src/test/resources/rest-api-spec/test/ingest/80_foreach.yaml b/modules/ingest-common/src/test/resources/rest-api-spec/test/ingest/80_foreach.yaml index 2ebfc089396..48080433fd2 100644 --- a/modules/ingest-common/src/test/resources/rest-api-spec/test/ingest/80_foreach.yaml +++ b/modules/ingest-common/src/test/resources/rest-api-spec/test/ingest/80_foreach.yaml @@ -1,3 +1,10 @@ +--- +teardown: + - do: + ingest.delete_pipeline: + id: "my_pipeline" + ignore: 404 + --- "Test foreach Processor": - do: diff --git a/modules/ingest-common/src/test/resources/rest-api-spec/test/ingest/90_simulate.yaml b/modules/ingest-common/src/test/resources/rest-api-spec/test/ingest/90_simulate.yaml index 7ce30ee3bde..8b08535c124 100644 --- a/modules/ingest-common/src/test/resources/rest-api-spec/test/ingest/90_simulate.yaml +++ b/modules/ingest-common/src/test/resources/rest-api-spec/test/ingest/90_simulate.yaml @@ -1,3 +1,10 @@ +--- +teardown: + - do: + ingest.delete_pipeline: + id: "my_pipeline" + ignore: 404 + --- "Test simulate with stored ingest pipeline": - do: diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/ingest.get_pipeline.json b/rest-api-spec/src/main/resources/rest-api-spec/api/ingest.get_pipeline.json index 6c50657ae1a..5574ddbf5b7 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/api/ingest.get_pipeline.json +++ b/rest-api-spec/src/main/resources/rest-api-spec/api/ingest.get_pipeline.json @@ -4,7 +4,7 @@ "methods": [ "GET" ], "url": { "path": "/_ingest/pipeline/{id}", - "paths": [ "/_ingest/pipeline/{id}" ], + "paths": [ "/_ingest/pipeline", "/_ingest/pipeline/{id}" ], "parts": { "id": { "type" : "string",