From 183386173cf3d6278261db06437b6e64a9234982 Mon Sep 17 00:00:00 2001 From: Tal Levy Date: Mon, 4 Jan 2016 16:10:42 -0800 Subject: [PATCH] cleanup simulate test and add docs --- docs/plugins/ingest.asciidoc | 275 ++++++++++++++++++ .../test/ingest/70_simulate.yaml | 2 +- 2 files changed, 276 insertions(+), 1 deletion(-) diff --git a/docs/plugins/ingest.asciidoc b/docs/plugins/ingest.asciidoc index ca841b0d2b2..ed9660ac3af 100644 --- a/docs/plugins/ingest.asciidoc +++ b/docs/plugins/ingest.asciidoc @@ -631,3 +631,278 @@ The delete pipeline api deletes pipelines by id. DELETE _ingest/pipeline/my-pipeline-id -------------------------------------------------- // AUTOSENSE + +==== Simulate pipeline API + +The simulate pipeline api executes a specific pipeline against +the set of documents provided in the body of the request. + +A simulate request may call upon an existing pipeline to be executed +against the provided documents, or supply a pipeline definition in +the body of the request. + +Here is the structure of a simulate request with a provided pipeline: + +[source,js] +-------------------------------------------------- +POST _ingest/pipeline/_simulate +{ + "pipeline" : { + // pipeline definition here + }, + "docs" : [ + { /** first document **/ }, + { /** second document **/ }, + // ... + ] +} +-------------------------------------------------- + +Here is the structure of a simulate request against a pre-existing pipeline: + +[source,js] +-------------------------------------------------- +POST _ingest/pipeline/my-pipeline-id/_simulate +{ + "docs" : [ + { /** first document **/ }, + { /** second document **/ }, + // ... + ] +} +-------------------------------------------------- + + +Here is an example simulate request with a provided pipeline and its response: + +[source,js] +-------------------------------------------------- +POST _ingest/pipeline/_simulate +{ + "pipeline" : + { + "description": "_description", + "processors": [ + { + "set" : { + "field" : "field2", + "value" : "_value" + } + } + ] + }, + "docs": [ + { + "_index": "index", + "_type": "type", + "_id": "id", + "_source": { + "foo": "bar" + } + }, + { + "_index": "index", + "_type": "type", + "_id": "id", + "_source": { + "foo": "rab" + } + } + ] +} +-------------------------------------------------- +// AUTOSENSE + +response: + +[source,js] +-------------------------------------------------- +{ + "docs": [ + { + "doc": { + "_id": "id", + "_ttl": null, + "_parent": null, + "_index": "index", + "_routing": null, + "_type": "type", + "_timestamp": null, + "_source": { + "field2": "_value", + "foo": "bar" + }, + "_ingest": { + "timestamp": "2016-01-04T23:53:27.186+0000" + } + } + }, + { + "doc": { + "_id": "id", + "_ttl": null, + "_parent": null, + "_index": "index", + "_routing": null, + "_type": "type", + "_timestamp": null, + "_source": { + "field2": "_value", + "foo": "rab" + }, + "_ingest": { + "timestamp": "2016-01-04T23:53:27.186+0000" + } + } + } + ] +} +-------------------------------------------------- + +It is often useful to see how each processor affects the ingest document +as it is passed through the pipeline. To see the intermediate results of +each processor in the simulat request, a `verbose` parameter may be added +to the request + +Here is an example verbose request and its response: + + +[source,js] +-------------------------------------------------- +POST _ingest/pipeline/_simulate?verbose +{ + "pipeline" : + { + "description": "_description", + "processors": [ + { + "set" : { + "field" : "field2", + "value" : "_value2" + } + }, + { + "set" : { + "field" : "field3", + "value" : "_value3" + } + } + ] + }, + "docs": [ + { + "_index": "index", + "_type": "type", + "_id": "id", + "_source": { + "foo": "bar" + } + }, + { + "_index": "index", + "_type": "type", + "_id": "id", + "_source": { + "foo": "rab" + } + } + ] +} +-------------------------------------------------- +// AUTOSENSE + +response: + +[source,js] +-------------------------------------------------- +{ + "docs": [ + { + "processor_results": [ + { + "processor_id": "processor[set]-0", + "doc": { + "_id": "id", + "_ttl": null, + "_parent": null, + "_index": "index", + "_routing": null, + "_type": "type", + "_timestamp": null, + "_source": { + "field2": "_value2", + "foo": "bar" + }, + "_ingest": { + "timestamp": "2016-01-05T00:02:51.383+0000" + } + } + }, + { + "processor_id": "processor[set]-1", + "doc": { + "_id": "id", + "_ttl": null, + "_parent": null, + "_index": "index", + "_routing": null, + "_type": "type", + "_timestamp": null, + "_source": { + "field3": "_value3", + "field2": "_value2", + "foo": "bar" + }, + "_ingest": { + "timestamp": "2016-01-05T00:02:51.383+0000" + } + } + } + ] + }, + { + "processor_results": [ + { + "processor_id": "processor[set]-0", + "doc": { + "_id": "id", + "_ttl": null, + "_parent": null, + "_index": "index", + "_routing": null, + "_type": "type", + "_timestamp": null, + "_source": { + "field2": "_value2", + "foo": "rab" + }, + "_ingest": { + "timestamp": "2016-01-05T00:02:51.384+0000" + } + } + }, + { + "processor_id": "processor[set]-1", + "doc": { + "_id": "id", + "_ttl": null, + "_parent": null, + "_index": "index", + "_routing": null, + "_type": "type", + "_timestamp": null, + "_source": { + "field3": "_value3", + "field2": "_value2", + "foo": "rab" + }, + "_ingest": { + "timestamp": "2016-01-05T00:02:51.384+0000" + } + } + } + ] + } + ] +} +-------------------------------------------------- diff --git a/plugins/ingest/src/test/resources/rest-api-spec/test/ingest/70_simulate.yaml b/plugins/ingest/src/test/resources/rest-api-spec/test/ingest/70_simulate.yaml index 932f67353cf..9947129788b 100644 --- a/plugins/ingest/src/test/resources/rest-api-spec/test/ingest/70_simulate.yaml +++ b/plugins/ingest/src/test/resources/rest-api-spec/test/ingest/70_simulate.yaml @@ -207,7 +207,7 @@ - length: { docs.0.processor_results.1.doc._source: 3 } - match: { docs.0.processor_results.1.doc._source.foo: "bar" } - match: { docs.0.processor_results.1.doc._source.field2: "_value" } - - match: { docs.0.processor_results.1.doc._source..field3: "third_val" } + - match: { docs.0.processor_results.1.doc._source.field3: "third_val" } - length: { docs.0.processor_results.1.doc._ingest: 1 } - is_true: docs.0.processor_results.1.doc._ingest.timestamp