Merge pull request #15763 from talevy/fix/simulate

[Ingest] Fix/simulate

docs/plugins/ingest.asciidoc

@@ -631,3 +631,278 @@ The delete pipeline api deletes pipelines by id.
 DELETE _ingest/pipeline/my-pipeline-id
 --------------------------------------------------
 // AUTOSENSE
+
+==== Simulate pipeline API
+
+The simulate pipeline api executes a specific pipeline against
+the set of documents provided in the body of the request.
+
+A simulate request may call upon an existing pipeline to be executed
+against the provided documents, or supply a pipeline definition in
+the body of the request.
+
+Here is the structure of a simulate request with a provided pipeline:
+
+[source,js]
+--------------------------------------------------
+POST _ingest/pipeline/_simulate
+{
+  "pipeline" : {
+    // pipeline definition here
+  },
+  "docs" : [
+    { /** first document **/ },
+    { /** second document **/ },
+    // ...
+  ]
+}
+--------------------------------------------------
+
+Here is the structure of a simulate request against a pre-existing pipeline:
+
+[source,js]
+--------------------------------------------------
+POST _ingest/pipeline/my-pipeline-id/_simulate
+{
+  "docs" : [
+    { /** first document **/ },
+    { /** second document **/ },
+    // ...
+  ]
+}
+--------------------------------------------------
+
+
+Here is an example simulate request with a provided pipeline and its response:
+
+[source,js]
+--------------------------------------------------
+POST _ingest/pipeline/_simulate
+{
+  "pipeline" :
+  {
+    "description": "_description",
+    "processors": [
+      {
+        "set" : {
+          "field" : "field2",
+          "value" : "_value"
+        }
+      }
+    ]
+  },
+  "docs": [
+    {
+      "_index": "index",
+      "_type": "type",
+      "_id": "id",
+      "_source": {
+        "foo": "bar"
+      }
+    },
+    {
+      "_index": "index",
+      "_type": "type",
+      "_id": "id",
+      "_source": {
+        "foo": "rab"
+      }
+    }
+  ]
+}
+--------------------------------------------------
+// AUTOSENSE
+
+response:
+
+[source,js]
+--------------------------------------------------
+{
+   "docs": [
+      {
+         "doc": {
+            "_id": "id",
+            "_ttl": null,
+            "_parent": null,
+            "_index": "index",
+            "_routing": null,
+            "_type": "type",
+            "_timestamp": null,
+            "_source": {
+               "field2": "_value",
+               "foo": "bar"
+            },
+            "_ingest": {
+               "timestamp": "2016-01-04T23:53:27.186+0000"
+            }
+         }
+      },
+      {
+         "doc": {
+            "_id": "id",
+            "_ttl": null,
+            "_parent": null,
+            "_index": "index",
+            "_routing": null,
+            "_type": "type",
+            "_timestamp": null,
+            "_source": {
+               "field2": "_value",
+               "foo": "rab"
+            },
+            "_ingest": {
+               "timestamp": "2016-01-04T23:53:27.186+0000"
+            }
+         }
+      }
+   ]
+}
+--------------------------------------------------
+
+It is often useful to see how each processor affects the ingest document
+as it is passed through the pipeline. To see the intermediate results of
+each processor in the simulat request, a `verbose` parameter may be added
+to the request
+
+Here is an example verbose request and its response:
+
+
+[source,js]
+--------------------------------------------------
+POST _ingest/pipeline/_simulate?verbose
+{
+  "pipeline" :
+  {
+    "description": "_description",
+    "processors": [
+      {
+        "set" : {
+          "field" : "field2",
+          "value" : "_value2"
+        }
+      },
+      {
+        "set" : {
+          "field" : "field3",
+          "value" : "_value3"
+        }
+      }
+    ]
+  },
+  "docs": [
+    {
+      "_index": "index",
+      "_type": "type",
+      "_id": "id",
+      "_source": {
+        "foo": "bar"
+      }
+    },
+    {
+      "_index": "index",
+      "_type": "type",
+      "_id": "id",
+      "_source": {
+        "foo": "rab"
+      }
+    }
+  ]
+}
+--------------------------------------------------
+// AUTOSENSE
+
+response:
+
+[source,js]
+--------------------------------------------------
+{
+   "docs": [
+      {
+         "processor_results": [
+            {
+               "processor_id": "processor[set]-0",
+               "doc": {
+                  "_id": "id",
+                  "_ttl": null,
+                  "_parent": null,
+                  "_index": "index",
+                  "_routing": null,
+                  "_type": "type",
+                  "_timestamp": null,
+                  "_source": {
+                     "field2": "_value2",
+                     "foo": "bar"
+                  },
+                  "_ingest": {
+                     "timestamp": "2016-01-05T00:02:51.383+0000"
+                  }
+               }
+            },
+            {
+               "processor_id": "processor[set]-1",
+               "doc": {
+                  "_id": "id",
+                  "_ttl": null,
+                  "_parent": null,
+                  "_index": "index",
+                  "_routing": null,
+                  "_type": "type",
+                  "_timestamp": null,
+                  "_source": {
+                     "field3": "_value3",
+                     "field2": "_value2",
+                     "foo": "bar"
+                  },
+                  "_ingest": {
+                     "timestamp": "2016-01-05T00:02:51.383+0000"
+                  }
+               }
+            }
+         ]
+      },
+      {
+         "processor_results": [
+            {
+               "processor_id": "processor[set]-0",
+               "doc": {
+                  "_id": "id",
+                  "_ttl": null,
+                  "_parent": null,
+                  "_index": "index",
+                  "_routing": null,
+                  "_type": "type",
+                  "_timestamp": null,
+                  "_source": {
+                     "field2": "_value2",
+                     "foo": "rab"
+                  },
+                  "_ingest": {
+                     "timestamp": "2016-01-05T00:02:51.384+0000"
+                  }
+               }
+            },
+            {
+               "processor_id": "processor[set]-1",
+               "doc": {
+                  "_id": "id",
+                  "_ttl": null,
+                  "_parent": null,
+                  "_index": "index",
+                  "_routing": null,
+                  "_type": "type",
+                  "_timestamp": null,
+                  "_source": {
+                     "field3": "_value3",
+                     "field2": "_value2",
+                     "foo": "rab"
+                  },
+                  "_ingest": {
+                     "timestamp": "2016-01-05T00:02:51.384+0000"
+                  }
+               }
+            }
+         ]
+      }
+   ]
+}
+--------------------------------------------------

plugins/ingest/src/test/resources/rest-api-spec/test/ingest/70_simulate.yaml

@@ -207,7 +207,7 @@
   - length: { docs.0.processor_results.1.doc._source: 3 }
   - match: { docs.0.processor_results.1.doc._source.foo: "bar" }
   - match: { docs.0.processor_results.1.doc._source.field2: "_value" }
-  - match: { docs.0.processor_results.1.doc._source..field3: "third_val" }
+  - match: { docs.0.processor_results.1.doc._source.field3: "third_val" }
   - length: { docs.0.processor_results.1.doc._ingest: 1 }
   - is_true: docs.0.processor_results.1.doc._ingest.timestamp