diff --git a/docs/reference/eql/eql-search-api.asciidoc b/docs/reference/eql/eql-search-api.asciidoc index 882d3d6e94b..70257f35ded 100644 --- a/docs/reference/eql/eql-search-api.asciidoc +++ b/docs/reference/eql/eql-search-api.asciidoc @@ -18,16 +18,18 @@ event. [source,console] ---- PUT /my_index/_bulk?refresh -{"index":{"_index" : "my_index", "_id" : "1"}} -{ "@timestamp": "2020-12-06T11:04:05.000Z", "agent": { "id": "8a4f500d" }, "event": { "category": "process", "id": "edwCRnyD","sequence": 1 }, "process": { "name": "cmd.exe", "path": "C:\\Windows\\System32\\cmd.exe" } } -{"index":{"_index" : "my_index", "_id" : "2"}} -{ "@timestamp": "2020-12-06T11:04:07.000Z", "agent": { "id": "8a4f500d" }, "event": { "category": "file", "id": "dGCHwoeS", "sequence": 2 }, "file": { "accessed": "2020-12-07T11:07:08.000Z", "name": "cmd.exe", "path": "C:\\Windows\\System32\\cmd.exe", "type": "file", "size": 16384 }, "process": { "name": "cmd.exe", "path": "C:\\Windows\\System32\\cmd.exe" } } -{"index":{"_index" : "my_index", "_id" : "3"}} -{ "@timestamp": "2020-12-07T11:06:07.000Z", "agent": { "id": "8a4f500d" }, "event": { "category": "process", "id": "cMyt5SZ2", "sequence": 3 }, "process": { "name": "cmd.exe", "path": "C:\\Windows\\System32\\cmd.exe" } } -{"index":{"_index" : "my_index", "_id" : "4"}} -{ "@timestamp": "2020-12-07T11:07:08.000Z", "agent": { "id": "8a4f500d" }, "event": { "category": "file", "id": "bYA7gPay", "sequence": 4 }, "file": { "accessed": "2020-12-07T11:07:08.000Z", "name": "cmd.exe", "path": "C:\\Windows\\System32\\cmd.exe", "type": "file", "size": 16384 }, "process": { "name": "cmd.exe", "path": "C:\\Windows\\System32\\cmd.exe" } } -{"index":{"_index" : "my_index", "_id" : "5"}} -{ "@timestamp": "2020-12-07T11:07:09.000Z", "agent": { "id": "8a4f500d" }, "event": { "category": "process", "id": "aR3NWVOs", "sequence": 5 }, "process": { "name": "regsvr32.exe", "path": "C:\\Windows\\System32\\regsvr32.exe" } } +{"index":{ }} +{ "@timestamp": "2020-12-06T11:04:05.000Z", "agent": { "id": "8a4f500d" }, "event": { "category": "process", "id": "edwCRnyD", "sequence": 1 }, "process": { "name": "cmd.exe", "executable": "C:\\Windows\\System32\\cmd.exe" } } +{"index":{ }} +{ "@timestamp": "2020-12-06T11:04:07.000Z", "agent": { "id": "8a4f500d" }, "event": { "category": "file", "id": "dGCHwoeS", "sequence": 2 }, "file": { "accessed": "2020-12-07T11:07:08.000Z", "name": "cmd.exe", "path": "C:\\Windows\\System32\\cmd.exe", "type": "file", "size": 16384 }, "process": { "name": "cmd.exe", "executable": "C:\\Windows\\System32\\cmd.exe" } } +{"index":{ }} +{ "@timestamp": "2020-12-07T11:06:07.000Z", "agent": { "id": "8a4f500d" }, "event": { "category": "process", "id": "cMyt5SZ2", "sequence": 3 }, "process": { "name": "cmd.exe", "executable": "C:\\Windows\\System32\\cmd.exe" } } +{"index":{ }} +{ "@timestamp": "2020-12-07T11:07:08.000Z", "agent": { "id": "8a4f500d" }, "event": { "category": "file", "id": "bYA7gPay", "sequence": 4 }, "file": { "accessed": "2020-12-07T11:07:08.000Z", "name": "cmd.exe", "path": "C:\\Windows\\System32\\cmd.exe", "type": "file", "size": 16384 }, "process": { "name": "cmd.exe", "executable": "C:\\Windows\\System32\\cmd.exe" } } +{"index":{ }} +{ "@timestamp": "2020-12-07T11:07:09.000Z", "agent": { "id": "8a4f500d" }, "event": { "category": "process", "id": "aR3NWVOs", "sequence": 5 }, "process": { "name": "regsvr32.exe", "executable": "C:\\Windows\\System32\\regsvr32.exe" } } +{"index":{ }} +{ "@timestamp": "2020-12-07T11:07:10.000Z", "agent": { "id": "8a4f500d" }, "event": { "category": "process", "id": "GTSmSqgz0U", "sequence": 6, "type": "termination" }, "process": { "name": "regsvr32.exe", "executable": "C:\\Windows\\System32\\regsvr32.exe" } } ---- // TESTSETUP //// @@ -304,6 +306,7 @@ parameter. If both parameters are specified, only the query parameter is used. `id`:: + -- +(string) Identifier for the search. This search ID is only provided if one of the following conditions is met: @@ -433,10 +436,25 @@ Name of the index containing the event. `_id`:: (string) -(string) Unique identifier for the event. This ID is only unique within the index. +`_version`:: +(integer) +Version of the document (event). This version is incremented each time the document is +updated. + +`_seq_no`:: +(integer) +Sequence number assigned to the document (event). ++ +Sequence numbers are used to ensure an older version of a document +doesn’t overwrite a newer version. See <>. + +`_primary_term`:: +(integer) +Primary term assigned to the document. See <>. + `_score`:: (float) Positive 32-bit floating point number used to determine the relevance of the @@ -445,14 +463,6 @@ Positive 32-bit floating point number used to determine the relevance of the `_source`:: (object) Original JSON body passed for the event at index time. - -`sort`:: -(array) -Array of field values used to sort the event. -+ -By default, the first item in the array is the event's -<>, converted to milliseconds -since the https://en.wikipedia.org/wiki/Unix_time[Unix epoch]. ====== ===== @@ -483,14 +493,6 @@ Positive 32-bit floating point number used to determine the relevance of the `_source`:: (object) Original JSON body passed for the event at index time. - -`sort`:: -(array) -Array of field values used to sort the event. -+ -By default, the first item in the array is the event's -<>, converted to milliseconds -since the https://en.wikipedia.org/wiki/Unix_time[Unix epoch]. ===== ==== @@ -542,7 +544,7 @@ the events in ascending, lexicographic order. { "_index": "my_index", "_type": "_doc", - "_id": "2", + "_id": "fwGeywNsBl8Y9Ys1x51b", "_score": null, "_source": { "@timestamp": "2020-12-06T11:04:07.000Z", @@ -563,14 +565,14 @@ the events in ascending, lexicographic order. }, "process": { "name": "cmd.exe", - "path": "C:\\Windows\\System32\\cmd.exe" + "executable": "C:\\Windows\\System32\\cmd.exe" } } }, { "_index": "my_index", "_type": "_doc", - "_id": "4", + "_id": "AtOJ4UjUBAAx3XR5kcCM", "_score": null, "_source": { "@timestamp": "2020-12-07T11:07:08.000Z", @@ -591,7 +593,7 @@ the events in ascending, lexicographic order. }, "process": { "name": "cmd.exe", - "path": "C:\\Windows\\System32\\cmd.exe" + "executable": "C:\\Windows\\System32\\cmd.exe" } } } @@ -600,6 +602,8 @@ the events in ascending, lexicographic order. } ---- // TESTRESPONSE[s/"took": 6/"took": $body.took/] +// TESTRESPONSE[s/"_id": "fwGeywNsBl8Y9Ys1x51b"/"_id": $body.hits.events.0._id/] +// TESTRESPONSE[s/"_id": "AtOJ4UjUBAAx3XR5kcCM"/"_id": $body.hits.events.1._id/] [[eql-search-api-sequence-ex]] ===== Sequence query example @@ -618,7 +622,7 @@ that: + -- * An `event.category` of `process` -* A `process.path` that contains the substring `regsvr32` +* A `process.executable` that contains the substring `regsvr32` -- These events must also share the same `agent.id` value. @@ -630,11 +634,10 @@ GET /my_index/_eql/search "query": """ sequence by agent.id [ file where file.name == "cmd.exe" and agent.id != "my_user" ] - [ process where stringContains(process.path, "regsvr32") ] + [ process where stringContains(process.executable, "regsvr32") ] """ } ---- -// TEST[s/search/search\?filter_path\=\-\*\.sequences\.\*events\.\*fields/] The API returns the following response. The `hits.sequences.join_keys` property contains the shared `agent.id` value for each matching event. Matching events in @@ -667,7 +670,10 @@ the events in ascending, lexicographic order. { "_index": "my_index", "_type": "_doc", - "_id": "4", + "_id": "AtOJ4UjUBAAx3XR5kcCM", + "_version": 1, + "_seq_no": 3, + "_primary_term": 1, "_score": null, "_source": { "@timestamp": "2020-12-07T11:07:08.000Z", @@ -688,14 +694,17 @@ the events in ascending, lexicographic order. }, "process": { "name": "cmd.exe", - "path": "C:\\Windows\\System32\\cmd.exe" + "executable": "C:\\Windows\\System32\\cmd.exe" } } }, { "_index": "my_index", "_type": "_doc", - "_id": "5", + "_id": "yDwnGIJouOYGBzP0ZE9n", + "_version": 1, + "_seq_no": 4, + "_primary_term": 1, "_score": null, "_source": { "@timestamp": "2020-12-07T11:07:09.000Z", @@ -709,7 +718,7 @@ the events in ascending, lexicographic order. }, "process": { "name": "regsvr32.exe", - "path": "C:\\Windows\\System32\\regsvr32.exe" + "executable": "C:\\Windows\\System32\\regsvr32.exe" } } } @@ -720,4 +729,5 @@ the events in ascending, lexicographic order. } ---- // TESTRESPONSE[s/"took": 6/"took": $body.took/] -// TESTRESPONSE[skip: response format updated] +// TESTRESPONSE[s/"_id": "AtOJ4UjUBAAx3XR5kcCM"/"_id": $body.hits.sequences.0.events.0._id/] +// TESTRESPONSE[s/"_id": "yDwnGIJouOYGBzP0ZE9n"/"_id": $body.hits.sequences.0.events.1._id/] diff --git a/docs/reference/eql/search.asciidoc b/docs/reference/eql/search.asciidoc index 28140bb5746..71e3892d3c8 100644 --- a/docs/reference/eql/search.asciidoc +++ b/docs/reference/eql/search.asciidoc @@ -23,18 +23,18 @@ The following <> request adds some example log data to the [source,console] ---- PUT /sec_logs/_bulk?refresh -{"index":{"_index" : "sec_logs", "_id" : "1"}} -{ "@timestamp": "2020-12-06T11:04:05.000Z", "agent": { "id": "8a4f500d" }, "event": { "category": "process", "id": "edwCRnyD","sequence": 1 }, "process": { "name": "cmd.exe", "path": "C:\\Windows\\System32\\cmd.exe" } } -{"index":{"_index" : "sec_logs", "_id" : "2"}} -{ "@timestamp": "2020-12-06T11:04:07.000Z", "agent": { "id": "8a4f500d" }, "event": { "category": "file", "id": "dGCHwoeS", "sequence": 2 }, "file": { "accessed": "2020-12-07T11:07:08.000Z", "name": "cmd.exe", "path": "C:\\Windows\\System32\\cmd.exe", "type": "file", "size": 16384 }, "process": { "name": "cmd.exe", "path": "C:\\Windows\\System32\\cmd.exe" } } -{"index":{"_index" : "sec_logs", "_id" : "3"}} -{ "@timestamp": "2020-12-07T11:06:07.000Z", "agent": { "id": "8a4f500d" }, "event": { "category": "process", "id": "cMyt5SZ2", "sequence": 3 }, "process": { "name": "cmd.exe", "path": "C:\\Windows\\System32\\cmd.exe" } } -{"index":{"_index" : "sec_logs", "_id" : "4"}} -{ "@timestamp": "2020-12-07T11:07:08.000Z", "agent": { "id": "8a4f500d" }, "event": { "category": "file", "id": "bYA7gPay", "sequence": 4 }, "file": { "accessed": "2020-12-07T11:07:08.000Z", "name": "cmd.exe", "path": "C:\\Windows\\System32\\cmd.exe", "type": "file", "size": 16384 }, "process": { "name": "cmd.exe", "path": "C:\\Windows\\System32\\cmd.exe" } } -{"index":{"_index" : "sec_logs", "_id" : "5"}} -{ "@timestamp": "2020-12-07T11:07:09.000Z", "agent": { "id": "8a4f500d" }, "event": { "category": "process", "id": "aR3NWVOs", "sequence": 5 }, "process": { "name": "regsvr32.exe", "path": "C:\\Windows\\System32\\regsvr32.exe" } } -{"index":{"_index" : "sec_logs", "_id" : "6"}} -{ "@timestamp": "2020-12-07T11:07:10.000Z", "agent": { "id": "8a4f500d" }, "event": { "category": "process", "id": "GTSmSqgz0U", "sequence": 6, "type": "termination" }, "process": { "name": "regsvr32.exe", "path": "C:\\Windows\\System32\\regsvr32.exe" } } +{"index":{ }} +{ "@timestamp": "2020-12-06T11:04:05.000Z", "agent": { "id": "8a4f500d" }, "event": { "category": "process", "id": "edwCRnyD", "sequence": 1 }, "process": { "name": "cmd.exe", "executable": "C:\\Windows\\System32\\cmd.exe" } } +{"index":{ }} +{ "@timestamp": "2020-12-06T11:04:07.000Z", "agent": { "id": "8a4f500d" }, "event": { "category": "file", "id": "dGCHwoeS", "sequence": 2 }, "file": { "accessed": "2020-12-07T11:07:08.000Z", "name": "cmd.exe", "path": "C:\\Windows\\System32\\cmd.exe", "type": "file", "size": 16384 }, "process": { "name": "cmd.exe", "executable": "C:\\Windows\\System32\\cmd.exe" } } +{"index":{ }} +{ "@timestamp": "2020-12-07T11:06:07.000Z", "agent": { "id": "8a4f500d" }, "event": { "category": "process", "id": "cMyt5SZ2", "sequence": 3 }, "process": { "name": "cmd.exe", "executable": "C:\\Windows\\System32\\cmd.exe" } } +{"index":{ }} +{ "@timestamp": "2020-12-07T11:07:08.000Z", "agent": { "id": "8a4f500d" }, "event": { "category": "file", "id": "bYA7gPay", "sequence": 4 }, "file": { "accessed": "2020-12-07T11:07:08.000Z", "name": "cmd.exe", "path": "C:\\Windows\\System32\\cmd.exe", "type": "file", "size": 16384 }, "process": { "name": "cmd.exe", "executable": "C:\\Windows\\System32\\cmd.exe" } } +{"index":{ }} +{ "@timestamp": "2020-12-07T11:07:09.000Z", "agent": { "id": "8a4f500d" }, "event": { "category": "process", "id": "aR3NWVOs", "sequence": 5 }, "process": { "name": "regsvr32.exe", "executable": "C:\\Windows\\System32\\regsvr32.exe" } } +{"index":{ }} +{ "@timestamp": "2020-12-07T11:07:10.000Z", "agent": { "id": "8a4f500d" }, "event": { "category": "process", "id": "GTSmSqgz0U", "sequence": 6, "type": "termination" }, "process": { "name": "regsvr32.exe", "executable": "C:\\Windows\\System32\\regsvr32.exe" } } ---- // TESTSETUP @@ -88,7 +88,7 @@ https://en.wikipedia.org/wiki/Unix_time[Unix epoch], in ascending order. { "_index": "sec_logs", "_type": "_doc", - "_id": "1", + "_id": "OQmfCaduce8zoHT93o4H", "_score": null, "_source": { "@timestamp": "2020-12-06T11:04:05.000Z", @@ -102,14 +102,14 @@ https://en.wikipedia.org/wiki/Unix_time[Unix epoch], in ascending order. }, "process": { "name": "cmd.exe", - "path": "C:\\Windows\\System32\\cmd.exe" + "executable": "C:\\Windows\\System32\\cmd.exe" } } }, { "_index": "sec_logs", "_type": "_doc", - "_id": "3", + "_id": "xLkCaj4EujzdNSxfYLbO", "_score": null, "_source": { "@timestamp": "2020-12-07T11:06:07.000Z", @@ -123,7 +123,7 @@ https://en.wikipedia.org/wiki/Unix_time[Unix epoch], in ascending order. }, "process": { "name": "cmd.exe", - "path": "C:\\Windows\\System32\\cmd.exe" + "executable": "C:\\Windows\\System32\\cmd.exe" } } } @@ -132,6 +132,8 @@ https://en.wikipedia.org/wiki/Unix_time[Unix epoch], in ascending order. } ---- // TESTRESPONSE[s/"took": 60/"took": $body.took/] +// TESTRESPONSE[s/"_id": "OQmfCaduce8zoHT93o4H"/"_id": $body.hits.events.0._id/] +// TESTRESPONSE[s/"_id": "xLkCaj4EujzdNSxfYLbO"/"_id": $body.hits.events.1._id/] ==== [discrete] @@ -170,7 +172,6 @@ GET /sec_logs/_eql/search """ } ---- -// TEST[s/search/search\?filter_path\=\-\*\.sequences\.events\.\*fields/] The API returns the following response. Matching events in the `hits.sequences.events` property are sorted by @@ -195,7 +196,10 @@ the https://en.wikipedia.org/wiki/Unix_time[Unix epoch], in ascending order. { "_index": "sec_logs", "_type": "_doc", - "_id": "4", + "_id": "AtOJ4UjUBAAx3XR5kcCM", + "_version" : 1, + "_seq_no" : 3, + "_primary_term" : 1, "_score": null, "_source": { "@timestamp": "2020-12-07T11:07:08.000Z", @@ -216,14 +220,17 @@ the https://en.wikipedia.org/wiki/Unix_time[Unix epoch], in ascending order. }, "process": { "name": "cmd.exe", - "path": "C:\\Windows\\System32\\cmd.exe" + "executable": "C:\\Windows\\System32\\cmd.exe" } } }, { "_index": "sec_logs", "_type": "_doc", - "_id": "5", + "_id": "yDwnGIJouOYGBzP0ZE9n", + "_version" : 1, + "_seq_no" : 4, + "_primary_term" : 1, "_score": null, "_source": { "@timestamp": "2020-12-07T11:07:09.000Z", @@ -237,7 +244,7 @@ the https://en.wikipedia.org/wiki/Unix_time[Unix epoch], in ascending order. }, "process": { "name": "regsvr32.exe", - "path": "C:\\Windows\\System32\\regsvr32.exe" + "executable": "C:\\Windows\\System32\\regsvr32.exe" } } } @@ -248,7 +255,8 @@ the https://en.wikipedia.org/wiki/Unix_time[Unix epoch], in ascending order. } ---- // TESTRESPONSE[s/"took": 60/"took": $body.took/] -// TESTRESPONSE[skip: response format updated] +// TESTRESPONSE[s/"_id": "AtOJ4UjUBAAx3XR5kcCM"/"_id": $body.hits.sequences.0.events.0._id/] +// TESTRESPONSE[s/"_id": "yDwnGIJouOYGBzP0ZE9n"/"_id": $body.hits.sequences.0.events.1._id/] You can use the <> to constrain a sequence to a specified timespan. @@ -268,7 +276,6 @@ GET /sec_logs/_eql/search """ } ---- -// TEST[s/search/search\?filter_path\=\-\*\.sequences\.events\.\*fields/] You can further constrain matching event sequences using the <>. @@ -303,7 +310,6 @@ GET /sec_logs/_eql/search """ } ---- -// TEST[s/search/search\?filter_path\=\-\*\.sequences\.\*events\.\*fields/] The API returns the following response. The `hits.sequences.join_keys` property contains the shared `agent.id` value for each matching event. @@ -329,7 +335,10 @@ contains the shared `agent.id` value for each matching event. { "_index": "sec_logs", "_type": "_doc", - "_id": "4", + "_id": "AtOJ4UjUBAAx3XR5kcCM", + "_version": 1, + "_seq_no": 3, + "_primary_term": 1, "_score": null, "_source": { "@timestamp": "2020-12-07T11:07:08.000Z", @@ -350,14 +359,17 @@ contains the shared `agent.id` value for each matching event. }, "process": { "name": "cmd.exe", - "path": "C:\\Windows\\System32\\cmd.exe" + "executable": "C:\\Windows\\System32\\cmd.exe" } } }, { "_index": "sec_logs", "_type": "_doc", - "_id": "5", + "_id": "yDwnGIJouOYGBzP0ZE9n", + "_version": 1, + "_seq_no": 4, + "_primary_term": 1, "_score": null, "_source": { "@timestamp": "2020-12-07T11:07:09.000Z", @@ -371,7 +383,7 @@ contains the shared `agent.id` value for each matching event. }, "process": { "name": "regsvr32.exe", - "path": "C:\\Windows\\System32\\regsvr32.exe" + "executable": "C:\\Windows\\System32\\regsvr32.exe" } } } @@ -382,7 +394,8 @@ contains the shared `agent.id` value for each matching event. } ---- // TESTRESPONSE[s/"took": 60/"took": $body.took/] -// TESTRESPONSE[skip: response format updated] +// TESTRESPONSE[s/"_id": "AtOJ4UjUBAAx3XR5kcCM"/"_id": $body.hits.sequences.0.events.0._id/] +// TESTRESPONSE[s/"_id": "yDwnGIJouOYGBzP0ZE9n"/"_id": $body.hits.sequences.0.events.1._id/] You can use the <> to specify an expiration event for sequences. Matching sequences must end before this event. @@ -403,7 +416,6 @@ GET /sec_logs/_eql/search """ } ---- -// TEST[s/search/search\?filter_path\=\-\*\.sequences\.\*events\.\*fields/] ==== [discrete] @@ -480,7 +492,7 @@ GET /sec_logs/_eql/search { "tiebreaker_field": "event.id", "query": """ - process where process.name == "cmd.exe" and stringContains(process.path, "System32") + process where process.name == "cmd.exe" and stringContains(process.executable, "System32") """ } ---- @@ -501,10 +513,10 @@ The API returns the following response. "relation": "eq" }, "events": [ - { + { "_index": "sec_logs", "_type": "_doc", - "_id": "1", + "_id": "OQmfCaduce8zoHT93o4H", "_score": null, "_source": { "@timestamp": "2020-12-06T11:04:05.000Z", @@ -518,14 +530,14 @@ The API returns the following response. }, "process": { "name": "cmd.exe", - "path": "C:\\Windows\\System32\\cmd.exe" + "executable": "C:\\Windows\\System32\\cmd.exe" } } - }, - { + }, + { "_index": "sec_logs", "_type": "_doc", - "_id": "3", + "_id": "xLkCaj4EujzdNSxfYLbO", "_score": null, "_source": { "@timestamp": "2020-12-07T11:06:07.000Z", @@ -539,15 +551,17 @@ The API returns the following response. }, "process": { "name": "cmd.exe", - "path": "C:\\Windows\\System32\\cmd.exe" + "executable": "C:\\Windows\\System32\\cmd.exe" } - } } + } ] } } ---- // TESTRESPONSE[s/"took": 34/"took": $body.took/] +// TESTRESPONSE[s/"_id": "OQmfCaduce8zoHT93o4H"/"_id": $body.hits.events.0._id/] +// TESTRESPONSE[s/"_id": "xLkCaj4EujzdNSxfYLbO"/"_id": $body.hits.events.1._id/] ==== @@ -852,11 +866,11 @@ search API's `case_sensitive` parameter to toggle case sensitivity on or off. [%collapsible] ==== The following search request contains a query that matches `process` events -with a `process.path` containing `System32`. +with a `process.executable` containing `System32`. Because the `case_sensitive` parameter is `true`, this query only matches -`process.path` values containing `System32` with the exact same capitalization. -A `process.path` value containing `system32` or `SYSTEM32` would not match this +`process.executable` values containing `System32` with the exact same capitalization. +A `process.executable` value containing `system32` or `SYSTEM32` would not match this query. [source,console] @@ -866,7 +880,7 @@ GET /sec_logs/_eql/search "keep_on_completion": true, "case_sensitive": true, "query": """ - process where stringContains(process.path, "System32") + process where stringContains(process.executable, "System32") """ } ----