diff --git a/docs/build.gradle b/docs/build.gradle index 778a2f53044..60c1f55f15f 100644 --- a/docs/build.gradle +++ b/docs/build.gradle @@ -186,15 +186,42 @@ buildRestTests.setups['messages'] = ''' refresh: true body: | {"index":{"_id": "0"}} - {"message": "trying out Elasticsearch" } + {"message": "trying out Elasticsearch"} {"index":{"_id": "1"}} - {"message": "some message with the number 1" } + {"message": "some message with the number 1"} {"index":{"_id": "2"}} - {"message": "some message with the number 2" } + {"message": "some message with the number 2"} {"index":{"_id": "3"}} - {"message": "some message with the number 3" } + {"message": "some message with the number 3"} {"index":{"_id": "4"}} - {"message": "some message with the number 4" }''' + {"message": "some message with the number 4"}''' + +// Used for EQL +buildRestTests.setups['sec_logs'] = ''' + - do: + indices.create: + index: my-index-000001 + body: + settings: + number_of_shards: 1 + number_of_replicas: 1 + - do: + bulk: + index: my-index-000001 + refresh: true + body: | + {"index":{}} + {"@timestamp": "2020-12-06T11:04:05.000Z", "event": { "category": "process", "id": "edwCRnyD", "sequence": 1 }, "process": { "pid": 2012, "name": "cmd.exe", "executable": "C:\\\\Windows\\\\System32\\\\cmd.exe" }} + {"index":{}} + {"@timestamp": "2020-12-06T11:04:07.000Z", "event": { "category": "file", "id": "dGCHwoeS", "sequence": 2 }, "file": { "accessed": "2020-12-07T11:07:08.000Z", "name": "cmd.exe", "path": "C:\\\\Windows\\\\System32\\\\cmd.exe", "type": "file", "size": 16384 }, "process": { "pid": 2012, "name": "cmd.exe", "executable": "C:\\\\Windows\\\\System32\\\\cmd.exe" }} + {"index":{}} + {"@timestamp": "2020-12-07T11:06:07.000Z", "event": { "category": "process", "id": "cMyt5SZ2", "sequence": 3 }, "process": { "pid": 2012, "name": "cmd.exe", "executable": "C:\\\\Windows\\\\System32\\\\cmd.exe" } } + {"index":{}} + {"@timestamp": "2020-12-07T11:07:08.000Z", "event": { "category": "file", "id": "bYA7gPay", "sequence": 4 }, "file": { "accessed": "2020-12-07T11:07:08.000Z", "name": "cmd.exe", "path": "C:\\\\Windows\\\\System32\\\\cmd.exe", "type": "file", "size": 16384 }, "process": { "pid": 2012, "name": "cmd.exe", "executable": "C:\\\\Windows\\\\System32\\\\cmd.exe" } } + {"index":{}} + {"@timestamp": "2020-12-07T11:07:09.000Z", "event": { "category": "process", "id": "aR3NWVOs", "sequence": 5 }, "process": { "pid": 2012, "name": "regsvr32.exe", "executable": "C:\\\\Windows\\\\System32\\\\regsvr32.exe" }} + {"index":{}} + {"@timestamp": "2020-12-07T11:07:10.000Z", "event": { "category": "process", "id": "GTSmSqgz0U", "sequence": 6, "type": "termination" }, "process": { "pid": 2012, "name": "regsvr32.exe", "executable": "C:\\\\Windows\\\\System32\\\\regsvr32.exe" }}''' buildRestTests.setups['host'] = ''' # Fetch the http host. We use the host of the master because we know there will always be a master. diff --git a/docs/reference/eql/delete-async-eql-search-api.asciidoc b/docs/reference/eql/delete-async-eql-search-api.asciidoc index 32bc8207a8e..3acf8efc67a 100644 --- a/docs/reference/eql/delete-async-eql-search-api.asciidoc +++ b/docs/reference/eql/delete-async-eql-search-api.asciidoc @@ -27,12 +27,12 @@ DELETE /_eql/search/FkpMRkJGS1gzVDRlM3g4ZzMyRGlLbkEaTXlJZHdNT09TU2VTZVBoNDM3cFZM [[delete-async-eql-search-api-prereqs]] ==== {api-prereq-title} -See <>. +See <>. [[delete-async-eql-search-api-limitations]] ===== Limitations -See <>. +See <>. [[delete-async-eql-search-api-path-params]] ==== {api-path-parms-title} diff --git a/docs/reference/eql/eql-search-api.asciidoc b/docs/reference/eql/eql-search-api.asciidoc index 3cadf18560f..dd7e90dbced 100644 --- a/docs/reference/eql/eql-search-api.asciidoc +++ b/docs/reference/eql/eql-search-api.asciidoc @@ -14,35 +14,16 @@ Returns search results for an <> query. In {es}, EQL assumes each document in a data stream or index corresponds to an event. -//// [source,console] ---- -PUT /my-index-00001/_bulk?refresh -{"index":{ }} -{ "@timestamp": "2020-12-06T11:04:05.000Z", "agent": { "id": "8a4f500d" }, "event": { "category": "process", "id": "edwCRnyD", "sequence": 1 }, "process": { "name": "cmd.exe", "executable": "C:\\Windows\\System32\\cmd.exe" } } -{"index":{ }} -{ "@timestamp": "2020-12-06T11:04:07.000Z", "agent": { "id": "8a4f500d" }, "event": { "category": "file", "id": "dGCHwoeS", "sequence": 2 }, "file": { "accessed": "2020-12-07T11:07:08.000Z", "name": "cmd.exe", "path": "C:\\Windows\\System32\\cmd.exe", "type": "file", "size": 16384 }, "process": { "name": "cmd.exe", "executable": "C:\\Windows\\System32\\cmd.exe" } } -{"index":{ }} -{ "@timestamp": "2020-12-07T11:06:07.000Z", "agent": { "id": "8a4f500d" }, "event": { "category": "process", "id": "cMyt5SZ2", "sequence": 3 }, "process": { "name": "cmd.exe", "executable": "C:\\Windows\\System32\\cmd.exe" } } -{"index":{ }} -{ "@timestamp": "2020-12-07T11:07:08.000Z", "agent": { "id": "8a4f500d" }, "event": { "category": "file", "id": "bYA7gPay", "sequence": 4 }, "file": { "accessed": "2020-12-07T11:07:08.000Z", "name": "cmd.exe", "path": "C:\\Windows\\System32\\cmd.exe", "type": "file", "size": 16384 }, "process": { "name": "cmd.exe", "executable": "C:\\Windows\\System32\\cmd.exe" } } -{"index":{ }} -{ "@timestamp": "2020-12-07T11:07:09.000Z", "agent": { "id": "8a4f500d" }, "event": { "category": "process", "id": "aR3NWVOs", "sequence": 5 }, "process": { "name": "regsvr32.exe", "executable": "C:\\Windows\\System32\\regsvr32.exe" } } -{"index":{ }} -{ "@timestamp": "2020-12-07T11:07:10.000Z", "agent": { "id": "8a4f500d" }, "event": { "category": "process", "id": "GTSmSqgz0U", "sequence": 6, "type": "termination" }, "process": { "name": "regsvr32.exe", "executable": "C:\\Windows\\System32\\regsvr32.exe" } } ----- -// TESTSETUP -//// - -[source,console] ----- -GET /my-index-00001/_eql/search +GET /my-index-000001/_eql/search { "query": """ process where process.name = "regsvr32.exe" """ } ---- +// TEST[setup:sec_logs] [[eql-search-api-request]] ==== {api-request-title} @@ -54,12 +35,12 @@ GET /my-index-00001/_eql/search [[eql-search-api-prereqs]] ==== {api-prereq-title} -See <>. +See <>. [[eql-search-api-limitations]] ===== Limitations -See <>. +See <>. [[eql-search-api-path-params]] ==== {api-path-parms-title} @@ -163,6 +144,9 @@ Field containing the event classification, such as `process`, `file`, or Defaults to `event.category`, as defined in the {ecs-ref}/ecs-event.html[Elastic Common Schema (ECS)]. If a data stream or index does not contain the `event.category` field, this value is required. ++ +The event category field is typically mapped as a <> or +<> field. `fetch_size`:: (Optional, integer) @@ -275,6 +259,9 @@ does not contain the `@timestamp` field, this value is required. Events in the API response are sorted by this field's value, converted to milliseconds since the https://en.wikipedia.org/wiki/Unix_time[Unix epoch], in ascending order. + +The timestamp field is typically mapped as a <> or +<> field. -- [[eql-search-api-wait-for-completion-timeout]] @@ -506,17 +493,18 @@ The following EQL search request searches for events with an `event.category` of `file` that meet the following conditions: * A `file.name` of `cmd.exe` -* An `agent.id` other than `8a4f526c` +* An `process.pid` other than `2013` [source,console] ---- -GET /my-index-00001/_eql/search +GET /my-index-000001/_eql/search { "query": """ - file where (file.name == "cmd.exe" and agent.id != "8a4f526c") + file where (file.name == "cmd.exe" and process.pid != 2013) """ } ---- +// TEST[setup:sec_logs] // TEST[s/search/search\?filter_path\=\-\*\.events\.\*fields/] The API returns the following response. Matching events in the `hits.events` @@ -542,15 +530,12 @@ the events in ascending, lexicographic order. }, "events": [ { - "_index": "my-index-00001", + "_index": "my-index-000001", "_type": "_doc", "_id": "fwGeywNsBl8Y9Ys1x51b", "_score": null, "_source": { "@timestamp": "2020-12-06T11:04:07.000Z", - "agent": { - "id": "8a4f500d" - }, "event": { "category": "file", "id": "dGCHwoeS", @@ -565,20 +550,18 @@ the events in ascending, lexicographic order. }, "process": { "name": "cmd.exe", - "executable": "C:\\Windows\\System32\\cmd.exe" + "executable": "C:\\Windows\\System32\\cmd.exe", + "pid": 2012 } } }, { - "_index": "my-index-00001", + "_index": "my-index-000001", "_type": "_doc", "_id": "AtOJ4UjUBAAx3XR5kcCM", "_score": null, "_source": { "@timestamp": "2020-12-07T11:07:08.000Z", - "agent": { - "id": "8a4f500d" - }, "event": { "category": "file", "id": "bYA7gPay", @@ -593,7 +576,8 @@ the events in ascending, lexicographic order. }, "process": { "name": "cmd.exe", - "executable": "C:\\Windows\\System32\\cmd.exe" + "executable": "C:\\Windows\\System32\\cmd.exe", + "pid": 2012 } } } @@ -616,7 +600,7 @@ that: -- * An `event.category` of `file` * A `file.name` of `cmd.exe` -* An `agent.id` other than `8a4f526c` +* An `process.pid` other than `2013` -- . Followed by an event with: + @@ -625,29 +609,24 @@ that: * A `process.executable` that contains the substring `regsvr32` -- -These events must also share the same `agent.id` value. +These events must also share the same `process.pid` value. [source,console] ---- -GET /my-index-00001/_eql/search +GET /my-index-000001/_eql/search { "query": """ - sequence by agent.id - [ file where file.name == "cmd.exe" and agent.id != "8a4f526c" ] + sequence by process.pid + [ file where file.name == "cmd.exe" and process.pid != 2013 ] [ process where stringContains(process.executable, "regsvr32") ] """ } ---- +// TEST[setup:sec_logs] -The API returns the following response. The `hits.sequences.join_keys` property -contains the shared `agent.id` value for each matching event. Matching events in -the `hits.sequences.events` property are sorted by -<>, converted to milliseconds since -the https://en.wikipedia.org/wiki/Unix_time[Unix epoch], in ascending order. - -If two or more events share the same timestamp, the -<> field is used to sort -the events in ascending, lexicographic order. +The API returns the following response. Matching sequences are included in the +`hits.sequences` property. The `hits.sequences.join_keys` property contains the +shared `process.pid` value for each matching event. [source,console-result] ---- @@ -664,11 +643,11 @@ the events in ascending, lexicographic order. "sequences": [ { "join_keys": [ - "8a4f500d" + "2012" ], "events": [ { - "_index": "my-index-00001", + "_index": "my-index-000001", "_type": "_doc", "_id": "AtOJ4UjUBAAx3XR5kcCM", "_version": 1, @@ -677,9 +656,6 @@ the events in ascending, lexicographic order. "_score": null, "_source": { "@timestamp": "2020-12-07T11:07:08.000Z", - "agent": { - "id": "8a4f500d" - }, "event": { "category": "file", "id": "bYA7gPay", @@ -692,14 +668,15 @@ the events in ascending, lexicographic order. "type": "file", "size": 16384 }, - "process": { + "process": { "name": "cmd.exe", - "executable": "C:\\Windows\\System32\\cmd.exe" + "executable": "C:\\Windows\\System32\\cmd.exe", + "pid": 2012 } } }, { - "_index": "my-index-00001", + "_index": "my-index-000001", "_type": "_doc", "_id": "yDwnGIJouOYGBzP0ZE9n", "_version": 1, @@ -708,17 +685,15 @@ the events in ascending, lexicographic order. "_score": null, "_source": { "@timestamp": "2020-12-07T11:07:09.000Z", - "agent": { - "id": "8a4f500d" - }, "event": { "category": "process", "id": "aR3NWVOs", "sequence": 5 }, - "process": { + "process": { "name": "regsvr32.exe", - "executable": "C:\\Windows\\System32\\regsvr32.exe" + "executable": "C:\\Windows\\System32\\regsvr32.exe", + "pid": 2012 } } } diff --git a/docs/reference/eql/eql.asciidoc b/docs/reference/eql/eql.asciidoc new file mode 100644 index 00000000000..7231a721341 --- /dev/null +++ b/docs/reference/eql/eql.asciidoc @@ -0,0 +1,720 @@ +[role="xpack"] +[testenv="basic"] +[[eql]] += EQL search +++++ +EQL +++++ + +experimental::[] + +{eql-ref}/index.html[Event Query Language (EQL)] is a query language used for +event-based, time-series data, such as logs. + +[discrete] +[[eql-advantages]] +== Advantages of EQL + +* *EQL lets you express relationships between events.* + +Many query languages allow you to match only single events. EQL lets you match a +sequence of events across different event categories and time spans. + +* *EQL has a low learning curve.* + +EQL syntax looks like other query languages. It lets you write and read queries +intuitively, which makes for quick, iterative searching. + +* *We designed EQL for security use cases.* + +While you can use EQL for any event-based data, we created EQL for threat +hunting. EQL not only supports indicator of compromise (IOC) searching but +makes it easy to describe activity that goes beyond IOCs. + +[discrete] +[[eql-required-fields]] +== Required fields + +EQL assumes each document in a data stream or index corresponds to an event. To +search using EQL, each document in the searched data stream or index must +include a _timestamp_ field and an _event category_ field. + +{es} EQL uses the `@timestamp` and `event.category` fields from the +{ecs-ref}[Elastic Common Schema (ECS)] as the default timestamp and event +category fields. If your searched documents use a different timestamp or event +category field, you must specify it in the search request. See +<>. + +[discrete] +[[run-an-eql-search]] +== Run an EQL search + +You can use the <> to run an EQL search. + +The following request searches `my-index-000001` for events with an +`event.category` of `process` and a `process.name` of `cmd.exe`. Each document +in `my-index-000001` includes a `@timestamp` and `event.category` field. + +[source,console] +---- +GET /my-index-000001/_eql/search +{ + "query": """ + process where process.name == "cmd.exe" + """ +} +---- +// TEST[setup:sec_logs] +// TEST[s/search/search\?filter_path\=\-\*\.events\.\*fields/] + +The API returns the following response. Matching events are included in the +`hits.events` property. These events are sorted by timestamp, converted to +milliseconds since the https://en.wikipedia.org/wiki/Unix_time[Unix epoch], in +ascending order. + +[source,console-result] +---- +{ + "is_partial": false, + "is_running": false, + "took": 60, + "timed_out": false, + "hits": { + "total": { + "value": 2, + "relation": "eq" + }, + "events": [ + { + "_index": "my-index-000001", + "_type": "_doc", + "_id": "OQmfCaduce8zoHT93o4H", + "_score": null, + "_source": { + "@timestamp": "2020-12-06T11:04:05.000Z", + "event": { + "category": "process", + "id": "edwCRnyD", + "sequence": 1 + }, + "process": { + "name": "cmd.exe", + "executable": "C:\\Windows\\System32\\cmd.exe", + "pid": 2012 + } + } + }, + { + "_index": "my-index-000001", + "_type": "_doc", + "_id": "xLkCaj4EujzdNSxfYLbO", + "_score": null, + "_source": { + "@timestamp": "2020-12-07T11:06:07.000Z", + "event": { + "category": "process", + "id": "cMyt5SZ2", + "sequence": 3 + }, + "process": { + "name": "cmd.exe", + "executable": "C:\\Windows\\System32\\cmd.exe", + "pid": 2012 + } + } + } + ] + } +} +---- +// TESTRESPONSE[s/"took": 60/"took": $body.took/] +// TESTRESPONSE[s/"_id": "OQmfCaduce8zoHT93o4H"/"_id": $body.hits.events.0._id/] +// TESTRESPONSE[s/"_id": "xLkCaj4EujzdNSxfYLbO"/"_id": $body.hits.events.1._id/] + +[discrete] +[[eql-search-sequence]] +=== Search for a sequence of events + +You can use EQL's <> to search for an ordered +series of events. + +The following EQL search request matches a sequence that: + +. Starts with an event with: ++ +-- +* An `event.category` of `file` +* A `file.name` of `cmd.exe` +-- +. Followed by an event with: ++ +-- +* An `event.category` of `process` +* A `process.name` that contains the substring `regsvr32` +-- + +[source,console] +---- +GET /my-index-000001/_eql/search +{ + "query": """ + sequence + [ file where file.name == "cmd.exe" ] + [ process where stringContains(process.name, "regsvr32") ] + """ +} +---- +// TEST[setup:sec_logs] + +The API returns the following response. Matching sequences are included in the +`hits.sequences` property. + +[source,console-result] +---- +{ + "is_partial": false, + "is_running": false, + "took": 60, + "timed_out": false, + "hits": { + "total": { + "value": 1, + "relation": "eq" + }, + "sequences": [ + { + "events": [ + { + "_index": "my-index-000001", + "_type": "_doc", + "_id": "AtOJ4UjUBAAx3XR5kcCM", + "_version" : 1, + "_seq_no" : 3, + "_primary_term" : 1, + "_score": null, + "_source": { + "@timestamp": "2020-12-07T11:07:08.000Z", + "event": { + "category": "file", + "id": "bYA7gPay", + "sequence": 4 + }, + "file": { + "accessed": "2020-12-07T11:07:08.000Z", + "name": "cmd.exe", + "path": "C:\\Windows\\System32\\cmd.exe", + "type": "file", + "size": 16384 + }, + "process": { + "name": "cmd.exe", + "executable": "C:\\Windows\\System32\\cmd.exe", + "pid": 2012 + } + } + }, + { + "_index": "my-index-000001", + "_type": "_doc", + "_id": "yDwnGIJouOYGBzP0ZE9n", + "_version" : 1, + "_seq_no" : 4, + "_primary_term" : 1, + "_score": null, + "_source": { + "@timestamp": "2020-12-07T11:07:09.000Z", + "event": { + "category": "process", + "id": "aR3NWVOs", + "sequence": 5 + }, + "process": { + "name": "regsvr32.exe", + "executable": "C:\\Windows\\System32\\regsvr32.exe", + "pid": 2012 + } + } + } + ] + } + ] + } +} +---- +// TESTRESPONSE[s/"took": 60/"took": $body.took/] +// TESTRESPONSE[s/"_id": "AtOJ4UjUBAAx3XR5kcCM"/"_id": $body.hits.sequences.0.events.0._id/] +// TESTRESPONSE[s/"_id": "yDwnGIJouOYGBzP0ZE9n"/"_id": $body.hits.sequences.0.events.1._id/] + +You can use the <> to +constrain a sequence to a specified timespan. + +The following EQL search request adds `with maxspan=1h` to the previous query. +This ensures all events in a matching sequence occur within `1h` (one hour) of +the first event's timestamp. + +[source,console] +---- +GET /my-index-000001/_eql/search +{ + "query": """ + sequence with maxspan=1h + [ file where file.name == "cmd.exe" ] + [ process where stringContains(process.name, "regsvr32") ] + """ +} +---- +// TEST[setup:sec_logs] + +You can further constrain matching event sequences using the +<>. + +The following EQL search request adds `by process.pid` to each event item. This +ensures events matching the sequence share the same `process.pid` field value. + +[source,console] +---- +GET /my-index-000001/_eql/search +{ + "query": """ + sequence with maxspan=1h + [ file where file.name == "cmd.exe" ] by process.pid + [ process where stringContains(process.name, "regsvr32") ] by process.pid + """ +} +---- +// TEST[setup:sec_logs] + +Because the `process.pid` field is shared across all events in the sequence, it +can be included using `sequence by`. The following query is equivalent to the +previous one. + +[source,console] +---- +GET /my-index-000001/_eql/search +{ + "query": """ + sequence by process.pid with maxspan=1h + [ file where file.name == "cmd.exe" ] + [ process where stringContains(process.name, "regsvr32") ] + """ +} +---- +// TEST[setup:sec_logs] + +The API returns the following response. The `hits.sequences.join_keys` property +contains the shared `process.pid` value for each matching event. + +[source,console-result] +---- +{ + "is_partial": false, + "is_running": false, + "took": 60, + "timed_out": false, + "hits": { + "total": { + "value": 1, + "relation": "eq" + }, + "sequences": [ + { + "join_keys": [ + "2012" + ], + "events": [ + { + "_index": "my-index-000001", + "_type": "_doc", + "_id": "AtOJ4UjUBAAx3XR5kcCM", + "_version": 1, + "_seq_no": 3, + "_primary_term": 1, + "_score": null, + "_source": { + "@timestamp": "2020-12-07T11:07:08.000Z", + "event": { + "category": "file", + "id": "bYA7gPay", + "sequence": 4 + }, + "file": { + "accessed": "2020-12-07T11:07:08.000Z", + "name": "cmd.exe", + "path": "C:\\Windows\\System32\\cmd.exe", + "type": "file", + "size": 16384 + }, + "process": { + "name": "cmd.exe", + "executable": "C:\\Windows\\System32\\cmd.exe", + "pid": 2012 + } + } + }, + { + "_index": "my-index-000001", + "_type": "_doc", + "_id": "yDwnGIJouOYGBzP0ZE9n", + "_version": 1, + "_seq_no": 4, + "_primary_term": 1, + "_score": null, + "_source": { + "@timestamp": "2020-12-07T11:07:09.000Z", + "event": { + "category": "process", + "id": "aR3NWVOs", + "sequence": 5 + }, + "process": { + "name": "regsvr32.exe", + "executable": "C:\\Windows\\System32\\regsvr32.exe", + "pid": 2012 + } + } + } + ] + } + ] + } +} +---- +// TESTRESPONSE[s/"took": 60/"took": $body.took/] +// TESTRESPONSE[s/"_id": "AtOJ4UjUBAAx3XR5kcCM"/"_id": $body.hits.sequences.0.events.0._id/] +// TESTRESPONSE[s/"_id": "yDwnGIJouOYGBzP0ZE9n"/"_id": $body.hits.sequences.0.events.1._id/] + +You can use the <> to specify an expiration +event for sequences. Matching sequences must end before this event. + +The following request adds `until [ process where event.type == "termination" ]` +to the previous query. This ensures matching sequences end before a `process` +event with an `event.type` of `termination`. + +[source,console] +---- +GET /my-index-000001/_eql/search +{ + "query": """ + sequence by process.pid with maxspan=1h + [ file where file.name == "cmd.exe" ] + [ process where stringContains(process.name, "regsvr32") ] + until [ process where event.type == "termination" ] + """ +} +---- +// TEST[setup:sec_logs] + +[discrete] +[[specify-a-timestamp-or-event-category-field]] +=== Specify a timestamp or event category field + +By default, the EQL search API uses `@timestamp` and `event.category` as the +required timestamp and event category fields. If your searched documents use +a different timestamp or event category field, you must specify it in the search +request using the `timestamp_field` or `event_category_field` parameters. + +The event category field is typically mapped as a <> or +<> field. The timestamp field is typically +mapped as a <> or <> field. + +NOTE: You cannot use a <> field or the sub-fields of a `nested` +field as the timestamp or event category field. See <>. + +The following request uses the `timestamp_field` parameter to specify +`file.accessed` as the timestamp field. The request also uses the +`event_category_field` parameter to specify `file.type` as the event category +field. + +[source,console] +---- +GET /my-index-000001/_eql/search +{ + "timestamp_field": "file.accessed", + "event_category_field": "file.type", + "query": """ + file where (file.size > 1 and file.type == "file") + """ +} +---- +// TEST[setup:sec_logs] + +[discrete] +[[eql-search-filter-query-dsl]] +=== Filter using query DSL + +You can use the `filter` parameter to specify an additional query using +<>. This query filters the documents on which the EQL query +runs. + +The following request uses a `range` query to filter `my-index-000001` to only +documents with a `file.size` value greater than `1` but less than `1000000` +bytes. The EQL query in `query` parameter then runs on these filtered documents. + +[source,console] +---- +GET /my-index-000001/_eql/search +{ + "filter": { + "range" : { + "file.size" : { + "gte" : 1, + "lte" : 1000000 + } + } + }, + "query": """ + file where (file.type == "file" and file.name == "cmd.exe") + """ +} +---- +// TEST[setup:sec_logs] + +[discrete] +[[eql-search-case-sensitive]] +=== Run a case-sensitive EQL search + +By default, matching for EQL queries is case-insensitive. You can use the +`case_sensitive` parameter to toggle case sensitivity on or off. + +The following search request contains a query that matches `process` events +with a `process.executable` containing `System32`. + +Because `case_sensitive` is `true`, this query only matches `process.executable` +values containing `System32` with the exact same capitalization. A +`process.executable` value containing `system32` or `SYSTEM32` would not match +this query. + +[source,console] +---- +GET /my-index-000001/_eql/search +{ + "keep_on_completion": true, + "case_sensitive": true, + "query": """ + process where stringContains(process.executable, "System32") + """ +} +---- +// TEST[setup:sec_logs] + +[discrete] +[[eql-search-async]] +=== Run an async EQL search + +EQL searches are designed to run on large volumes of data quickly, often +returning results in milliseconds. For this reason, EQL searches are +_synchronous_ by default. The search request waits for complete results before +returning a response. + +However, complete results can take longer for searches across: + +* <> +* <> +* Many shards + +To avoid long waits, you can use the `wait_for_completion_timeout` parameter to +run an _asynchronous_, or _async_, EQL search. + +Set `wait_for_completion_timeout` to a duration you'd like to wait +for complete search results. If the search request does not finish within this +period, the search becomes async and returns a response that includes: + +* A search ID, which can be used to monitor the progress of the async search. +* An `is_partial` value of `true`, meaning the response does not contain + complete search results. +* An `is_running` value of `true`, meaning the search is async and ongoing. + +The async search continues to run in the background without blocking +other requests. + +The following request searches the `frozen-my-index-000001` index, which has been +<> for storage and is rarely searched. + +Because searches on frozen indices are expected to take longer to complete, the +request contains a `wait_for_completion_timeout` parameter value of `2s` (two +seconds). If the request does not return complete results in two seconds, the +search becomes async and returns a search ID. + +[source,console] +---- +GET /frozen-my-index-000001/_eql/search +{ + "wait_for_completion_timeout": "2s", + "query": """ + process where process.name == "cmd.exe" + """ +} +---- +// TEST[setup:sec_logs] +// TEST[s/frozen-my-index-000001/my-index-000001/] + +After two seconds, the request returns the following response. Note `is_partial` +and `is_running` properties are `true`, indicating an async search. + +[source,console-result] +---- +{ + "id": "FmNJRUZ1YWZCU3dHY1BIOUhaenVSRkEaaXFlZ3h4c1RTWFNocDdnY2FSaERnUTozNDE=", + "is_partial": true, + "is_running": true, + "took": 2000, + "timed_out": false, + "hits": ... +} +---- +// TESTRESPONSE[s/FmNJRUZ1YWZCU3dHY1BIOUhaenVSRkEaaXFlZ3h4c1RTWFNocDdnY2FSaERnUTozNDE=/$body.id/] +// TESTRESPONSE[s/"is_partial": true/"is_partial": $body.is_partial/] +// TESTRESPONSE[s/"is_running": true/"is_running": $body.is_running/] +// TESTRESPONSE[s/"took": 2000/"took": $body.took/] +// TESTRESPONSE[s/"hits": \.\.\./"hits": $body.hits/] + +You can use the the search ID and the <> to check the progress of an async search. + +The get async EQL search API also accepts a `wait_for_completion_timeout` +parameter. If ongoing search does not complete during this period, the response +returns an `is_partial` value of `true` and no search results. + +The following get async EQL search API request checks the progress of the +previous async EQL search. The request specifies a `wait_for_completion_timeout` +query parameter value of `2s` (two seconds). + +[source,console] +---- +GET /_eql/search/FmNJRUZ1YWZCU3dHY1BIOUhaenVSRkEaaXFlZ3h4c1RTWFNocDdnY2FSaERnUTozNDE=?wait_for_completion_timeout=2s +---- +// TEST[skip: no access to search ID] + +The request returns the following response. Note `is_partial` and `is_running` +are `false`, indicating the async search has finished and the search results +in the `hits` property are complete. + +[source,console-result] +---- +{ + "id": "FmNJRUZ1YWZCU3dHY1BIOUhaenVSRkEaaXFlZ3h4c1RTWFNocDdnY2FSaERnUTozNDE=", + "is_partial": false, + "is_running": false, + "took": 2000, + "timed_out": false, + "hits": ... +} +---- +// TESTRESPONSE[s/FmNJRUZ1YWZCU3dHY1BIOUhaenVSRkEaaXFlZ3h4c1RTWFNocDdnY2FSaERnUTozNDE=/$body.id/] +// TESTRESPONSE[s/"took": 2000/"took": $body.took/] +// TESTRESPONSE[s/"hits": \.\.\./"hits": $body.hits/] + +[discrete] +[[eql-search-store-async-eql-search]] +=== Change the search retention period + +By default, the EQL search API stores async searches for five days. After this +period, any searches and their results are deleted. You can use the `keep_alive` +parameter to change this retention period. + +In the following EQL search request, the `keep_alive` parameter is `2d` (two +days). If the search becomes async, its results +are stored on the cluster for two days. After two days, the async +search and its results are deleted, even if it's still ongoing. + +[source,console] +---- +GET /my-index-000001/_eql/search +{ + "keep_alive": "2d", + "wait_for_completion_timeout": "2s", + "query": """ + process where process.name == "cmd.exe" + """ +} +---- +// TEST[setup:sec_logs] + +You can use the <>'s +`keep_alive`parameter to later change the retention period. The new +retention period starts after the get request executes. + +The following request sets the `keep_alive` query parameter to `5d` (five days). +The async search and its results are deleted five days after the get request +executes. + +[source,console] +---- +GET /_eql/search/FmNJRUZ1YWZCU3dHY1BIOUhaenVSRkEaaXFlZ3h4c1RTWFNocDdnY2FSaERnUTozNDE=?keep_alive=5d +---- +// TEST[skip: no access to search ID] + +You can use the <> to +manually delete an async EQL search before the `keep_alive` period ends. If the +search is still ongoing, this cancels the search request. + +The following request deletes an async EQL search and its results. + +[source,console] +---- +DELETE /_eql/search/FmNJRUZ1YWZCU3dHY1BIOUhaenVSRkEaaXFlZ3h4c1RTWFNocDdnY2FSaERnUTozNDE=?keep_alive=5d +---- +// TEST[skip: no access to search ID] + +[discrete] +[[eql-search-store-sync-eql-search]] +=== Store synchronous EQL searches + +By default, the EQL search API only stores async searches that cannot be +completed within the period set by `wait_for_completion_timeout`. + +To save the results of searches that complete during this period, set the +`keep_on_completion` parameter to `true`. + +In the following search request, `keep_on_completion` is `true`. This means the +search results are stored on the cluster, even if the search completes within +the `2s` (two-second) period set by the `wait_for_completion_timeout` parameter. + +[source,console] +---- +GET /my-index-000001/_eql/search +{ + "keep_on_completion": true, + "wait_for_completion_timeout": "2s", + "query": """ + process where process.name == "cmd.exe" + """ +} +---- +// TEST[setup:sec_logs] + +The API returns the following response. A search ID is provided in the `id` +property. `is_partial` and `is_running` are `false`, indicating the EQL search +was synchronous and returned complete results in `hits`. + +[source,console-result] +---- +{ + "id": "FjlmbndxNmJjU0RPdExBTGg0elNOOEEaQk9xSjJBQzBRMldZa1VVQ2pPa01YUToxMDY=", + "is_partial": false, + "is_running": false, + "took": 52, + "timed_out": false, + "hits": ... +} +---- +// TESTRESPONSE[s/FjlmbndxNmJjU0RPdExBTGg0elNOOEEaQk9xSjJBQzBRMldZa1VVQ2pPa01YUToxMDY=/$body.id/] +// TESTRESPONSE[s/"took": 52/"took": $body.took/] +// TESTRESPONSE[s/"hits": \.\.\./"hits": $body.hits/] + +You can use the search ID and the <> to retrieve the same results later. + +[source,console] +---- +GET /_eql/search/FjlmbndxNmJjU0RPdExBTGg0elNOOEEaQk9xSjJBQzBRMldZa1VVQ2pPa01YUToxMDY= +---- +// TEST[skip: no access to search ID] + +Saved synchronous searches are still subject to the retention period set by the +`keep_alive` parameter. After this period, the search and its results are +deleted. + +You can also manually delete saved synchronous searches using the +<>. + +include::syntax.asciidoc[] +include::functions.asciidoc[] +include::pipes.asciidoc[] \ No newline at end of file diff --git a/docs/reference/eql/get-async-eql-search-api.asciidoc b/docs/reference/eql/get-async-eql-search-api.asciidoc index db2f0bf5ee1..92ef0665eb8 100644 --- a/docs/reference/eql/get-async-eql-search-api.asciidoc +++ b/docs/reference/eql/get-async-eql-search-api.asciidoc @@ -27,12 +27,12 @@ GET /_eql/search/FkpMRkJGS1gzVDRlM3g4ZzMyRGlLbkEaTXlJZHdNT09TU2VTZVBoNDM3cFZMUTo [[get-async-eql-search-api-prereqs]] ==== {api-prereq-title} -See <>. +See <>. [[get-async-eql-search-api-limitations]] ===== Limitations -See <>. +See <>. [[get-async-eql-search-api-path-params]] ==== {api-path-parms-title} diff --git a/docs/reference/eql/index.asciidoc b/docs/reference/eql/index.asciidoc deleted file mode 100644 index 427ac856af7..00000000000 --- a/docs/reference/eql/index.asciidoc +++ /dev/null @@ -1,61 +0,0 @@ -[role="xpack"] -[testenv="basic"] -[[eql]] -= EQL for event-based search -++++ -EQL -++++ - -experimental::[] - -{eql-ref}/index.html[Event Query Language (EQL)] is a query language used for -logs and other event-based data. - -You can use EQL in {es} to easily express relationships between events and -quickly match events with shared properties. You can use EQL and query -DSL together to better filter your searches. - -[discrete] -[[eql-advantages]] -=== Advantages of EQL - -* *EQL lets you express relationships between events.* + -Many query languages allow you to match only single events. EQL lets you match a -sequence of events across different event categories and time spans. - -* *EQL has a low learning curve.* + -EQL syntax looks like other query languages. It lets you write and read queries -intuitively, which makes for quick, iterative searching. - -* *We designed EQL for security use cases.* + -While you can use EQL for any event-based data, we created EQL for threat -hunting. EQL not only supports indicator of compromise (IOC) searching but -makes it easy to describe activity that goes beyond IOCs. - -[discrete] -[[when-to-use-eql]] -=== When to use EQL - -Consider using EQL if you: - -* Use {es} for threat hunting or other security use cases -* Search time-series data or logs, such as network or system logs -* Want an easy way to explore relationships between events - -[discrete] -[[eql-toc]] -=== In this section - -* <> -* <> -* <> -* <> -* <> -* <> - -include::requirements.asciidoc[] -include::search.asciidoc[] -include::syntax.asciidoc[] -include::functions.asciidoc[] -include::pipes.asciidoc[] -include::limitations.asciidoc[] diff --git a/docs/reference/eql/limitations.asciidoc b/docs/reference/eql/limitations.asciidoc deleted file mode 100644 index 872d9cce05b..00000000000 --- a/docs/reference/eql/limitations.asciidoc +++ /dev/null @@ -1,43 +0,0 @@ -[role="xpack"] -[testenv="basic"] -[[eql-limitations]] -== EQL limitations -++++ -Limitations -++++ - -experimental::[] - -[discrete] -[[eql-nested-fields]] -=== EQL search on nested fields is not supported - -You cannot use EQL to search the values of a <> field or the -sub-fields of a `nested` field. However, data streams and indices containing -`nested` field mappings are otherwise supported. - -[discrete] -[[eql-unsupported-syntax]] -=== Unsupported syntax - -{es} supports a subset of {eql-ref}/index.html[EQL syntax]. {es} cannot run EQL -queries that contain: - -* Array functions: -** {eql-ref}/functions.html#arrayContains[`arrayContains`] -** {eql-ref}/functions.html#arrayCount[`arrayCount`] -** {eql-ref}/functions.html#arraySearch[`arraySearch`] - -* {eql-ref}/joins.html[Joins] - -* {eql-ref}/basic-syntax.html#event-relationships[Lineage-related keywords]: -** `child of` -** `descendant of` -** `event of` - -* The following {eql-ref}/pipes.html[pipes]: -** {eql-ref}/pipes.html#count[`count`] -** {eql-ref}/pipes.html#filter[`filter`] -** {eql-ref}/pipes.html#sort[`sort`] -** {eql-ref}/pipes.html#unique[`unique`] -** {eql-ref}/pipes.html#unique-count[`unique_count`] diff --git a/docs/reference/eql/requirements.asciidoc b/docs/reference/eql/requirements.asciidoc deleted file mode 100644 index 1afd928dcc3..00000000000 --- a/docs/reference/eql/requirements.asciidoc +++ /dev/null @@ -1,43 +0,0 @@ -[role="xpack"] -[testenv="basic"] -[[eql-requirements]] -== EQL requirements -++++ -Requirements -++++ - -experimental::[] - -EQL is schema-less and works well with most common log formats. - -[TIP] -==== -While no schema is required to use EQL in {es}, we recommend the -{ecs-ref}[Elastic Common Schema (ECS)]. The <> is -designed to work with core ECS fields by default. -==== - -[discrete] -[[eql-required-fields]] -=== Required fields - -In {es}, EQL assumes each document in a data stream or index corresponds to an -event. - -To search a data stream or index using EQL, each document in the data stream or -index must contain the following field archetypes: - -Event category:: -A field containing the event classification, such as `process`, `file`, or -`network`. This is typically mapped as a <> field. - -Timestamp:: -A field containing the date and/or time the event occurred. This is typically -mapped as a <> or <> field. - -[NOTE] -==== -You cannot use a <> field data type or the sub-fields of a -`nested` field as the timestamp or event category field. See -<>. -==== diff --git a/docs/reference/eql/search.asciidoc b/docs/reference/eql/search.asciidoc deleted file mode 100644 index 17f45d0670f..00000000000 --- a/docs/reference/eql/search.asciidoc +++ /dev/null @@ -1,835 +0,0 @@ -[role="xpack"] -[testenv="basic"] -[[eql-search]] -== Run an EQL search - -experimental::[] - -To start using EQL in {es}, first ensure your event data meets -<>. You can then use the <> to search event data stored in one or more {es} data streams or -indices. The API requires a query written in {es}'s supported <>. - -To get started, ingest or add the data to an {es} data stream or index. - -The following <> request adds some example log data to the -`sec_logs` index. This log data follows the {ecs-ref}[Elastic Common Schema -(ECS)]. - -[source,console] ----- -PUT /sec_logs/_bulk?refresh -{"index":{ }} -{ "@timestamp": "2020-12-06T11:04:05.000Z", "agent": { "id": "8a4f500d" }, "event": { "category": "process", "id": "edwCRnyD", "sequence": 1 }, "process": { "name": "cmd.exe", "executable": "C:\\Windows\\System32\\cmd.exe" } } -{"index":{ }} -{ "@timestamp": "2020-12-06T11:04:07.000Z", "agent": { "id": "8a4f500d" }, "event": { "category": "file", "id": "dGCHwoeS", "sequence": 2 }, "file": { "accessed": "2020-12-07T11:07:08.000Z", "name": "cmd.exe", "path": "C:\\Windows\\System32\\cmd.exe", "type": "file", "size": 16384 }, "process": { "name": "cmd.exe", "executable": "C:\\Windows\\System32\\cmd.exe" } } -{"index":{ }} -{ "@timestamp": "2020-12-07T11:06:07.000Z", "agent": { "id": "8a4f500d" }, "event": { "category": "process", "id": "cMyt5SZ2", "sequence": 3 }, "process": { "name": "cmd.exe", "executable": "C:\\Windows\\System32\\cmd.exe" } } -{"index":{ }} -{ "@timestamp": "2020-12-07T11:07:08.000Z", "agent": { "id": "8a4f500d" }, "event": { "category": "file", "id": "bYA7gPay", "sequence": 4 }, "file": { "accessed": "2020-12-07T11:07:08.000Z", "name": "cmd.exe", "path": "C:\\Windows\\System32\\cmd.exe", "type": "file", "size": 16384 }, "process": { "name": "cmd.exe", "executable": "C:\\Windows\\System32\\cmd.exe" } } -{"index":{ }} -{ "@timestamp": "2020-12-07T11:07:09.000Z", "agent": { "id": "8a4f500d" }, "event": { "category": "process", "id": "aR3NWVOs", "sequence": 5 }, "process": { "name": "regsvr32.exe", "executable": "C:\\Windows\\System32\\regsvr32.exe" } } -{"index":{ }} -{ "@timestamp": "2020-12-07T11:07:10.000Z", "agent": { "id": "8a4f500d" }, "event": { "category": "process", "id": "GTSmSqgz0U", "sequence": 6, "type": "termination" }, "process": { "name": "regsvr32.exe", "executable": "C:\\Windows\\System32\\regsvr32.exe" } } ----- -// TESTSETUP - -[TIP] -===== -You also can set up {beats-ref}/getting-started.html[{beats}], such as -{auditbeat-ref}/auditbeat-installation-configuration.html[{auditbeat}] or -{winlogbeat-ref}/winlogbeat-installation-configuration.html[{winlogbeat}], to automatically -send and index your event data in {es}. See -{beats-ref}/getting-started.html[Getting started with {beats}]. -===== - -You can now use the EQL search API to search this index using an EQL query. - -The following request searches the `sec_logs` index using the EQL query -specified in the `query` parameter. The EQL query matches events with an -`event.category` of `process` that have a `process.name` of `cmd.exe`. - -[source,console] ----- -GET /sec_logs/_eql/search -{ - "query": """ - process where process.name == "cmd.exe" - """ -} ----- -// TEST[s/search/search\?filter_path\=\-\*\.events\.\*fields/] - -Because the `sec_log` index follows the ECS, you don't need to specify the -required <> fields. The request -uses the `event.category` and `@timestamp` fields by default. - -The API returns the following response containing the matching events. Events -in the response are sorted by timestamp, converted to milliseconds since the -https://en.wikipedia.org/wiki/Unix_time[Unix epoch], in ascending order. - -[source,console-result] ----- -{ - "is_partial": false, - "is_running": false, - "took": 60, - "timed_out": false, - "hits": { - "total": { - "value": 2, - "relation": "eq" - }, - "events": [ - { - "_index": "sec_logs", - "_type": "_doc", - "_id": "OQmfCaduce8zoHT93o4H", - "_score": null, - "_source": { - "@timestamp": "2020-12-06T11:04:05.000Z", - "agent": { - "id": "8a4f500d" - }, - "event": { - "category": "process", - "id": "edwCRnyD", - "sequence": 1 - }, - "process": { - "name": "cmd.exe", - "executable": "C:\\Windows\\System32\\cmd.exe" - } - } - }, - { - "_index": "sec_logs", - "_type": "_doc", - "_id": "xLkCaj4EujzdNSxfYLbO", - "_score": null, - "_source": { - "@timestamp": "2020-12-07T11:06:07.000Z", - "agent": { - "id": "8a4f500d" - }, - "event": { - "category": "process", - "id": "cMyt5SZ2", - "sequence": 3 - }, - "process": { - "name": "cmd.exe", - "executable": "C:\\Windows\\System32\\cmd.exe" - } - } - } - ] - } -} ----- -// TESTRESPONSE[s/"took": 60/"took": $body.took/] -// TESTRESPONSE[s/"_id": "OQmfCaduce8zoHT93o4H"/"_id": $body.hits.events.0._id/] -// TESTRESPONSE[s/"_id": "xLkCaj4EujzdNSxfYLbO"/"_id": $body.hits.events.1._id/] - -[discrete] -[[eql-search-sequence]] -=== Search for a sequence of events - -Many query languages allow you to match single events. However, EQL's -<> lets you match an ordered series of events. - -The following EQL search request matches a sequence that: - -. Starts with an event with: -+ --- -* An `event.category` of `file` -* A `file.name` of `cmd.exe` --- -. Followed by an event with: -+ --- -* An `event.category` of `process` -* A `process.name` that contains the substring `regsvr32` --- - -[source,console] ----- -GET /sec_logs/_eql/search -{ - "query": """ - sequence - [ file where file.name == "cmd.exe" ] - [ process where stringContains(process.name, "regsvr32") ] - """ -} ----- - -The API returns the following response. Matching events in -the `hits.sequences.events` property are sorted by -<>, converted to milliseconds since -the https://en.wikipedia.org/wiki/Unix_time[Unix epoch], in ascending order. - -[source,console-result] ----- -{ - "is_partial": false, - "is_running": false, - "took": 60, - "timed_out": false, - "hits": { - "total": { - "value": 1, - "relation": "eq" - }, - "sequences": [ - { - "events": [ - { - "_index": "sec_logs", - "_type": "_doc", - "_id": "AtOJ4UjUBAAx3XR5kcCM", - "_version" : 1, - "_seq_no" : 3, - "_primary_term" : 1, - "_score": null, - "_source": { - "@timestamp": "2020-12-07T11:07:08.000Z", - "agent": { - "id": "8a4f500d" - }, - "event": { - "category": "file", - "id": "bYA7gPay", - "sequence": 4 - }, - "file": { - "accessed": "2020-12-07T11:07:08.000Z", - "name": "cmd.exe", - "path": "C:\\Windows\\System32\\cmd.exe", - "type": "file", - "size": 16384 - }, - "process": { - "name": "cmd.exe", - "executable": "C:\\Windows\\System32\\cmd.exe" - } - } - }, - { - "_index": "sec_logs", - "_type": "_doc", - "_id": "yDwnGIJouOYGBzP0ZE9n", - "_version" : 1, - "_seq_no" : 4, - "_primary_term" : 1, - "_score": null, - "_source": { - "@timestamp": "2020-12-07T11:07:09.000Z", - "agent": { - "id": "8a4f500d" - }, - "event": { - "category": "process", - "id": "aR3NWVOs", - "sequence": 5 - }, - "process": { - "name": "regsvr32.exe", - "executable": "C:\\Windows\\System32\\regsvr32.exe" - } - } - } - ] - } - ] - } -} ----- -// TESTRESPONSE[s/"took": 60/"took": $body.took/] -// TESTRESPONSE[s/"_id": "AtOJ4UjUBAAx3XR5kcCM"/"_id": $body.hits.sequences.0.events.0._id/] -// TESTRESPONSE[s/"_id": "yDwnGIJouOYGBzP0ZE9n"/"_id": $body.hits.sequences.0.events.1._id/] - -You can use the <> to -constrain a sequence to a specified timespan. - -The following EQL search request adds `with maxspan=1h` to the previous query. -This ensures all events in a matching sequence occur within one hour (`1h`) of -the first event's timestamp. - -[source,console] ----- -GET /sec_logs/_eql/search -{ - "query": """ - sequence with maxspan=1h - [ file where file.name == "cmd.exe" ] - [ process where stringContains(process.name, "regsvr32") ] - """ -} ----- - -You can further constrain matching event sequences using the -<>. - -The following EQL search request adds `by agent.id` to each event item. This -ensures events matching the sequence share the same `agent.id` field value. - -[source,console] ----- -GET /sec_logs/_eql/search -{ - "query": """ - sequence with maxspan=1h - [ file where file.name == "cmd.exe" ] by agent.id - [ process where stringContains(process.name, "regsvr32") ] by agent.id - """ -} ----- - -Because the `agent.id` field is shared across all events in the sequence, it -can be included using `sequence by`. The following query is equivalent to the -prior one. - -[source,console] ----- -GET /sec_logs/_eql/search -{ - "query": """ - sequence by agent.id with maxspan=1h - [ file where file.name == "cmd.exe" ] - [ process where stringContains(process.name, "regsvr32") ] - """ -} ----- - -The API returns the following response. The `hits.sequences.join_keys` property -contains the shared `agent.id` value for each matching event. - -[source,console-result] ----- -{ - "is_partial": false, - "is_running": false, - "took": 60, - "timed_out": false, - "hits": { - "total": { - "value": 1, - "relation": "eq" - }, - "sequences": [ - { - "join_keys": [ - "8a4f500d" - ], - "events": [ - { - "_index": "sec_logs", - "_type": "_doc", - "_id": "AtOJ4UjUBAAx3XR5kcCM", - "_version": 1, - "_seq_no": 3, - "_primary_term": 1, - "_score": null, - "_source": { - "@timestamp": "2020-12-07T11:07:08.000Z", - "agent": { - "id": "8a4f500d" - }, - "event": { - "category": "file", - "id": "bYA7gPay", - "sequence": 4 - }, - "file": { - "accessed": "2020-12-07T11:07:08.000Z", - "name": "cmd.exe", - "path": "C:\\Windows\\System32\\cmd.exe", - "type": "file", - "size": 16384 - }, - "process": { - "name": "cmd.exe", - "executable": "C:\\Windows\\System32\\cmd.exe" - } - } - }, - { - "_index": "sec_logs", - "_type": "_doc", - "_id": "yDwnGIJouOYGBzP0ZE9n", - "_version": 1, - "_seq_no": 4, - "_primary_term": 1, - "_score": null, - "_source": { - "@timestamp": "2020-12-07T11:07:09.000Z", - "agent": { - "id": "8a4f500d" - }, - "event": { - "category": "process", - "id": "aR3NWVOs", - "sequence": 5 - }, - "process": { - "name": "regsvr32.exe", - "executable": "C:\\Windows\\System32\\regsvr32.exe" - } - } - } - ] - } - ] - } -} ----- -// TESTRESPONSE[s/"took": 60/"took": $body.took/] -// TESTRESPONSE[s/"_id": "AtOJ4UjUBAAx3XR5kcCM"/"_id": $body.hits.sequences.0.events.0._id/] -// TESTRESPONSE[s/"_id": "yDwnGIJouOYGBzP0ZE9n"/"_id": $body.hits.sequences.0.events.1._id/] - -You can use the <> to specify an expiration -event for sequences. Matching sequences must end before this event. - -The following request adds -`until [ process where event.type == "termination" ]` to the previous EQL query. -This ensures matching sequences end before a process termination event. - -[source,console] ----- -GET /sec_logs/_eql/search -{ - "query": """ - sequence by agent.id with maxspan=1h - [ file where file.name == "cmd.exe" ] - [ process where stringContains(process.name, "regsvr32") ] - until [ process where event.type == "termination" ] - """ -} ----- - -[discrete] -[[eql-search-specify-event-category-field]] -=== Specify an event category field - -By default, the EQL search API uses `event.category` as the -<>. You can use the -`event_category_field` parameter to specify another event category field. - -The following request specifies `file.type` as the event category -field. - -[source,console] ----- -GET /sec_logs/_eql/search -{ - "event_category_field": "file.type", - "query": """ - file where agent.id == "8a4f500d" - """ -} ----- - -[discrete] -[[eql-search-specify-timestamp-field]] -=== Specify a timestamp field - -By default, EQL searches use `@timestamp` as the <>. You can use the EQL search API's `timestamp_field` parameter -to specify another timestamp field. - -The following request specifies `file.accessed` as the event -timestamp field. - -[source,console] ----- -GET /sec_logs/_eql/search -{ - "timestamp_field": "file.accessed", - "query": """ - file where (file.size > 1 and file.type == "file") - """ -} ----- - -[discrete] -[[eql-search-specify-a-sort-tiebreaker]] -=== Specify a sort tiebreaker - -By default, the EQL search API sorts matching events in the search response by -timestamp. However, if two or more events share the same timestamp, a tiebreaker -field is used to sort the events in ascending, lexicographic order. - -The EQL search API uses `event.sequence` as the default tiebreaker field. You -can use the `tiebreaker_field` parameter to specify another field. - -The following request specifies `event.start` as the tiebreaker field. - -[source,console] ----- -GET /sec_logs/_eql/search -{ - "tiebreaker_field": "event.id", - "query": """ - process where process.name == "cmd.exe" and stringContains(process.executable, "System32") - """ -} ----- -// TEST[s/search/search\?filter_path\=\-\*\.events\.\*fields/] - -The API returns the following response. - -[source,console-result] ----- -{ - "is_partial": false, - "is_running": false, - "took": 34, - "timed_out": false, - "hits": { - "total": { - "value": 2, - "relation": "eq" - }, - "events": [ - { - "_index": "sec_logs", - "_type": "_doc", - "_id": "OQmfCaduce8zoHT93o4H", - "_score": null, - "_source": { - "@timestamp": "2020-12-06T11:04:05.000Z", - "agent": { - "id": "8a4f500d" - }, - "event": { - "category": "process", - "id": "edwCRnyD", - "sequence": 1 - }, - "process": { - "name": "cmd.exe", - "executable": "C:\\Windows\\System32\\cmd.exe" - } - } - }, - { - "_index": "sec_logs", - "_type": "_doc", - "_id": "xLkCaj4EujzdNSxfYLbO", - "_score": null, - "_source": { - "@timestamp": "2020-12-07T11:06:07.000Z", - "agent": { - "id": "8a4f500d" - }, - "event": { - "category": "process", - "id": "cMyt5SZ2", - "sequence": 3 - }, - "process": { - "name": "cmd.exe", - "executable": "C:\\Windows\\System32\\cmd.exe" - } - } - } - ] - } -} ----- -// TESTRESPONSE[s/"took": 34/"took": $body.took/] -// TESTRESPONSE[s/"_id": "OQmfCaduce8zoHT93o4H"/"_id": $body.hits.events.0._id/] -// TESTRESPONSE[s/"_id": "xLkCaj4EujzdNSxfYLbO"/"_id": $body.hits.events.1._id/] - - -[discrete] -[[eql-search-filter-query-dsl]] -=== Filter using query DSL - -You can use the EQL search API's `filter` parameter to specify an additional -query using <>. This query filters the documents on which -the EQL query runs. - -The following request uses a `range` query to filter the `sec_logs` -index down to only documents with a `file.size` value greater than `1` but less -than `1000000` bytes. The EQL query in `query` parameter then runs on these -filtered documents. - -[source,console] ----- -GET /sec_logs/_eql/search -{ - "filter": { - "range" : { - "file.size" : { - "gte" : 1, - "lte" : 1000000 - } - } - }, - "query": """ - file where (file.type == "file" and file.name == "cmd.exe") - """ -} ----- - -[discrete] -[[eql-search-async]] -=== Run an async EQL search - -EQL searches in {es} are designed to run on large volumes of data quickly, -often returning results in milliseconds. Because of this, the EQL search API -runs _synchronous_ searches by default. This means the search request waits for -complete results before returning a response. - -However, complete results can take longer for searches across: - -* <> -* <> -* Many shards - -To avoid long waits, you can use the EQL search API's -`wait_for_completion_timeout` parameter to run an _asynchronous_, or _async_, -search. - -Set the `wait_for_completion_timeout` parameter to a duration you'd like to wait -for complete search results. If the search request does not finish within this -period, the search becomes an async search. The EQL search -API returns a response that includes: - -* A search ID, which can be used to monitor the progress of the async search and - retrieve complete results when it finishes. -* An `is_partial` value of `true`, indicating the response does not contain - complete search results. -* An `is_running` value of `true`, indicating the search is async and ongoing. - -The async search continues to run in the background without blocking -other requests. - -The following request searches the `frozen_sec_logs` index, which has been -<> for storage and is rarely searched. - -Because searches on frozen indices are expected to take longer to complete, the -request contains a `wait_for_completion_timeout` parameter value of `2s` -(two seconds). - -If the request does not return complete results in two seconds, the search -becomes an async search and a search ID is returned. - -[source,console] ----- -GET /frozen_sec_logs/_eql/search -{ - "wait_for_completion_timeout": "2s", - "query": """ - process where process.name == "cmd.exe" - """ -} ----- -// TEST[s/frozen_sec_logs/sec_logs/] - -After two seconds, the request returns the following response. Note the -`is_partial` and `is_running` properties are `true`, indicating an ongoing async -search. - -[source,console-result] ----- -{ - "id": "FmNJRUZ1YWZCU3dHY1BIOUhaenVSRkEaaXFlZ3h4c1RTWFNocDdnY2FSaERnUTozNDE=", - "is_partial": true, - "is_running": true, - "took": 2000, - "timed_out": false, - "hits": ... -} ----- -// TESTRESPONSE[s/FmNJRUZ1YWZCU3dHY1BIOUhaenVSRkEaaXFlZ3h4c1RTWFNocDdnY2FSaERnUTozNDE=/$body.id/] -// TESTRESPONSE[s/"is_partial": true/"is_partial": $body.is_partial/] -// TESTRESPONSE[s/"is_running": true/"is_running": $body.is_running/] -// TESTRESPONSE[s/"took": 2000/"took": $body.took/] -// TESTRESPONSE[s/"hits": \.\.\./"hits": $body.hits/] - -You can use the the returned search ID and the <> to check the progress of an ongoing async search. - -The get async EQL search API also accepts a `wait_for_completion_timeout` query -parameter. Set the `wait_for_completion_timeout` parameter to a duration you'd -like to wait for complete search results. If the request does not complete -during this period, the response returns an `is_partial` value of `true` and no -search results. - -The following get async EQL search API request checks the progress of the -previous async EQL search. The request specifies a `wait_for_completion_timeout` -query parameter value of `2s` (two seconds). - -[source,console] ----- -GET /_eql/search/FmNJRUZ1YWZCU3dHY1BIOUhaenVSRkEaaXFlZ3h4c1RTWFNocDdnY2FSaERnUTozNDE=?wait_for_completion_timeout=2s ----- -// TEST[skip: no access to search ID] - -The request returns the following response. Note the `is_partial` and -`is_running` properties are `false`, indicating the async EQL search has -finished and the search results in the `hits` property are complete. - -[source,console-result] ----- -{ - "id": "FmNJRUZ1YWZCU3dHY1BIOUhaenVSRkEaaXFlZ3h4c1RTWFNocDdnY2FSaERnUTozNDE=", - "is_partial": false, - "is_running": false, - "took": 2000, - "timed_out": false, - "hits": ... -} ----- -// TESTRESPONSE[s/FmNJRUZ1YWZCU3dHY1BIOUhaenVSRkEaaXFlZ3h4c1RTWFNocDdnY2FSaERnUTozNDE=/$body.id/] -// TESTRESPONSE[s/"took": 2000/"took": $body.took/] -// TESTRESPONSE[s/"_index": "frozen_sec_logs"/"_index": "sec_logs"/] -// TESTRESPONSE[s/"hits": \.\.\./"hits": $body.hits/] - -[discrete] -[[eql-search-store-async-eql-search]] -=== Change the search retention period - -By default, the EQL search API only stores async searches and their results for -five days. After this period, any ongoing searches or saved results are deleted. - -You can use the EQL search API's `keep_alive` parameter to change the duration -of this period. - -In the following EQL search API request, the `keep_alive` parameter is `2d` (two -days). This means that if the search becomes async, its results -are stored on the cluster for two days. After two days, the async -search and its results are deleted, even if it's still ongoing. - -[source,console] ----- -GET /sec_logs/_eql/search -{ - "keep_alive": "2d", - "wait_for_completion_timeout": "2s", - "query": """ - process where process.name == "cmd.exe" - """ -} ----- - -You can use the <>'s -`keep_alive` query parameter to later change the retention period. The new -retention period starts after the get async EQL search API request executes. - -The following get async EQL search API request sets the `keep_alive` query -parameter to `5d` (five days). The async search and its results are deleted five -days after the get async EQL search API request executes. - -[source,console] ----- -GET /_eql/search/FmNJRUZ1YWZCU3dHY1BIOUhaenVSRkEaaXFlZ3h4c1RTWFNocDdnY2FSaERnUTozNDE=?keep_alive=5d ----- -// TEST[skip: no access to search ID] - -You can use the <> to -manually delete an async EQL search before the `keep_alive` period ends. If the -search is still ongoing, this cancels the search request. - -The following delete async EQL search API request deletes an async EQL search -and its results. - -[source,console] ----- -DELETE /_eql/search/FmNJRUZ1YWZCU3dHY1BIOUhaenVSRkEaaXFlZ3h4c1RTWFNocDdnY2FSaERnUTozNDE=?keep_alive=5d ----- -// TEST[skip: no access to search ID] - -[discrete] -[[eql-search-store-sync-eql-search]] -=== Store synchronous EQL searches - -By default, the EQL search API only stores async searches that cannot be -completed within the period set by the `wait_for_completion_timeout` parameter. - -To save the results of searches that complete during this period, set the -`keep_on_completion` parameter to `true`. - -In the following EQL search API request, the `keep_on_completion` parameter is -`true`. This means the search results are stored on the cluster, even if -the search completes within the `2s` (two-second) period set by the -`wait_for_completion_timeout` parameter. - -[source,console] ----- -GET /sec_logs/_eql/search -{ - "keep_on_completion": true, - "wait_for_completion_timeout": "2s", - "query": """ - process where process.name == "cmd.exe" - """ -} ----- - -The API returns the following response. Note that a search ID is provided in the -`id` property. The `is_partial` and `is_running` properties are `false`, -indicating the EQL search was synchronous and returned complete search results. - -[source,console-result] ----- -{ - "id": "FjlmbndxNmJjU0RPdExBTGg0elNOOEEaQk9xSjJBQzBRMldZa1VVQ2pPa01YUToxMDY=", - "is_partial": false, - "is_running": false, - "took": 52, - "timed_out": false, - "hits": ... -} ----- -// TESTRESPONSE[s/FjlmbndxNmJjU0RPdExBTGg0elNOOEEaQk9xSjJBQzBRMldZa1VVQ2pPa01YUToxMDY=/$body.id/] -// TESTRESPONSE[s/"took": 52/"took": $body.took/] -// TESTRESPONSE[s/"hits": \.\.\./"hits": $body.hits/] - -You can use the search ID and the <> to retrieve the same results later. - -[source,console] ----- -GET /_eql/search/FjlmbndxNmJjU0RPdExBTGg0elNOOEEaQk9xSjJBQzBRMldZa1VVQ2pPa01YUToxMDY= ----- -// TEST[skip: no access to search ID] - -Saved synchronous searches are still subject to the storage retention period set -by the `keep_alive` parameter. After this period, the search and its saved -results are deleted. - -You can also manually delete saved synchronous searches using the -<>. - -[discrete] -[[eql-search-case-sensitive]] -=== Run a case-sensitive EQL search - -By default, matching for EQL queries is case-insensitive. You can use the EQL -search API's `case_sensitive` parameter to toggle case sensitivity on or off. - -The following search request contains a query that matches `process` events -with a `process.executable` containing `System32`. - -Because the `case_sensitive` parameter is `true`, this query only matches -`process.executable` values containing `System32` with the exact same capitalization. -A `process.executable` value containing `system32` or `SYSTEM32` would not match this -query. - -[source,console] ----- -GET /sec_logs/_eql/search -{ - "keep_on_completion": true, - "case_sensitive": true, - "query": """ - process where stringContains(process.executable, "System32") - """ -} ----- diff --git a/docs/reference/eql/syntax.asciidoc b/docs/reference/eql/syntax.asciidoc index 7d72f07332a..579aee6fa8b 100644 --- a/docs/reference/eql/syntax.asciidoc +++ b/docs/reference/eql/syntax.asciidoc @@ -8,10 +8,7 @@ experimental::[] -[IMPORTANT] -==== -{es} supports a subset of EQL syntax. See <>. -==== +[IMPORTANT: {es} supports a subset of EQL syntax. See <. [discrete] [[eql-basic-syntax]] @@ -683,3 +680,43 @@ You can pass the output of a pipe to another pipe. This lets you use multiple pipes with a single query. For a list of supported pipes, see <>. + +[discrete] +[[eql-syntax-limitations]] +=== Limitations + +{es} EQL does not support the following features and syntax. + +[discrete] +[[eql-nested-fields]] +==== EQL search on nested fields + +You cannot use EQL to search the values of a <> field or the +sub-fields of a `nested` field. However, data streams and indices containing +`nested` field mappings are otherwise supported. + +[discrete] +[[eql-unsupported-syntax]] +==== Unsupported syntax + +{es} supports a subset of {eql-ref}/index.html[EQL syntax]. {es} cannot run EQL +queries that contain: + +* Array functions: +** {eql-ref}/functions.html#arrayContains[`arrayContains`] +** {eql-ref}/functions.html#arrayCount[`arrayCount`] +** {eql-ref}/functions.html#arraySearch[`arraySearch`] + +* {eql-ref}/joins.html[Joins] + +* {eql-ref}/basic-syntax.html#event-relationships[Lineage-related keywords]: +** `child of` +** `descendant of` +** `event of` + +* The following {eql-ref}/pipes.html[pipes]: +** {eql-ref}/pipes.html#count[`count`] +** {eql-ref}/pipes.html#filter[`filter`] +** {eql-ref}/pipes.html#sort[`sort`] +** {eql-ref}/pipes.html#unique[`unique`] +** {eql-ref}/pipes.html#unique-count[`unique_count`] diff --git a/docs/reference/index.asciidoc b/docs/reference/index.asciidoc index 27661caff42..8af5a03220c 100644 --- a/docs/reference/index.asciidoc +++ b/docs/reference/index.asciidoc @@ -28,7 +28,7 @@ include::search/search-your-data.asciidoc[] include::query-dsl.asciidoc[] -include::eql/index.asciidoc[] +include::eql/eql.asciidoc[] include::sql/index.asciidoc[] diff --git a/docs/reference/redirects.asciidoc b/docs/reference/redirects.asciidoc index 55a093814c6..34136b24741 100644 --- a/docs/reference/redirects.asciidoc +++ b/docs/reference/redirects.asciidoc @@ -959,6 +959,21 @@ See <>. See <>. +[role="exclude",id="eql-search"] +=== Run an EQL search + +See <>. + +[role="exclude",id="eql-limitations"] +=== EQL limitations + +See <>. + +[role="exclude",id="eql-requirements"] +=== EQL requirements + +See <>. + //// [role="exclude",id="search-request-body"] === Request body search