Changes: * Moves sample data to reusable rest test * Combines EQL index, requirements, and run a search pages * Combines EQL syntax and limitations pages * Adds related redirects
This commit is contained in:
James Rodewig
GitHub
parent
2b6891b584
commit
a1c27b0833
|
|
@ -196,6 +196,33 @@ buildRestTests.setups['messages'] = '''
|
|||
{"index":{"_id": "4"}}
|
||||
{"message": "some message with the number 4"}'''
|
||||
|
||||
// Used for EQL
|
||||
buildRestTests.setups['sec_logs'] = '''
|
||||
- do:
|
||||
indices.create:
|
||||
index: my-index-000001
|
||||
body:
|
||||
settings:
|
||||
number_of_shards: 1
|
||||
number_of_replicas: 1
|
||||
- do:
|
||||
bulk:
|
||||
index: my-index-000001
|
||||
refresh: true
|
||||
body: |
|
||||
{"index":{}}
|
||||
{"@timestamp": "2020-12-06T11:04:05.000Z", "event": { "category": "process", "id": "edwCRnyD", "sequence": 1 }, "process": { "pid": 2012, "name": "cmd.exe", "executable": "C:\\\\Windows\\\\System32\\\\cmd.exe" }}
|
||||
{"index":{}}
|
||||
{"@timestamp": "2020-12-06T11:04:07.000Z", "event": { "category": "file", "id": "dGCHwoeS", "sequence": 2 }, "file": { "accessed": "2020-12-07T11:07:08.000Z", "name": "cmd.exe", "path": "C:\\\\Windows\\\\System32\\\\cmd.exe", "type": "file", "size": 16384 }, "process": { "pid": 2012, "name": "cmd.exe", "executable": "C:\\\\Windows\\\\System32\\\\cmd.exe" }}
|
||||
{"index":{}}
|
||||
{"@timestamp": "2020-12-07T11:06:07.000Z", "event": { "category": "process", "id": "cMyt5SZ2", "sequence": 3 }, "process": { "pid": 2012, "name": "cmd.exe", "executable": "C:\\\\Windows\\\\System32\\\\cmd.exe" } }
|
||||
{"index":{}}
|
||||
{"@timestamp": "2020-12-07T11:07:08.000Z", "event": { "category": "file", "id": "bYA7gPay", "sequence": 4 }, "file": { "accessed": "2020-12-07T11:07:08.000Z", "name": "cmd.exe", "path": "C:\\\\Windows\\\\System32\\\\cmd.exe", "type": "file", "size": 16384 }, "process": { "pid": 2012, "name": "cmd.exe", "executable": "C:\\\\Windows\\\\System32\\\\cmd.exe" } }
|
||||
{"index":{}}
|
||||
{"@timestamp": "2020-12-07T11:07:09.000Z", "event": { "category": "process", "id": "aR3NWVOs", "sequence": 5 }, "process": { "pid": 2012, "name": "regsvr32.exe", "executable": "C:\\\\Windows\\\\System32\\\\regsvr32.exe" }}
|
||||
{"index":{}}
|
||||
{"@timestamp": "2020-12-07T11:07:10.000Z", "event": { "category": "process", "id": "GTSmSqgz0U", "sequence": 6, "type": "termination" }, "process": { "pid": 2012, "name": "regsvr32.exe", "executable": "C:\\\\Windows\\\\System32\\\\regsvr32.exe" }}'''
|
||||
|
||||
buildRestTests.setups['host'] = '''
|
||||
# Fetch the http host. We use the host of the master because we know there will always be a master.
|
||||
- do:
|
||||
|
|
|
|||
|
|
@ -27,12 +27,12 @@ DELETE /_eql/search/FkpMRkJGS1gzVDRlM3g4ZzMyRGlLbkEaTXlJZHdNT09TU2VTZVBoNDM3cFZM
|
|||
[[delete-async-eql-search-api-prereqs]]
|
||||
==== {api-prereq-title}
|
||||
|
||||
See <<eql-requirements,EQL requirements>>.
|
||||
See <<eql-required-fields>>.
|
||||
|
||||
[[delete-async-eql-search-api-limitations]]
|
||||
===== Limitations
|
||||
|
||||
See <<eql-limitations,EQL limitations>>.
|
||||
See <<eql-syntax-limitations,EQL limitations>>.
|
||||
|
||||
[[delete-async-eql-search-api-path-params]]
|
||||
==== {api-path-parms-title}
|
||||
|
|
|
|||
|
|
@ -14,35 +14,16 @@ Returns search results for an <<eql,Event Query Language (EQL)>> query.
|
|||
In {es}, EQL assumes each document in a data stream or index corresponds to an
|
||||
event.
|
||||
|
||||
////
|
||||
[source,console]
|
||||
----
|
||||
PUT /my-index-00001/_bulk?refresh
|
||||
{"index":{ }}
|
||||
{ "@timestamp": "2020-12-06T11:04:05.000Z", "agent": { "id": "8a4f500d" }, "event": { "category": "process", "id": "edwCRnyD", "sequence": 1 }, "process": { "name": "cmd.exe", "executable": "C:\\Windows\\System32\\cmd.exe" } }
|
||||
{"index":{ }}
|
||||
{ "@timestamp": "2020-12-06T11:04:07.000Z", "agent": { "id": "8a4f500d" }, "event": { "category": "file", "id": "dGCHwoeS", "sequence": 2 }, "file": { "accessed": "2020-12-07T11:07:08.000Z", "name": "cmd.exe", "path": "C:\\Windows\\System32\\cmd.exe", "type": "file", "size": 16384 }, "process": { "name": "cmd.exe", "executable": "C:\\Windows\\System32\\cmd.exe" } }
|
||||
{"index":{ }}
|
||||
{ "@timestamp": "2020-12-07T11:06:07.000Z", "agent": { "id": "8a4f500d" }, "event": { "category": "process", "id": "cMyt5SZ2", "sequence": 3 }, "process": { "name": "cmd.exe", "executable": "C:\\Windows\\System32\\cmd.exe" } }
|
||||
{"index":{ }}
|
||||
{ "@timestamp": "2020-12-07T11:07:08.000Z", "agent": { "id": "8a4f500d" }, "event": { "category": "file", "id": "bYA7gPay", "sequence": 4 }, "file": { "accessed": "2020-12-07T11:07:08.000Z", "name": "cmd.exe", "path": "C:\\Windows\\System32\\cmd.exe", "type": "file", "size": 16384 }, "process": { "name": "cmd.exe", "executable": "C:\\Windows\\System32\\cmd.exe" } }
|
||||
{"index":{ }}
|
||||
{ "@timestamp": "2020-12-07T11:07:09.000Z", "agent": { "id": "8a4f500d" }, "event": { "category": "process", "id": "aR3NWVOs", "sequence": 5 }, "process": { "name": "regsvr32.exe", "executable": "C:\\Windows\\System32\\regsvr32.exe" } }
|
||||
{"index":{ }}
|
||||
{ "@timestamp": "2020-12-07T11:07:10.000Z", "agent": { "id": "8a4f500d" }, "event": { "category": "process", "id": "GTSmSqgz0U", "sequence": 6, "type": "termination" }, "process": { "name": "regsvr32.exe", "executable": "C:\\Windows\\System32\\regsvr32.exe" } }
|
||||
----
|
||||
// TESTSETUP
|
||||
////
|
||||
|
||||
[source,console]
|
||||
----
|
||||
GET /my-index-00001/_eql/search
|
||||
GET /my-index-000001/_eql/search
|
||||
{
|
||||
"query": """
|
||||
process where process.name = "regsvr32.exe"
|
||||
"""
|
||||
}
|
||||
----
|
||||
// TEST[setup:sec_logs]
|
||||
|
||||
[[eql-search-api-request]]
|
||||
==== {api-request-title}
|
||||
|
|
@ -54,12 +35,12 @@ GET /my-index-00001/_eql/search
|
|||
[[eql-search-api-prereqs]]
|
||||
==== {api-prereq-title}
|
||||
|
||||
See <<eql-requirements,EQL requirements>>.
|
||||
See <<eql-required-fields>>.
|
||||
|
||||
[[eql-search-api-limitations]]
|
||||
===== Limitations
|
||||
|
||||
See <<eql-limitations,EQL limitations>>.
|
||||
See <<eql-syntax-limitations,EQL limitations>>.
|
||||
|
||||
[[eql-search-api-path-params]]
|
||||
==== {api-path-parms-title}
|
||||
|
|
@ -163,6 +144,9 @@ Field containing the event classification, such as `process`, `file`, or
|
|||
Defaults to `event.category`, as defined in the {ecs-ref}/ecs-event.html[Elastic
|
||||
Common Schema (ECS)]. If a data stream or index does not contain the
|
||||
`event.category` field, this value is required.
|
||||
+
|
||||
The event category field is typically mapped as a <<keyword,`keyword`>> or
|
||||
<<constant-keyword,constant keyword>> field.
|
||||
|
||||
`fetch_size`::
|
||||
(Optional, integer)
|
||||
|
|
@ -275,6 +259,9 @@ does not contain the `@timestamp` field, this value is required.
|
|||
Events in the API response are sorted by this field's value, converted to
|
||||
milliseconds since the https://en.wikipedia.org/wiki/Unix_time[Unix epoch], in
|
||||
ascending order.
|
||||
|
||||
The timestamp field is typically mapped as a <<date,`date`>> or
|
||||
<<date_nanos,`date_nanos`>> field.
|
||||
--
|
||||
|
||||
[[eql-search-api-wait-for-completion-timeout]]
|
||||
|
|
@ -506,17 +493,18 @@ The following EQL search request searches for events with an `event.category` of
|
|||
`file` that meet the following conditions:
|
||||
|
||||
* A `file.name` of `cmd.exe`
|
||||
* An `agent.id` other than `8a4f526c`
|
||||
* An `process.pid` other than `2013`
|
||||
|
||||
[source,console]
|
||||
----
|
||||
GET /my-index-00001/_eql/search
|
||||
GET /my-index-000001/_eql/search
|
||||
{
|
||||
"query": """
|
||||
file where (file.name == "cmd.exe" and agent.id != "8a4f526c")
|
||||
file where (file.name == "cmd.exe" and process.pid != 2013)
|
||||
"""
|
||||
}
|
||||
----
|
||||
// TEST[setup:sec_logs]
|
||||
// TEST[s/search/search\?filter_path\=\-\*\.events\.\*fields/]
|
||||
|
||||
The API returns the following response. Matching events in the `hits.events`
|
||||
|
|
@ -542,15 +530,12 @@ the events in ascending, lexicographic order.
|
|||
},
|
||||
"events": [
|
||||
{
|
||||
"_index": "my-index-00001",
|
||||
"_index": "my-index-000001",
|
||||
"_type": "_doc",
|
||||
"_id": "fwGeywNsBl8Y9Ys1x51b",
|
||||
"_score": null,
|
||||
"_source": {
|
||||
"@timestamp": "2020-12-06T11:04:07.000Z",
|
||||
"agent": {
|
||||
"id": "8a4f500d"
|
||||
},
|
||||
"event": {
|
||||
"category": "file",
|
||||
"id": "dGCHwoeS",
|
||||
|
|
@ -565,20 +550,18 @@ the events in ascending, lexicographic order.
|
|||
},
|
||||
"process": {
|
||||
"name": "cmd.exe",
|
||||
"executable": "C:\\Windows\\System32\\cmd.exe"
|
||||
"executable": "C:\\Windows\\System32\\cmd.exe",
|
||||
"pid": 2012
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"_index": "my-index-00001",
|
||||
"_index": "my-index-000001",
|
||||
"_type": "_doc",
|
||||
"_id": "AtOJ4UjUBAAx3XR5kcCM",
|
||||
"_score": null,
|
||||
"_source": {
|
||||
"@timestamp": "2020-12-07T11:07:08.000Z",
|
||||
"agent": {
|
||||
"id": "8a4f500d"
|
||||
},
|
||||
"event": {
|
||||
"category": "file",
|
||||
"id": "bYA7gPay",
|
||||
|
|
@ -593,7 +576,8 @@ the events in ascending, lexicographic order.
|
|||
},
|
||||
"process": {
|
||||
"name": "cmd.exe",
|
||||
"executable": "C:\\Windows\\System32\\cmd.exe"
|
||||
"executable": "C:\\Windows\\System32\\cmd.exe",
|
||||
"pid": 2012
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -616,7 +600,7 @@ that:
|
|||
--
|
||||
* An `event.category` of `file`
|
||||
* A `file.name` of `cmd.exe`
|
||||
* An `agent.id` other than `8a4f526c`
|
||||
* An `process.pid` other than `2013`
|
||||
--
|
||||
. Followed by an event with:
|
||||
+
|
||||
|
|
@ -625,29 +609,24 @@ that:
|
|||
* A `process.executable` that contains the substring `regsvr32`
|
||||
--
|
||||
|
||||
These events must also share the same `agent.id` value.
|
||||
These events must also share the same `process.pid` value.
|
||||
|
||||
[source,console]
|
||||
----
|
||||
GET /my-index-00001/_eql/search
|
||||
GET /my-index-000001/_eql/search
|
||||
{
|
||||
"query": """
|
||||
sequence by agent.id
|
||||
[ file where file.name == "cmd.exe" and agent.id != "8a4f526c" ]
|
||||
sequence by process.pid
|
||||
[ file where file.name == "cmd.exe" and process.pid != 2013 ]
|
||||
[ process where stringContains(process.executable, "regsvr32") ]
|
||||
"""
|
||||
}
|
||||
----
|
||||
// TEST[setup:sec_logs]
|
||||
|
||||
The API returns the following response. The `hits.sequences.join_keys` property
|
||||
contains the shared `agent.id` value for each matching event. Matching events in
|
||||
the `hits.sequences.events` property are sorted by
|
||||
<<eql-search-api-timestamp-field,timestamp>>, converted to milliseconds since
|
||||
the https://en.wikipedia.org/wiki/Unix_time[Unix epoch], in ascending order.
|
||||
|
||||
If two or more events share the same timestamp, the
|
||||
<<eql-search-api-tiebreaker-field,`tiebreaker_field`>> field is used to sort
|
||||
the events in ascending, lexicographic order.
|
||||
The API returns the following response. Matching sequences are included in the
|
||||
`hits.sequences` property. The `hits.sequences.join_keys` property contains the
|
||||
shared `process.pid` value for each matching event.
|
||||
|
||||
[source,console-result]
|
||||
----
|
||||
|
|
@ -664,11 +643,11 @@ the events in ascending, lexicographic order.
|
|||
"sequences": [
|
||||
{
|
||||
"join_keys": [
|
||||
"8a4f500d"
|
||||
"2012"
|
||||
],
|
||||
"events": [
|
||||
{
|
||||
"_index": "my-index-00001",
|
||||
"_index": "my-index-000001",
|
||||
"_type": "_doc",
|
||||
"_id": "AtOJ4UjUBAAx3XR5kcCM",
|
||||
"_version": 1,
|
||||
|
|
@ -677,9 +656,6 @@ the events in ascending, lexicographic order.
|
|||
"_score": null,
|
||||
"_source": {
|
||||
"@timestamp": "2020-12-07T11:07:08.000Z",
|
||||
"agent": {
|
||||
"id": "8a4f500d"
|
||||
},
|
||||
"event": {
|
||||
"category": "file",
|
||||
"id": "bYA7gPay",
|
||||
|
|
@ -694,12 +670,13 @@ the events in ascending, lexicographic order.
|
|||
},
|
||||
"process": {
|
||||
"name": "cmd.exe",
|
||||
"executable": "C:\\Windows\\System32\\cmd.exe"
|
||||
"executable": "C:\\Windows\\System32\\cmd.exe",
|
||||
"pid": 2012
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"_index": "my-index-00001",
|
||||
"_index": "my-index-000001",
|
||||
"_type": "_doc",
|
||||
"_id": "yDwnGIJouOYGBzP0ZE9n",
|
||||
"_version": 1,
|
||||
|
|
@ -708,9 +685,6 @@ the events in ascending, lexicographic order.
|
|||
"_score": null,
|
||||
"_source": {
|
||||
"@timestamp": "2020-12-07T11:07:09.000Z",
|
||||
"agent": {
|
||||
"id": "8a4f500d"
|
||||
},
|
||||
"event": {
|
||||
"category": "process",
|
||||
"id": "aR3NWVOs",
|
||||
|
|
@ -718,7 +692,8 @@ the events in ascending, lexicographic order.
|
|||
},
|
||||
"process": {
|
||||
"name": "regsvr32.exe",
|
||||
"executable": "C:\\Windows\\System32\\regsvr32.exe"
|
||||
"executable": "C:\\Windows\\System32\\regsvr32.exe",
|
||||
"pid": 2012
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -0,0 +1,720 @@
|
|||
[role="xpack"]
|
||||
[testenv="basic"]
|
||||
[[eql]]
|
||||
= EQL search
|
||||
++++
|
||||
<titleabbrev>EQL</titleabbrev>
|
||||
++++
|
||||
|
||||
experimental::[]
|
||||
|
||||
{eql-ref}/index.html[Event Query Language (EQL)] is a query language used for
|
||||
event-based, time-series data, such as logs.
|
||||
|
||||
[discrete]
|
||||
[[eql-advantages]]
|
||||
== Advantages of EQL
|
||||
|
||||
* *EQL lets you express relationships between events.* +
|
||||
Many query languages allow you to match only single events. EQL lets you match a
|
||||
sequence of events across different event categories and time spans.
|
||||
|
||||
* *EQL has a low learning curve.* +
|
||||
EQL syntax looks like other query languages. It lets you write and read queries
|
||||
intuitively, which makes for quick, iterative searching.
|
||||
|
||||
* *We designed EQL for security use cases.* +
|
||||
While you can use EQL for any event-based data, we created EQL for threat
|
||||
hunting. EQL not only supports indicator of compromise (IOC) searching but
|
||||
makes it easy to describe activity that goes beyond IOCs.
|
||||
|
||||
[discrete]
|
||||
[[eql-required-fields]]
|
||||
== Required fields
|
||||
|
||||
EQL assumes each document in a data stream or index corresponds to an event. To
|
||||
search using EQL, each document in the searched data stream or index must
|
||||
include a _timestamp_ field and an _event category_ field.
|
||||
|
||||
{es} EQL uses the `@timestamp` and `event.category` fields from the
|
||||
{ecs-ref}[Elastic Common Schema (ECS)] as the default timestamp and event
|
||||
category fields. If your searched documents use a different timestamp or event
|
||||
category field, you must specify it in the search request. See
|
||||
<<specify-a-timestamp-or-event-category-field>>.
|
||||
|
||||
[discrete]
|
||||
[[run-an-eql-search]]
|
||||
== Run an EQL search
|
||||
|
||||
You can use the <<eql-search-api,EQL search API>> to run an EQL search.
|
||||
|
||||
The following request searches `my-index-000001` for events with an
|
||||
`event.category` of `process` and a `process.name` of `cmd.exe`. Each document
|
||||
in `my-index-000001` includes a `@timestamp` and `event.category` field.
|
||||
|
||||
[source,console]
|
||||
----
|
||||
GET /my-index-000001/_eql/search
|
||||
{
|
||||
"query": """
|
||||
process where process.name == "cmd.exe"
|
||||
"""
|
||||
}
|
||||
----
|
||||
// TEST[setup:sec_logs]
|
||||
// TEST[s/search/search\?filter_path\=\-\*\.events\.\*fields/]
|
||||
|
||||
The API returns the following response. Matching events are included in the
|
||||
`hits.events` property. These events are sorted by timestamp, converted to
|
||||
milliseconds since the https://en.wikipedia.org/wiki/Unix_time[Unix epoch], in
|
||||
ascending order.
|
||||
|
||||
[source,console-result]
|
||||
----
|
||||
{
|
||||
"is_partial": false,
|
||||
"is_running": false,
|
||||
"took": 60,
|
||||
"timed_out": false,
|
||||
"hits": {
|
||||
"total": {
|
||||
"value": 2,
|
||||
"relation": "eq"
|
||||
},
|
||||
"events": [
|
||||
{
|
||||
"_index": "my-index-000001",
|
||||
"_type": "_doc",
|
||||
"_id": "OQmfCaduce8zoHT93o4H",
|
||||
"_score": null,
|
||||
"_source": {
|
||||
"@timestamp": "2020-12-06T11:04:05.000Z",
|
||||
"event": {
|
||||
"category": "process",
|
||||
"id": "edwCRnyD",
|
||||
"sequence": 1
|
||||
},
|
||||
"process": {
|
||||
"name": "cmd.exe",
|
||||
"executable": "C:\\Windows\\System32\\cmd.exe",
|
||||
"pid": 2012
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"_index": "my-index-000001",
|
||||
"_type": "_doc",
|
||||
"_id": "xLkCaj4EujzdNSxfYLbO",
|
||||
"_score": null,
|
||||
"_source": {
|
||||
"@timestamp": "2020-12-07T11:06:07.000Z",
|
||||
"event": {
|
||||
"category": "process",
|
||||
"id": "cMyt5SZ2",
|
||||
"sequence": 3
|
||||
},
|
||||
"process": {
|
||||
"name": "cmd.exe",
|
||||
"executable": "C:\\Windows\\System32\\cmd.exe",
|
||||
"pid": 2012
|
||||
}
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
----
|
||||
// TESTRESPONSE[s/"took": 60/"took": $body.took/]
|
||||
// TESTRESPONSE[s/"_id": "OQmfCaduce8zoHT93o4H"/"_id": $body.hits.events.0._id/]
|
||||
// TESTRESPONSE[s/"_id": "xLkCaj4EujzdNSxfYLbO"/"_id": $body.hits.events.1._id/]
|
||||
|
||||
[discrete]
|
||||
[[eql-search-sequence]]
|
||||
=== Search for a sequence of events
|
||||
|
||||
You can use EQL's <<eql-sequences,sequence syntax>> to search for an ordered
|
||||
series of events.
|
||||
|
||||
The following EQL search request matches a sequence that:
|
||||
|
||||
. Starts with an event with:
|
||||
+
|
||||
--
|
||||
* An `event.category` of `file`
|
||||
* A `file.name` of `cmd.exe`
|
||||
--
|
||||
. Followed by an event with:
|
||||
+
|
||||
--
|
||||
* An `event.category` of `process`
|
||||
* A `process.name` that contains the substring `regsvr32`
|
||||
--
|
||||
|
||||
[source,console]
|
||||
----
|
||||
GET /my-index-000001/_eql/search
|
||||
{
|
||||
"query": """
|
||||
sequence
|
||||
[ file where file.name == "cmd.exe" ]
|
||||
[ process where stringContains(process.name, "regsvr32") ]
|
||||
"""
|
||||
}
|
||||
----
|
||||
// TEST[setup:sec_logs]
|
||||
|
||||
The API returns the following response. Matching sequences are included in the
|
||||
`hits.sequences` property.
|
||||
|
||||
[source,console-result]
|
||||
----
|
||||
{
|
||||
"is_partial": false,
|
||||
"is_running": false,
|
||||
"took": 60,
|
||||
"timed_out": false,
|
||||
"hits": {
|
||||
"total": {
|
||||
"value": 1,
|
||||
"relation": "eq"
|
||||
},
|
||||
"sequences": [
|
||||
{
|
||||
"events": [
|
||||
{
|
||||
"_index": "my-index-000001",
|
||||
"_type": "_doc",
|
||||
"_id": "AtOJ4UjUBAAx3XR5kcCM",
|
||||
"_version" : 1,
|
||||
"_seq_no" : 3,
|
||||
"_primary_term" : 1,
|
||||
"_score": null,
|
||||
"_source": {
|
||||
"@timestamp": "2020-12-07T11:07:08.000Z",
|
||||
"event": {
|
||||
"category": "file",
|
||||
"id": "bYA7gPay",
|
||||
"sequence": 4
|
||||
},
|
||||
"file": {
|
||||
"accessed": "2020-12-07T11:07:08.000Z",
|
||||
"name": "cmd.exe",
|
||||
"path": "C:\\Windows\\System32\\cmd.exe",
|
||||
"type": "file",
|
||||
"size": 16384
|
||||
},
|
||||
"process": {
|
||||
"name": "cmd.exe",
|
||||
"executable": "C:\\Windows\\System32\\cmd.exe",
|
||||
"pid": 2012
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"_index": "my-index-000001",
|
||||
"_type": "_doc",
|
||||
"_id": "yDwnGIJouOYGBzP0ZE9n",
|
||||
"_version" : 1,
|
||||
"_seq_no" : 4,
|
||||
"_primary_term" : 1,
|
||||
"_score": null,
|
||||
"_source": {
|
||||
"@timestamp": "2020-12-07T11:07:09.000Z",
|
||||
"event": {
|
||||
"category": "process",
|
||||
"id": "aR3NWVOs",
|
||||
"sequence": 5
|
||||
},
|
||||
"process": {
|
||||
"name": "regsvr32.exe",
|
||||
"executable": "C:\\Windows\\System32\\regsvr32.exe",
|
||||
"pid": 2012
|
||||
}
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
----
|
||||
// TESTRESPONSE[s/"took": 60/"took": $body.took/]
|
||||
// TESTRESPONSE[s/"_id": "AtOJ4UjUBAAx3XR5kcCM"/"_id": $body.hits.sequences.0.events.0._id/]
|
||||
// TESTRESPONSE[s/"_id": "yDwnGIJouOYGBzP0ZE9n"/"_id": $body.hits.sequences.0.events.1._id/]
|
||||
|
||||
You can use the <<eql-with-maxspan-keywords,`with maxspan` keywords>> to
|
||||
constrain a sequence to a specified timespan.
|
||||
|
||||
The following EQL search request adds `with maxspan=1h` to the previous query.
|
||||
This ensures all events in a matching sequence occur within `1h` (one hour) of
|
||||
the first event's timestamp.
|
||||
|
||||
[source,console]
|
||||
----
|
||||
GET /my-index-000001/_eql/search
|
||||
{
|
||||
"query": """
|
||||
sequence with maxspan=1h
|
||||
[ file where file.name == "cmd.exe" ]
|
||||
[ process where stringContains(process.name, "regsvr32") ]
|
||||
"""
|
||||
}
|
||||
----
|
||||
// TEST[setup:sec_logs]
|
||||
|
||||
You can further constrain matching event sequences using the
|
||||
<<eql-by-keyword,`by` keyword>>.
|
||||
|
||||
The following EQL search request adds `by process.pid` to each event item. This
|
||||
ensures events matching the sequence share the same `process.pid` field value.
|
||||
|
||||
[source,console]
|
||||
----
|
||||
GET /my-index-000001/_eql/search
|
||||
{
|
||||
"query": """
|
||||
sequence with maxspan=1h
|
||||
[ file where file.name == "cmd.exe" ] by process.pid
|
||||
[ process where stringContains(process.name, "regsvr32") ] by process.pid
|
||||
"""
|
||||
}
|
||||
----
|
||||
// TEST[setup:sec_logs]
|
||||
|
||||
Because the `process.pid` field is shared across all events in the sequence, it
|
||||
can be included using `sequence by`. The following query is equivalent to the
|
||||
previous one.
|
||||
|
||||
[source,console]
|
||||
----
|
||||
GET /my-index-000001/_eql/search
|
||||
{
|
||||
"query": """
|
||||
sequence by process.pid with maxspan=1h
|
||||
[ file where file.name == "cmd.exe" ]
|
||||
[ process where stringContains(process.name, "regsvr32") ]
|
||||
"""
|
||||
}
|
||||
----
|
||||
// TEST[setup:sec_logs]
|
||||
|
||||
The API returns the following response. The `hits.sequences.join_keys` property
|
||||
contains the shared `process.pid` value for each matching event.
|
||||
|
||||
[source,console-result]
|
||||
----
|
||||
{
|
||||
"is_partial": false,
|
||||
"is_running": false,
|
||||
"took": 60,
|
||||
"timed_out": false,
|
||||
"hits": {
|
||||
"total": {
|
||||
"value": 1,
|
||||
"relation": "eq"
|
||||
},
|
||||
"sequences": [
|
||||
{
|
||||
"join_keys": [
|
||||
"2012"
|
||||
],
|
||||
"events": [
|
||||
{
|
||||
"_index": "my-index-000001",
|
||||
"_type": "_doc",
|
||||
"_id": "AtOJ4UjUBAAx3XR5kcCM",
|
||||
"_version": 1,
|
||||
"_seq_no": 3,
|
||||
"_primary_term": 1,
|
||||
"_score": null,
|
||||
"_source": {
|
||||
"@timestamp": "2020-12-07T11:07:08.000Z",
|
||||
"event": {
|
||||
"category": "file",
|
||||
"id": "bYA7gPay",
|
||||
"sequence": 4
|
||||
},
|
||||
"file": {
|
||||
"accessed": "2020-12-07T11:07:08.000Z",
|
||||
"name": "cmd.exe",
|
||||
"path": "C:\\Windows\\System32\\cmd.exe",
|
||||
"type": "file",
|
||||
"size": 16384
|
||||
},
|
||||
"process": {
|
||||
"name": "cmd.exe",
|
||||
"executable": "C:\\Windows\\System32\\cmd.exe",
|
||||
"pid": 2012
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"_index": "my-index-000001",
|
||||
"_type": "_doc",
|
||||
"_id": "yDwnGIJouOYGBzP0ZE9n",
|
||||
"_version": 1,
|
||||
"_seq_no": 4,
|
||||
"_primary_term": 1,
|
||||
"_score": null,
|
||||
"_source": {
|
||||
"@timestamp": "2020-12-07T11:07:09.000Z",
|
||||
"event": {
|
||||
"category": "process",
|
||||
"id": "aR3NWVOs",
|
||||
"sequence": 5
|
||||
},
|
||||
"process": {
|
||||
"name": "regsvr32.exe",
|
||||
"executable": "C:\\Windows\\System32\\regsvr32.exe",
|
||||
"pid": 2012
|
||||
}
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
----
|
||||
// TESTRESPONSE[s/"took": 60/"took": $body.took/]
|
||||
// TESTRESPONSE[s/"_id": "AtOJ4UjUBAAx3XR5kcCM"/"_id": $body.hits.sequences.0.events.0._id/]
|
||||
// TESTRESPONSE[s/"_id": "yDwnGIJouOYGBzP0ZE9n"/"_id": $body.hits.sequences.0.events.1._id/]
|
||||
|
||||
You can use the <<eql-until-keyword,`until` keyword>> to specify an expiration
|
||||
event for sequences. Matching sequences must end before this event.
|
||||
|
||||
The following request adds `until [ process where event.type == "termination" ]`
|
||||
to the previous query. This ensures matching sequences end before a `process`
|
||||
event with an `event.type` of `termination`.
|
||||
|
||||
[source,console]
|
||||
----
|
||||
GET /my-index-000001/_eql/search
|
||||
{
|
||||
"query": """
|
||||
sequence by process.pid with maxspan=1h
|
||||
[ file where file.name == "cmd.exe" ]
|
||||
[ process where stringContains(process.name, "regsvr32") ]
|
||||
until [ process where event.type == "termination" ]
|
||||
"""
|
||||
}
|
||||
----
|
||||
// TEST[setup:sec_logs]
|
||||
|
||||
[discrete]
|
||||
[[specify-a-timestamp-or-event-category-field]]
|
||||
=== Specify a timestamp or event category field
|
||||
|
||||
By default, the EQL search API uses `@timestamp` and `event.category` as the
|
||||
required timestamp and event category fields. If your searched documents use
|
||||
a different timestamp or event category field, you must specify it in the search
|
||||
request using the `timestamp_field` or `event_category_field` parameters.
|
||||
|
||||
The event category field is typically mapped as a <<keyword,`keyword`>> or
|
||||
<<constant-keyword,constant keyword>> field. The timestamp field is typically
|
||||
mapped as a <<date,`date`>> or <<date_nanos,`date_nanos`>> field.
|
||||
|
||||
NOTE: You cannot use a <<nested,`nested`>> field or the sub-fields of a `nested`
|
||||
field as the timestamp or event category field. See <<eql-nested-fields>>.
|
||||
|
||||
The following request uses the `timestamp_field` parameter to specify
|
||||
`file.accessed` as the timestamp field. The request also uses the
|
||||
`event_category_field` parameter to specify `file.type` as the event category
|
||||
field.
|
||||
|
||||
[source,console]
|
||||
----
|
||||
GET /my-index-000001/_eql/search
|
||||
{
|
||||
"timestamp_field": "file.accessed",
|
||||
"event_category_field": "file.type",
|
||||
"query": """
|
||||
file where (file.size > 1 and file.type == "file")
|
||||
"""
|
||||
}
|
||||
----
|
||||
// TEST[setup:sec_logs]
|
||||
|
||||
[discrete]
|
||||
[[eql-search-filter-query-dsl]]
|
||||
=== Filter using query DSL
|
||||
|
||||
You can use the `filter` parameter to specify an additional query using
|
||||
<<query-dsl,query DSL>>. This query filters the documents on which the EQL query
|
||||
runs.
|
||||
|
||||
The following request uses a `range` query to filter `my-index-000001` to only
|
||||
documents with a `file.size` value greater than `1` but less than `1000000`
|
||||
bytes. The EQL query in `query` parameter then runs on these filtered documents.
|
||||
|
||||
[source,console]
|
||||
----
|
||||
GET /my-index-000001/_eql/search
|
||||
{
|
||||
"filter": {
|
||||
"range" : {
|
||||
"file.size" : {
|
||||
"gte" : 1,
|
||||
"lte" : 1000000
|
||||
}
|
||||
}
|
||||
},
|
||||
"query": """
|
||||
file where (file.type == "file" and file.name == "cmd.exe")
|
||||
"""
|
||||
}
|
||||
----
|
||||
// TEST[setup:sec_logs]
|
||||
|
||||
[discrete]
|
||||
[[eql-search-case-sensitive]]
|
||||
=== Run a case-sensitive EQL search
|
||||
|
||||
By default, matching for EQL queries is case-insensitive. You can use the
|
||||
`case_sensitive` parameter to toggle case sensitivity on or off.
|
||||
|
||||
The following search request contains a query that matches `process` events
|
||||
with a `process.executable` containing `System32`.
|
||||
|
||||
Because `case_sensitive` is `true`, this query only matches `process.executable`
|
||||
values containing `System32` with the exact same capitalization. A
|
||||
`process.executable` value containing `system32` or `SYSTEM32` would not match
|
||||
this query.
|
||||
|
||||
[source,console]
|
||||
----
|
||||
GET /my-index-000001/_eql/search
|
||||
{
|
||||
"keep_on_completion": true,
|
||||
"case_sensitive": true,
|
||||
"query": """
|
||||
process where stringContains(process.executable, "System32")
|
||||
"""
|
||||
}
|
||||
----
|
||||
// TEST[setup:sec_logs]
|
||||
|
||||
[discrete]
|
||||
[[eql-search-async]]
|
||||
=== Run an async EQL search
|
||||
|
||||
EQL searches are designed to run on large volumes of data quickly, often
|
||||
returning results in milliseconds. For this reason, EQL searches are
|
||||
_synchronous_ by default. The search request waits for complete results before
|
||||
returning a response.
|
||||
|
||||
However, complete results can take longer for searches across:
|
||||
|
||||
* <<frozen-indices,Frozen indices>>
|
||||
* <<modules-cross-cluster-search,Multiple clusters>>
|
||||
* Many shards
|
||||
|
||||
To avoid long waits, you can use the `wait_for_completion_timeout` parameter to
|
||||
run an _asynchronous_, or _async_, EQL search.
|
||||
|
||||
Set `wait_for_completion_timeout` to a duration you'd like to wait
|
||||
for complete search results. If the search request does not finish within this
|
||||
period, the search becomes async and returns a response that includes:
|
||||
|
||||
* A search ID, which can be used to monitor the progress of the async search.
|
||||
* An `is_partial` value of `true`, meaning the response does not contain
|
||||
complete search results.
|
||||
* An `is_running` value of `true`, meaning the search is async and ongoing.
|
||||
|
||||
The async search continues to run in the background without blocking
|
||||
other requests.
|
||||
|
||||
The following request searches the `frozen-my-index-000001` index, which has been
|
||||
<<frozen-indices,frozen>> for storage and is rarely searched.
|
||||
|
||||
Because searches on frozen indices are expected to take longer to complete, the
|
||||
request contains a `wait_for_completion_timeout` parameter value of `2s` (two
|
||||
seconds). If the request does not return complete results in two seconds, the
|
||||
search becomes async and returns a search ID.
|
||||
|
||||
[source,console]
|
||||
----
|
||||
GET /frozen-my-index-000001/_eql/search
|
||||
{
|
||||
"wait_for_completion_timeout": "2s",
|
||||
"query": """
|
||||
process where process.name == "cmd.exe"
|
||||
"""
|
||||
}
|
||||
----
|
||||
// TEST[setup:sec_logs]
|
||||
// TEST[s/frozen-my-index-000001/my-index-000001/]
|
||||
|
||||
After two seconds, the request returns the following response. Note `is_partial`
|
||||
and `is_running` properties are `true`, indicating an async search.
|
||||
|
||||
[source,console-result]
|
||||
----
|
||||
{
|
||||
"id": "FmNJRUZ1YWZCU3dHY1BIOUhaenVSRkEaaXFlZ3h4c1RTWFNocDdnY2FSaERnUTozNDE=",
|
||||
"is_partial": true,
|
||||
"is_running": true,
|
||||
"took": 2000,
|
||||
"timed_out": false,
|
||||
"hits": ...
|
||||
}
|
||||
----
|
||||
// TESTRESPONSE[s/FmNJRUZ1YWZCU3dHY1BIOUhaenVSRkEaaXFlZ3h4c1RTWFNocDdnY2FSaERnUTozNDE=/$body.id/]
|
||||
// TESTRESPONSE[s/"is_partial": true/"is_partial": $body.is_partial/]
|
||||
// TESTRESPONSE[s/"is_running": true/"is_running": $body.is_running/]
|
||||
// TESTRESPONSE[s/"took": 2000/"took": $body.took/]
|
||||
// TESTRESPONSE[s/"hits": \.\.\./"hits": $body.hits/]
|
||||
|
||||
You can use the the search ID and the <<get-async-eql-search-api,get async EQL
|
||||
search API>> to check the progress of an async search.
|
||||
|
||||
The get async EQL search API also accepts a `wait_for_completion_timeout`
|
||||
parameter. If ongoing search does not complete during this period, the response
|
||||
returns an `is_partial` value of `true` and no search results.
|
||||
|
||||
The following get async EQL search API request checks the progress of the
|
||||
previous async EQL search. The request specifies a `wait_for_completion_timeout`
|
||||
query parameter value of `2s` (two seconds).
|
||||
|
||||
[source,console]
|
||||
----
|
||||
GET /_eql/search/FmNJRUZ1YWZCU3dHY1BIOUhaenVSRkEaaXFlZ3h4c1RTWFNocDdnY2FSaERnUTozNDE=?wait_for_completion_timeout=2s
|
||||
----
|
||||
// TEST[skip: no access to search ID]
|
||||
|
||||
The request returns the following response. Note `is_partial` and `is_running`
|
||||
are `false`, indicating the async search has finished and the search results
|
||||
in the `hits` property are complete.
|
||||
|
||||
[source,console-result]
|
||||
----
|
||||
{
|
||||
"id": "FmNJRUZ1YWZCU3dHY1BIOUhaenVSRkEaaXFlZ3h4c1RTWFNocDdnY2FSaERnUTozNDE=",
|
||||
"is_partial": false,
|
||||
"is_running": false,
|
||||
"took": 2000,
|
||||
"timed_out": false,
|
||||
"hits": ...
|
||||
}
|
||||
----
|
||||
// TESTRESPONSE[s/FmNJRUZ1YWZCU3dHY1BIOUhaenVSRkEaaXFlZ3h4c1RTWFNocDdnY2FSaERnUTozNDE=/$body.id/]
|
||||
// TESTRESPONSE[s/"took": 2000/"took": $body.took/]
|
||||
// TESTRESPONSE[s/"hits": \.\.\./"hits": $body.hits/]
|
||||
|
||||
[discrete]
|
||||
[[eql-search-store-async-eql-search]]
|
||||
=== Change the search retention period
|
||||
|
||||
By default, the EQL search API stores async searches for five days. After this
|
||||
period, any searches and their results are deleted. You can use the `keep_alive`
|
||||
parameter to change this retention period.
|
||||
|
||||
In the following EQL search request, the `keep_alive` parameter is `2d` (two
|
||||
days). If the search becomes async, its results
|
||||
are stored on the cluster for two days. After two days, the async
|
||||
search and its results are deleted, even if it's still ongoing.
|
||||
|
||||
[source,console]
|
||||
----
|
||||
GET /my-index-000001/_eql/search
|
||||
{
|
||||
"keep_alive": "2d",
|
||||
"wait_for_completion_timeout": "2s",
|
||||
"query": """
|
||||
process where process.name == "cmd.exe"
|
||||
"""
|
||||
}
|
||||
----
|
||||
// TEST[setup:sec_logs]
|
||||
|
||||
You can use the <<get-async-eql-search-api,get async EQL search API>>'s
|
||||
`keep_alive`parameter to later change the retention period. The new
|
||||
retention period starts after the get request executes.
|
||||
|
||||
The following request sets the `keep_alive` query parameter to `5d` (five days).
|
||||
The async search and its results are deleted five days after the get request
|
||||
executes.
|
||||
|
||||
[source,console]
|
||||
----
|
||||
GET /_eql/search/FmNJRUZ1YWZCU3dHY1BIOUhaenVSRkEaaXFlZ3h4c1RTWFNocDdnY2FSaERnUTozNDE=?keep_alive=5d
|
||||
----
|
||||
// TEST[skip: no access to search ID]
|
||||
|
||||
You can use the <<delete-async-eql-search-api,delete async EQL search API>> to
|
||||
manually delete an async EQL search before the `keep_alive` period ends. If the
|
||||
search is still ongoing, this cancels the search request.
|
||||
|
||||
The following request deletes an async EQL search and its results.
|
||||
|
||||
[source,console]
|
||||
----
|
||||
DELETE /_eql/search/FmNJRUZ1YWZCU3dHY1BIOUhaenVSRkEaaXFlZ3h4c1RTWFNocDdnY2FSaERnUTozNDE=?keep_alive=5d
|
||||
----
|
||||
// TEST[skip: no access to search ID]
|
||||
|
||||
[discrete]
|
||||
[[eql-search-store-sync-eql-search]]
|
||||
=== Store synchronous EQL searches
|
||||
|
||||
By default, the EQL search API only stores async searches that cannot be
|
||||
completed within the period set by `wait_for_completion_timeout`.
|
||||
|
||||
To save the results of searches that complete during this period, set the
|
||||
`keep_on_completion` parameter to `true`.
|
||||
|
||||
In the following search request, `keep_on_completion` is `true`. This means the
|
||||
search results are stored on the cluster, even if the search completes within
|
||||
the `2s` (two-second) period set by the `wait_for_completion_timeout` parameter.
|
||||
|
||||
[source,console]
|
||||
----
|
||||
GET /my-index-000001/_eql/search
|
||||
{
|
||||
"keep_on_completion": true,
|
||||
"wait_for_completion_timeout": "2s",
|
||||
"query": """
|
||||
process where process.name == "cmd.exe"
|
||||
"""
|
||||
}
|
||||
----
|
||||
// TEST[setup:sec_logs]
|
||||
|
||||
The API returns the following response. A search ID is provided in the `id`
|
||||
property. `is_partial` and `is_running` are `false`, indicating the EQL search
|
||||
was synchronous and returned complete results in `hits`.
|
||||
|
||||
[source,console-result]
|
||||
----
|
||||
{
|
||||
"id": "FjlmbndxNmJjU0RPdExBTGg0elNOOEEaQk9xSjJBQzBRMldZa1VVQ2pPa01YUToxMDY=",
|
||||
"is_partial": false,
|
||||
"is_running": false,
|
||||
"took": 52,
|
||||
"timed_out": false,
|
||||
"hits": ...
|
||||
}
|
||||
----
|
||||
// TESTRESPONSE[s/FjlmbndxNmJjU0RPdExBTGg0elNOOEEaQk9xSjJBQzBRMldZa1VVQ2pPa01YUToxMDY=/$body.id/]
|
||||
// TESTRESPONSE[s/"took": 52/"took": $body.took/]
|
||||
// TESTRESPONSE[s/"hits": \.\.\./"hits": $body.hits/]
|
||||
|
||||
You can use the search ID and the <<get-async-eql-search-api,get async EQL
|
||||
search API>> to retrieve the same results later.
|
||||
|
||||
[source,console]
|
||||
----
|
||||
GET /_eql/search/FjlmbndxNmJjU0RPdExBTGg0elNOOEEaQk9xSjJBQzBRMldZa1VVQ2pPa01YUToxMDY=
|
||||
----
|
||||
// TEST[skip: no access to search ID]
|
||||
|
||||
Saved synchronous searches are still subject to the retention period set by the
|
||||
`keep_alive` parameter. After this period, the search and its results are
|
||||
deleted.
|
||||
|
||||
You can also manually delete saved synchronous searches using the
|
||||
<<delete-async-eql-search-api,delete async EQL search API>>.
|
||||
|
||||
include::syntax.asciidoc[]
|
||||
include::functions.asciidoc[]
|
||||
include::pipes.asciidoc[]
|
||||
|
|
@ -27,12 +27,12 @@ GET /_eql/search/FkpMRkJGS1gzVDRlM3g4ZzMyRGlLbkEaTXlJZHdNT09TU2VTZVBoNDM3cFZMUTo
|
|||
[[get-async-eql-search-api-prereqs]]
|
||||
==== {api-prereq-title}
|
||||
|
||||
See <<eql-requirements,EQL requirements>>.
|
||||
See <<eql-required-fields>>.
|
||||
|
||||
[[get-async-eql-search-api-limitations]]
|
||||
===== Limitations
|
||||
|
||||
See <<eql-limitations,EQL limitations>>.
|
||||
See <<eql-syntax-limitations,EQL limitations>>.
|
||||
|
||||
[[get-async-eql-search-api-path-params]]
|
||||
==== {api-path-parms-title}
|
||||
|
|
|
|||
|
|
@ -1,61 +0,0 @@
|
|||
[role="xpack"]
|
||||
[testenv="basic"]
|
||||
[[eql]]
|
||||
= EQL for event-based search
|
||||
++++
|
||||
<titleabbrev>EQL</titleabbrev>
|
||||
++++
|
||||
|
||||
experimental::[]
|
||||
|
||||
{eql-ref}/index.html[Event Query Language (EQL)] is a query language used for
|
||||
logs and other event-based data.
|
||||
|
||||
You can use EQL in {es} to easily express relationships between events and
|
||||
quickly match events with shared properties. You can use EQL and query
|
||||
DSL together to better filter your searches.
|
||||
|
||||
[discrete]
|
||||
[[eql-advantages]]
|
||||
=== Advantages of EQL
|
||||
|
||||
* *EQL lets you express relationships between events.* +
|
||||
Many query languages allow you to match only single events. EQL lets you match a
|
||||
sequence of events across different event categories and time spans.
|
||||
|
||||
* *EQL has a low learning curve.* +
|
||||
EQL syntax looks like other query languages. It lets you write and read queries
|
||||
intuitively, which makes for quick, iterative searching.
|
||||
|
||||
* *We designed EQL for security use cases.* +
|
||||
While you can use EQL for any event-based data, we created EQL for threat
|
||||
hunting. EQL not only supports indicator of compromise (IOC) searching but
|
||||
makes it easy to describe activity that goes beyond IOCs.
|
||||
|
||||
[discrete]
|
||||
[[when-to-use-eql]]
|
||||
=== When to use EQL
|
||||
|
||||
Consider using EQL if you:
|
||||
|
||||
* Use {es} for threat hunting or other security use cases
|
||||
* Search time-series data or logs, such as network or system logs
|
||||
* Want an easy way to explore relationships between events
|
||||
|
||||
[discrete]
|
||||
[[eql-toc]]
|
||||
=== In this section
|
||||
|
||||
* <<eql-requirements>>
|
||||
* <<eql-search>>
|
||||
* <<eql-syntax>>
|
||||
* <<eql-function-ref>>
|
||||
* <<eql-pipe-ref>>
|
||||
* <<eql-limitations>>
|
||||
|
||||
include::requirements.asciidoc[]
|
||||
include::search.asciidoc[]
|
||||
include::syntax.asciidoc[]
|
||||
include::functions.asciidoc[]
|
||||
include::pipes.asciidoc[]
|
||||
include::limitations.asciidoc[]
|
||||
|
|
@ -1,43 +0,0 @@
|
|||
[role="xpack"]
|
||||
[testenv="basic"]
|
||||
[[eql-limitations]]
|
||||
== EQL limitations
|
||||
++++
|
||||
<titleabbrev>Limitations</titleabbrev>
|
||||
++++
|
||||
|
||||
experimental::[]
|
||||
|
||||
[discrete]
|
||||
[[eql-nested-fields]]
|
||||
=== EQL search on nested fields is not supported
|
||||
|
||||
You cannot use EQL to search the values of a <<nested,`nested`>> field or the
|
||||
sub-fields of a `nested` field. However, data streams and indices containing
|
||||
`nested` field mappings are otherwise supported.
|
||||
|
||||
[discrete]
|
||||
[[eql-unsupported-syntax]]
|
||||
=== Unsupported syntax
|
||||
|
||||
{es} supports a subset of {eql-ref}/index.html[EQL syntax]. {es} cannot run EQL
|
||||
queries that contain:
|
||||
|
||||
* Array functions:
|
||||
** {eql-ref}/functions.html#arrayContains[`arrayContains`]
|
||||
** {eql-ref}/functions.html#arrayCount[`arrayCount`]
|
||||
** {eql-ref}/functions.html#arraySearch[`arraySearch`]
|
||||
|
||||
* {eql-ref}/joins.html[Joins]
|
||||
|
||||
* {eql-ref}/basic-syntax.html#event-relationships[Lineage-related keywords]:
|
||||
** `child of`
|
||||
** `descendant of`
|
||||
** `event of`
|
||||
|
||||
* The following {eql-ref}/pipes.html[pipes]:
|
||||
** {eql-ref}/pipes.html#count[`count`]
|
||||
** {eql-ref}/pipes.html#filter[`filter`]
|
||||
** {eql-ref}/pipes.html#sort[`sort`]
|
||||
** {eql-ref}/pipes.html#unique[`unique`]
|
||||
** {eql-ref}/pipes.html#unique-count[`unique_count`]
|
||||
|
|
@ -1,43 +0,0 @@
|
|||
[role="xpack"]
|
||||
[testenv="basic"]
|
||||
[[eql-requirements]]
|
||||
== EQL requirements
|
||||
++++
|
||||
<titleabbrev>Requirements</titleabbrev>
|
||||
++++
|
||||
|
||||
experimental::[]
|
||||
|
||||
EQL is schema-less and works well with most common log formats.
|
||||
|
||||
[TIP]
|
||||
====
|
||||
While no schema is required to use EQL in {es}, we recommend the
|
||||
{ecs-ref}[Elastic Common Schema (ECS)]. The <<eql-search-api,EQL search API>> is
|
||||
designed to work with core ECS fields by default.
|
||||
====
|
||||
|
||||
[discrete]
|
||||
[[eql-required-fields]]
|
||||
=== Required fields
|
||||
|
||||
In {es}, EQL assumes each document in a data stream or index corresponds to an
|
||||
event.
|
||||
|
||||
To search a data stream or index using EQL, each document in the data stream or
|
||||
index must contain the following field archetypes:
|
||||
|
||||
Event category::
|
||||
A field containing the event classification, such as `process`, `file`, or
|
||||
`network`. This is typically mapped as a <<keyword,`keyword`>> field.
|
||||
|
||||
Timestamp::
|
||||
A field containing the date and/or time the event occurred. This is typically
|
||||
mapped as a <<date,`date`>> or <<date_nanos,`date_nanos`>> field.
|
||||
|
||||
[NOTE]
|
||||
====
|
||||
You cannot use a <<nested,`nested`>> field data type or the sub-fields of a
|
||||
`nested` field as the timestamp or event category field. See
|
||||
<<eql-nested-fields>>.
|
||||
====
|
||||
|
|
@ -1,835 +0,0 @@
|
|||
[role="xpack"]
|
||||
[testenv="basic"]
|
||||
[[eql-search]]
|
||||
== Run an EQL search
|
||||
|
||||
experimental::[]
|
||||
|
||||
To start using EQL in {es}, first ensure your event data meets
|
||||
<<eql-requirements,EQL requirements>>. You can then use the <<eql-search-api,EQL
|
||||
search API>> to search event data stored in one or more {es} data streams or
|
||||
indices. The API requires a query written in {es}'s supported <<eql-syntax,EQL
|
||||
syntax>>.
|
||||
|
||||
To get started, ingest or add the data to an {es} data stream or index.
|
||||
|
||||
The following <<docs-bulk,bulk API>> request adds some example log data to the
|
||||
`sec_logs` index. This log data follows the {ecs-ref}[Elastic Common Schema
|
||||
(ECS)].
|
||||
|
||||
[source,console]
|
||||
----
|
||||
PUT /sec_logs/_bulk?refresh
|
||||
{"index":{ }}
|
||||
{ "@timestamp": "2020-12-06T11:04:05.000Z", "agent": { "id": "8a4f500d" }, "event": { "category": "process", "id": "edwCRnyD", "sequence": 1 }, "process": { "name": "cmd.exe", "executable": "C:\\Windows\\System32\\cmd.exe" } }
|
||||
{"index":{ }}
|
||||
{ "@timestamp": "2020-12-06T11:04:07.000Z", "agent": { "id": "8a4f500d" }, "event": { "category": "file", "id": "dGCHwoeS", "sequence": 2 }, "file": { "accessed": "2020-12-07T11:07:08.000Z", "name": "cmd.exe", "path": "C:\\Windows\\System32\\cmd.exe", "type": "file", "size": 16384 }, "process": { "name": "cmd.exe", "executable": "C:\\Windows\\System32\\cmd.exe" } }
|
||||
{"index":{ }}
|
||||
{ "@timestamp": "2020-12-07T11:06:07.000Z", "agent": { "id": "8a4f500d" }, "event": { "category": "process", "id": "cMyt5SZ2", "sequence": 3 }, "process": { "name": "cmd.exe", "executable": "C:\\Windows\\System32\\cmd.exe" } }
|
||||
{"index":{ }}
|
||||
{ "@timestamp": "2020-12-07T11:07:08.000Z", "agent": { "id": "8a4f500d" }, "event": { "category": "file", "id": "bYA7gPay", "sequence": 4 }, "file": { "accessed": "2020-12-07T11:07:08.000Z", "name": "cmd.exe", "path": "C:\\Windows\\System32\\cmd.exe", "type": "file", "size": 16384 }, "process": { "name": "cmd.exe", "executable": "C:\\Windows\\System32\\cmd.exe" } }
|
||||
{"index":{ }}
|
||||
{ "@timestamp": "2020-12-07T11:07:09.000Z", "agent": { "id": "8a4f500d" }, "event": { "category": "process", "id": "aR3NWVOs", "sequence": 5 }, "process": { "name": "regsvr32.exe", "executable": "C:\\Windows\\System32\\regsvr32.exe" } }
|
||||
{"index":{ }}
|
||||
{ "@timestamp": "2020-12-07T11:07:10.000Z", "agent": { "id": "8a4f500d" }, "event": { "category": "process", "id": "GTSmSqgz0U", "sequence": 6, "type": "termination" }, "process": { "name": "regsvr32.exe", "executable": "C:\\Windows\\System32\\regsvr32.exe" } }
|
||||
----
|
||||
// TESTSETUP
|
||||
|
||||
[TIP]
|
||||
=====
|
||||
You also can set up {beats-ref}/getting-started.html[{beats}], such as
|
||||
{auditbeat-ref}/auditbeat-installation-configuration.html[{auditbeat}] or
|
||||
{winlogbeat-ref}/winlogbeat-installation-configuration.html[{winlogbeat}], to automatically
|
||||
send and index your event data in {es}. See
|
||||
{beats-ref}/getting-started.html[Getting started with {beats}].
|
||||
=====
|
||||
|
||||
You can now use the EQL search API to search this index using an EQL query.
|
||||
|
||||
The following request searches the `sec_logs` index using the EQL query
|
||||
specified in the `query` parameter. The EQL query matches events with an
|
||||
`event.category` of `process` that have a `process.name` of `cmd.exe`.
|
||||
|
||||
[source,console]
|
||||
----
|
||||
GET /sec_logs/_eql/search
|
||||
{
|
||||
"query": """
|
||||
process where process.name == "cmd.exe"
|
||||
"""
|
||||
}
|
||||
----
|
||||
// TEST[s/search/search\?filter_path\=\-\*\.events\.\*fields/]
|
||||
|
||||
Because the `sec_log` index follows the ECS, you don't need to specify the
|
||||
required <<eql-required-fields,event category or timestamp>> fields. The request
|
||||
uses the `event.category` and `@timestamp` fields by default.
|
||||
|
||||
The API returns the following response containing the matching events. Events
|
||||
in the response are sorted by timestamp, converted to milliseconds since the
|
||||
https://en.wikipedia.org/wiki/Unix_time[Unix epoch], in ascending order.
|
||||
|
||||
[source,console-result]
|
||||
----
|
||||
{
|
||||
"is_partial": false,
|
||||
"is_running": false,
|
||||
"took": 60,
|
||||
"timed_out": false,
|
||||
"hits": {
|
||||
"total": {
|
||||
"value": 2,
|
||||
"relation": "eq"
|
||||
},
|
||||
"events": [
|
||||
{
|
||||
"_index": "sec_logs",
|
||||
"_type": "_doc",
|
||||
"_id": "OQmfCaduce8zoHT93o4H",
|
||||
"_score": null,
|
||||
"_source": {
|
||||
"@timestamp": "2020-12-06T11:04:05.000Z",
|
||||
"agent": {
|
||||
"id": "8a4f500d"
|
||||
},
|
||||
"event": {
|
||||
"category": "process",
|
||||
"id": "edwCRnyD",
|
||||
"sequence": 1
|
||||
},
|
||||
"process": {
|
||||
"name": "cmd.exe",
|
||||
"executable": "C:\\Windows\\System32\\cmd.exe"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"_index": "sec_logs",
|
||||
"_type": "_doc",
|
||||
"_id": "xLkCaj4EujzdNSxfYLbO",
|
||||
"_score": null,
|
||||
"_source": {
|
||||
"@timestamp": "2020-12-07T11:06:07.000Z",
|
||||
"agent": {
|
||||
"id": "8a4f500d"
|
||||
},
|
||||
"event": {
|
||||
"category": "process",
|
||||
"id": "cMyt5SZ2",
|
||||
"sequence": 3
|
||||
},
|
||||
"process": {
|
||||
"name": "cmd.exe",
|
||||
"executable": "C:\\Windows\\System32\\cmd.exe"
|
||||
}
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
----
|
||||
// TESTRESPONSE[s/"took": 60/"took": $body.took/]
|
||||
// TESTRESPONSE[s/"_id": "OQmfCaduce8zoHT93o4H"/"_id": $body.hits.events.0._id/]
|
||||
// TESTRESPONSE[s/"_id": "xLkCaj4EujzdNSxfYLbO"/"_id": $body.hits.events.1._id/]
|
||||
|
||||
[discrete]
|
||||
[[eql-search-sequence]]
|
||||
=== Search for a sequence of events
|
||||
|
||||
Many query languages allow you to match single events. However, EQL's
|
||||
<<eql-sequences,sequence syntax>> lets you match an ordered series of events.
|
||||
|
||||
The following EQL search request matches a sequence that:
|
||||
|
||||
. Starts with an event with:
|
||||
+
|
||||
--
|
||||
* An `event.category` of `file`
|
||||
* A `file.name` of `cmd.exe`
|
||||
--
|
||||
. Followed by an event with:
|
||||
+
|
||||
--
|
||||
* An `event.category` of `process`
|
||||
* A `process.name` that contains the substring `regsvr32`
|
||||
--
|
||||
|
||||
[source,console]
|
||||
----
|
||||
GET /sec_logs/_eql/search
|
||||
{
|
||||
"query": """
|
||||
sequence
|
||||
[ file where file.name == "cmd.exe" ]
|
||||
[ process where stringContains(process.name, "regsvr32") ]
|
||||
"""
|
||||
}
|
||||
----
|
||||
|
||||
The API returns the following response. Matching events in
|
||||
the `hits.sequences.events` property are sorted by
|
||||
<<eql-search-api-timestamp-field,timestamp>>, converted to milliseconds since
|
||||
the https://en.wikipedia.org/wiki/Unix_time[Unix epoch], in ascending order.
|
||||
|
||||
[source,console-result]
|
||||
----
|
||||
{
|
||||
"is_partial": false,
|
||||
"is_running": false,
|
||||
"took": 60,
|
||||
"timed_out": false,
|
||||
"hits": {
|
||||
"total": {
|
||||
"value": 1,
|
||||
"relation": "eq"
|
||||
},
|
||||
"sequences": [
|
||||
{
|
||||
"events": [
|
||||
{
|
||||
"_index": "sec_logs",
|
||||
"_type": "_doc",
|
||||
"_id": "AtOJ4UjUBAAx3XR5kcCM",
|
||||
"_version" : 1,
|
||||
"_seq_no" : 3,
|
||||
"_primary_term" : 1,
|
||||
"_score": null,
|
||||
"_source": {
|
||||
"@timestamp": "2020-12-07T11:07:08.000Z",
|
||||
"agent": {
|
||||
"id": "8a4f500d"
|
||||
},
|
||||
"event": {
|
||||
"category": "file",
|
||||
"id": "bYA7gPay",
|
||||
"sequence": 4
|
||||
},
|
||||
"file": {
|
||||
"accessed": "2020-12-07T11:07:08.000Z",
|
||||
"name": "cmd.exe",
|
||||
"path": "C:\\Windows\\System32\\cmd.exe",
|
||||
"type": "file",
|
||||
"size": 16384
|
||||
},
|
||||
"process": {
|
||||
"name": "cmd.exe",
|
||||
"executable": "C:\\Windows\\System32\\cmd.exe"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"_index": "sec_logs",
|
||||
"_type": "_doc",
|
||||
"_id": "yDwnGIJouOYGBzP0ZE9n",
|
||||
"_version" : 1,
|
||||
"_seq_no" : 4,
|
||||
"_primary_term" : 1,
|
||||
"_score": null,
|
||||
"_source": {
|
||||
"@timestamp": "2020-12-07T11:07:09.000Z",
|
||||
"agent": {
|
||||
"id": "8a4f500d"
|
||||
},
|
||||
"event": {
|
||||
"category": "process",
|
||||
"id": "aR3NWVOs",
|
||||
"sequence": 5
|
||||
},
|
||||
"process": {
|
||||
"name": "regsvr32.exe",
|
||||
"executable": "C:\\Windows\\System32\\regsvr32.exe"
|
||||
}
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
----
|
||||
// TESTRESPONSE[s/"took": 60/"took": $body.took/]
|
||||
// TESTRESPONSE[s/"_id": "AtOJ4UjUBAAx3XR5kcCM"/"_id": $body.hits.sequences.0.events.0._id/]
|
||||
// TESTRESPONSE[s/"_id": "yDwnGIJouOYGBzP0ZE9n"/"_id": $body.hits.sequences.0.events.1._id/]
|
||||
|
||||
You can use the <<eql-with-maxspan-keywords,`with maxspan` keywords>> to
|
||||
constrain a sequence to a specified timespan.
|
||||
|
||||
The following EQL search request adds `with maxspan=1h` to the previous query.
|
||||
This ensures all events in a matching sequence occur within one hour (`1h`) of
|
||||
the first event's timestamp.
|
||||
|
||||
[source,console]
|
||||
----
|
||||
GET /sec_logs/_eql/search
|
||||
{
|
||||
"query": """
|
||||
sequence with maxspan=1h
|
||||
[ file where file.name == "cmd.exe" ]
|
||||
[ process where stringContains(process.name, "regsvr32") ]
|
||||
"""
|
||||
}
|
||||
----
|
||||
|
||||
You can further constrain matching event sequences using the
|
||||
<<eql-by-keyword,`by` keyword>>.
|
||||
|
||||
The following EQL search request adds `by agent.id` to each event item. This
|
||||
ensures events matching the sequence share the same `agent.id` field value.
|
||||
|
||||
[source,console]
|
||||
----
|
||||
GET /sec_logs/_eql/search
|
||||
{
|
||||
"query": """
|
||||
sequence with maxspan=1h
|
||||
[ file where file.name == "cmd.exe" ] by agent.id
|
||||
[ process where stringContains(process.name, "regsvr32") ] by agent.id
|
||||
"""
|
||||
}
|
||||
----
|
||||
|
||||
Because the `agent.id` field is shared across all events in the sequence, it
|
||||
can be included using `sequence by`. The following query is equivalent to the
|
||||
prior one.
|
||||
|
||||
[source,console]
|
||||
----
|
||||
GET /sec_logs/_eql/search
|
||||
{
|
||||
"query": """
|
||||
sequence by agent.id with maxspan=1h
|
||||
[ file where file.name == "cmd.exe" ]
|
||||
[ process where stringContains(process.name, "regsvr32") ]
|
||||
"""
|
||||
}
|
||||
----
|
||||
|
||||
The API returns the following response. The `hits.sequences.join_keys` property
|
||||
contains the shared `agent.id` value for each matching event.
|
||||
|
||||
[source,console-result]
|
||||
----
|
||||
{
|
||||
"is_partial": false,
|
||||
"is_running": false,
|
||||
"took": 60,
|
||||
"timed_out": false,
|
||||
"hits": {
|
||||
"total": {
|
||||
"value": 1,
|
||||
"relation": "eq"
|
||||
},
|
||||
"sequences": [
|
||||
{
|
||||
"join_keys": [
|
||||
"8a4f500d"
|
||||
],
|
||||
"events": [
|
||||
{
|
||||
"_index": "sec_logs",
|
||||
"_type": "_doc",
|
||||
"_id": "AtOJ4UjUBAAx3XR5kcCM",
|
||||
"_version": 1,
|
||||
"_seq_no": 3,
|
||||
"_primary_term": 1,
|
||||
"_score": null,
|
||||
"_source": {
|
||||
"@timestamp": "2020-12-07T11:07:08.000Z",
|
||||
"agent": {
|
||||
"id": "8a4f500d"
|
||||
},
|
||||
"event": {
|
||||
"category": "file",
|
||||
"id": "bYA7gPay",
|
||||
"sequence": 4
|
||||
},
|
||||
"file": {
|
||||
"accessed": "2020-12-07T11:07:08.000Z",
|
||||
"name": "cmd.exe",
|
||||
"path": "C:\\Windows\\System32\\cmd.exe",
|
||||
"type": "file",
|
||||
"size": 16384
|
||||
},
|
||||
"process": {
|
||||
"name": "cmd.exe",
|
||||
"executable": "C:\\Windows\\System32\\cmd.exe"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"_index": "sec_logs",
|
||||
"_type": "_doc",
|
||||
"_id": "yDwnGIJouOYGBzP0ZE9n",
|
||||
"_version": 1,
|
||||
"_seq_no": 4,
|
||||
"_primary_term": 1,
|
||||
"_score": null,
|
||||
"_source": {
|
||||
"@timestamp": "2020-12-07T11:07:09.000Z",
|
||||
"agent": {
|
||||
"id": "8a4f500d"
|
||||
},
|
||||
"event": {
|
||||
"category": "process",
|
||||
"id": "aR3NWVOs",
|
||||
"sequence": 5
|
||||
},
|
||||
"process": {
|
||||
"name": "regsvr32.exe",
|
||||
"executable": "C:\\Windows\\System32\\regsvr32.exe"
|
||||
}
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
----
|
||||
// TESTRESPONSE[s/"took": 60/"took": $body.took/]
|
||||
// TESTRESPONSE[s/"_id": "AtOJ4UjUBAAx3XR5kcCM"/"_id": $body.hits.sequences.0.events.0._id/]
|
||||
// TESTRESPONSE[s/"_id": "yDwnGIJouOYGBzP0ZE9n"/"_id": $body.hits.sequences.0.events.1._id/]
|
||||
|
||||
You can use the <<eql-until-keyword,`until` keyword>> to specify an expiration
|
||||
event for sequences. Matching sequences must end before this event.
|
||||
|
||||
The following request adds
|
||||
`until [ process where event.type == "termination" ]` to the previous EQL query.
|
||||
This ensures matching sequences end before a process termination event.
|
||||
|
||||
[source,console]
|
||||
----
|
||||
GET /sec_logs/_eql/search
|
||||
{
|
||||
"query": """
|
||||
sequence by agent.id with maxspan=1h
|
||||
[ file where file.name == "cmd.exe" ]
|
||||
[ process where stringContains(process.name, "regsvr32") ]
|
||||
until [ process where event.type == "termination" ]
|
||||
"""
|
||||
}
|
||||
----
|
||||
|
||||
[discrete]
|
||||
[[eql-search-specify-event-category-field]]
|
||||
=== Specify an event category field
|
||||
|
||||
By default, the EQL search API uses `event.category` as the
|
||||
<<eql-required-fields,event category field>>. You can use the
|
||||
`event_category_field` parameter to specify another event category field.
|
||||
|
||||
The following request specifies `file.type` as the event category
|
||||
field.
|
||||
|
||||
[source,console]
|
||||
----
|
||||
GET /sec_logs/_eql/search
|
||||
{
|
||||
"event_category_field": "file.type",
|
||||
"query": """
|
||||
file where agent.id == "8a4f500d"
|
||||
"""
|
||||
}
|
||||
----
|
||||
|
||||
[discrete]
|
||||
[[eql-search-specify-timestamp-field]]
|
||||
=== Specify a timestamp field
|
||||
|
||||
By default, EQL searches use `@timestamp` as the <<eql-required-fields,event
|
||||
timestamp field>>. You can use the EQL search API's `timestamp_field` parameter
|
||||
to specify another timestamp field.
|
||||
|
||||
The following request specifies `file.accessed` as the event
|
||||
timestamp field.
|
||||
|
||||
[source,console]
|
||||
----
|
||||
GET /sec_logs/_eql/search
|
||||
{
|
||||
"timestamp_field": "file.accessed",
|
||||
"query": """
|
||||
file where (file.size > 1 and file.type == "file")
|
||||
"""
|
||||
}
|
||||
----
|
||||
|
||||
[discrete]
|
||||
[[eql-search-specify-a-sort-tiebreaker]]
|
||||
=== Specify a sort tiebreaker
|
||||
|
||||
By default, the EQL search API sorts matching events in the search response by
|
||||
timestamp. However, if two or more events share the same timestamp, a tiebreaker
|
||||
field is used to sort the events in ascending, lexicographic order.
|
||||
|
||||
The EQL search API uses `event.sequence` as the default tiebreaker field. You
|
||||
can use the `tiebreaker_field` parameter to specify another field.
|
||||
|
||||
The following request specifies `event.start` as the tiebreaker field.
|
||||
|
||||
[source,console]
|
||||
----
|
||||
GET /sec_logs/_eql/search
|
||||
{
|
||||
"tiebreaker_field": "event.id",
|
||||
"query": """
|
||||
process where process.name == "cmd.exe" and stringContains(process.executable, "System32")
|
||||
"""
|
||||
}
|
||||
----
|
||||
// TEST[s/search/search\?filter_path\=\-\*\.events\.\*fields/]
|
||||
|
||||
The API returns the following response.
|
||||
|
||||
[source,console-result]
|
||||
----
|
||||
{
|
||||
"is_partial": false,
|
||||
"is_running": false,
|
||||
"took": 34,
|
||||
"timed_out": false,
|
||||
"hits": {
|
||||
"total": {
|
||||
"value": 2,
|
||||
"relation": "eq"
|
||||
},
|
||||
"events": [
|
||||
{
|
||||
"_index": "sec_logs",
|
||||
"_type": "_doc",
|
||||
"_id": "OQmfCaduce8zoHT93o4H",
|
||||
"_score": null,
|
||||
"_source": {
|
||||
"@timestamp": "2020-12-06T11:04:05.000Z",
|
||||
"agent": {
|
||||
"id": "8a4f500d"
|
||||
},
|
||||
"event": {
|
||||
"category": "process",
|
||||
"id": "edwCRnyD",
|
||||
"sequence": 1
|
||||
},
|
||||
"process": {
|
||||
"name": "cmd.exe",
|
||||
"executable": "C:\\Windows\\System32\\cmd.exe"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"_index": "sec_logs",
|
||||
"_type": "_doc",
|
||||
"_id": "xLkCaj4EujzdNSxfYLbO",
|
||||
"_score": null,
|
||||
"_source": {
|
||||
"@timestamp": "2020-12-07T11:06:07.000Z",
|
||||
"agent": {
|
||||
"id": "8a4f500d"
|
||||
},
|
||||
"event": {
|
||||
"category": "process",
|
||||
"id": "cMyt5SZ2",
|
||||
"sequence": 3
|
||||
},
|
||||
"process": {
|
||||
"name": "cmd.exe",
|
||||
"executable": "C:\\Windows\\System32\\cmd.exe"
|
||||
}
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
----
|
||||
// TESTRESPONSE[s/"took": 34/"took": $body.took/]
|
||||
// TESTRESPONSE[s/"_id": "OQmfCaduce8zoHT93o4H"/"_id": $body.hits.events.0._id/]
|
||||
// TESTRESPONSE[s/"_id": "xLkCaj4EujzdNSxfYLbO"/"_id": $body.hits.events.1._id/]
|
||||
|
||||
|
||||
[discrete]
|
||||
[[eql-search-filter-query-dsl]]
|
||||
=== Filter using query DSL
|
||||
|
||||
You can use the EQL search API's `filter` parameter to specify an additional
|
||||
query using <<query-dsl,query DSL>>. This query filters the documents on which
|
||||
the EQL query runs.
|
||||
|
||||
The following request uses a `range` query to filter the `sec_logs`
|
||||
index down to only documents with a `file.size` value greater than `1` but less
|
||||
than `1000000` bytes. The EQL query in `query` parameter then runs on these
|
||||
filtered documents.
|
||||
|
||||
[source,console]
|
||||
----
|
||||
GET /sec_logs/_eql/search
|
||||
{
|
||||
"filter": {
|
||||
"range" : {
|
||||
"file.size" : {
|
||||
"gte" : 1,
|
||||
"lte" : 1000000
|
||||
}
|
||||
}
|
||||
},
|
||||
"query": """
|
||||
file where (file.type == "file" and file.name == "cmd.exe")
|
||||
"""
|
||||
}
|
||||
----
|
||||
|
||||
[discrete]
|
||||
[[eql-search-async]]
|
||||
=== Run an async EQL search
|
||||
|
||||
EQL searches in {es} are designed to run on large volumes of data quickly,
|
||||
often returning results in milliseconds. Because of this, the EQL search API
|
||||
runs _synchronous_ searches by default. This means the search request waits for
|
||||
complete results before returning a response.
|
||||
|
||||
However, complete results can take longer for searches across:
|
||||
|
||||
* <<frozen-indices,Frozen indices>>
|
||||
* <<modules-cross-cluster-search,Multiple clusters>>
|
||||
* Many shards
|
||||
|
||||
To avoid long waits, you can use the EQL search API's
|
||||
`wait_for_completion_timeout` parameter to run an _asynchronous_, or _async_,
|
||||
search.
|
||||
|
||||
Set the `wait_for_completion_timeout` parameter to a duration you'd like to wait
|
||||
for complete search results. If the search request does not finish within this
|
||||
period, the search becomes an async search. The EQL search
|
||||
API returns a response that includes:
|
||||
|
||||
* A search ID, which can be used to monitor the progress of the async search and
|
||||
retrieve complete results when it finishes.
|
||||
* An `is_partial` value of `true`, indicating the response does not contain
|
||||
complete search results.
|
||||
* An `is_running` value of `true`, indicating the search is async and ongoing.
|
||||
|
||||
The async search continues to run in the background without blocking
|
||||
other requests.
|
||||
|
||||
The following request searches the `frozen_sec_logs` index, which has been
|
||||
<<frozen-indices,frozen>> for storage and is rarely searched.
|
||||
|
||||
Because searches on frozen indices are expected to take longer to complete, the
|
||||
request contains a `wait_for_completion_timeout` parameter value of `2s`
|
||||
(two seconds).
|
||||
|
||||
If the request does not return complete results in two seconds, the search
|
||||
becomes an async search and a search ID is returned.
|
||||
|
||||
[source,console]
|
||||
----
|
||||
GET /frozen_sec_logs/_eql/search
|
||||
{
|
||||
"wait_for_completion_timeout": "2s",
|
||||
"query": """
|
||||
process where process.name == "cmd.exe"
|
||||
"""
|
||||
}
|
||||
----
|
||||
// TEST[s/frozen_sec_logs/sec_logs/]
|
||||
|
||||
After two seconds, the request returns the following response. Note the
|
||||
`is_partial` and `is_running` properties are `true`, indicating an ongoing async
|
||||
search.
|
||||
|
||||
[source,console-result]
|
||||
----
|
||||
{
|
||||
"id": "FmNJRUZ1YWZCU3dHY1BIOUhaenVSRkEaaXFlZ3h4c1RTWFNocDdnY2FSaERnUTozNDE=",
|
||||
"is_partial": true,
|
||||
"is_running": true,
|
||||
"took": 2000,
|
||||
"timed_out": false,
|
||||
"hits": ...
|
||||
}
|
||||
----
|
||||
// TESTRESPONSE[s/FmNJRUZ1YWZCU3dHY1BIOUhaenVSRkEaaXFlZ3h4c1RTWFNocDdnY2FSaERnUTozNDE=/$body.id/]
|
||||
// TESTRESPONSE[s/"is_partial": true/"is_partial": $body.is_partial/]
|
||||
// TESTRESPONSE[s/"is_running": true/"is_running": $body.is_running/]
|
||||
// TESTRESPONSE[s/"took": 2000/"took": $body.took/]
|
||||
// TESTRESPONSE[s/"hits": \.\.\./"hits": $body.hits/]
|
||||
|
||||
You can use the the returned search ID and the <<get-async-eql-search-api,get
|
||||
async EQL search API>> to check the progress of an ongoing async search.
|
||||
|
||||
The get async EQL search API also accepts a `wait_for_completion_timeout` query
|
||||
parameter. Set the `wait_for_completion_timeout` parameter to a duration you'd
|
||||
like to wait for complete search results. If the request does not complete
|
||||
during this period, the response returns an `is_partial` value of `true` and no
|
||||
search results.
|
||||
|
||||
The following get async EQL search API request checks the progress of the
|
||||
previous async EQL search. The request specifies a `wait_for_completion_timeout`
|
||||
query parameter value of `2s` (two seconds).
|
||||
|
||||
[source,console]
|
||||
----
|
||||
GET /_eql/search/FmNJRUZ1YWZCU3dHY1BIOUhaenVSRkEaaXFlZ3h4c1RTWFNocDdnY2FSaERnUTozNDE=?wait_for_completion_timeout=2s
|
||||
----
|
||||
// TEST[skip: no access to search ID]
|
||||
|
||||
The request returns the following response. Note the `is_partial` and
|
||||
`is_running` properties are `false`, indicating the async EQL search has
|
||||
finished and the search results in the `hits` property are complete.
|
||||
|
||||
[source,console-result]
|
||||
----
|
||||
{
|
||||
"id": "FmNJRUZ1YWZCU3dHY1BIOUhaenVSRkEaaXFlZ3h4c1RTWFNocDdnY2FSaERnUTozNDE=",
|
||||
"is_partial": false,
|
||||
"is_running": false,
|
||||
"took": 2000,
|
||||
"timed_out": false,
|
||||
"hits": ...
|
||||
}
|
||||
----
|
||||
// TESTRESPONSE[s/FmNJRUZ1YWZCU3dHY1BIOUhaenVSRkEaaXFlZ3h4c1RTWFNocDdnY2FSaERnUTozNDE=/$body.id/]
|
||||
// TESTRESPONSE[s/"took": 2000/"took": $body.took/]
|
||||
// TESTRESPONSE[s/"_index": "frozen_sec_logs"/"_index": "sec_logs"/]
|
||||
// TESTRESPONSE[s/"hits": \.\.\./"hits": $body.hits/]
|
||||
|
||||
[discrete]
|
||||
[[eql-search-store-async-eql-search]]
|
||||
=== Change the search retention period
|
||||
|
||||
By default, the EQL search API only stores async searches and their results for
|
||||
five days. After this period, any ongoing searches or saved results are deleted.
|
||||
|
||||
You can use the EQL search API's `keep_alive` parameter to change the duration
|
||||
of this period.
|
||||
|
||||
In the following EQL search API request, the `keep_alive` parameter is `2d` (two
|
||||
days). This means that if the search becomes async, its results
|
||||
are stored on the cluster for two days. After two days, the async
|
||||
search and its results are deleted, even if it's still ongoing.
|
||||
|
||||
[source,console]
|
||||
----
|
||||
GET /sec_logs/_eql/search
|
||||
{
|
||||
"keep_alive": "2d",
|
||||
"wait_for_completion_timeout": "2s",
|
||||
"query": """
|
||||
process where process.name == "cmd.exe"
|
||||
"""
|
||||
}
|
||||
----
|
||||
|
||||
You can use the <<get-async-eql-search-api,get async EQL search API>>'s
|
||||
`keep_alive` query parameter to later change the retention period. The new
|
||||
retention period starts after the get async EQL search API request executes.
|
||||
|
||||
The following get async EQL search API request sets the `keep_alive` query
|
||||
parameter to `5d` (five days). The async search and its results are deleted five
|
||||
days after the get async EQL search API request executes.
|
||||
|
||||
[source,console]
|
||||
----
|
||||
GET /_eql/search/FmNJRUZ1YWZCU3dHY1BIOUhaenVSRkEaaXFlZ3h4c1RTWFNocDdnY2FSaERnUTozNDE=?keep_alive=5d
|
||||
----
|
||||
// TEST[skip: no access to search ID]
|
||||
|
||||
You can use the <<delete-async-eql-search-api,delete async EQL search API>> to
|
||||
manually delete an async EQL search before the `keep_alive` period ends. If the
|
||||
search is still ongoing, this cancels the search request.
|
||||
|
||||
The following delete async EQL search API request deletes an async EQL search
|
||||
and its results.
|
||||
|
||||
[source,console]
|
||||
----
|
||||
DELETE /_eql/search/FmNJRUZ1YWZCU3dHY1BIOUhaenVSRkEaaXFlZ3h4c1RTWFNocDdnY2FSaERnUTozNDE=?keep_alive=5d
|
||||
----
|
||||
// TEST[skip: no access to search ID]
|
||||
|
||||
[discrete]
|
||||
[[eql-search-store-sync-eql-search]]
|
||||
=== Store synchronous EQL searches
|
||||
|
||||
By default, the EQL search API only stores async searches that cannot be
|
||||
completed within the period set by the `wait_for_completion_timeout` parameter.
|
||||
|
||||
To save the results of searches that complete during this period, set the
|
||||
`keep_on_completion` parameter to `true`.
|
||||
|
||||
In the following EQL search API request, the `keep_on_completion` parameter is
|
||||
`true`. This means the search results are stored on the cluster, even if
|
||||
the search completes within the `2s` (two-second) period set by the
|
||||
`wait_for_completion_timeout` parameter.
|
||||
|
||||
[source,console]
|
||||
----
|
||||
GET /sec_logs/_eql/search
|
||||
{
|
||||
"keep_on_completion": true,
|
||||
"wait_for_completion_timeout": "2s",
|
||||
"query": """
|
||||
process where process.name == "cmd.exe"
|
||||
"""
|
||||
}
|
||||
----
|
||||
|
||||
The API returns the following response. Note that a search ID is provided in the
|
||||
`id` property. The `is_partial` and `is_running` properties are `false`,
|
||||
indicating the EQL search was synchronous and returned complete search results.
|
||||
|
||||
[source,console-result]
|
||||
----
|
||||
{
|
||||
"id": "FjlmbndxNmJjU0RPdExBTGg0elNOOEEaQk9xSjJBQzBRMldZa1VVQ2pPa01YUToxMDY=",
|
||||
"is_partial": false,
|
||||
"is_running": false,
|
||||
"took": 52,
|
||||
"timed_out": false,
|
||||
"hits": ...
|
||||
}
|
||||
----
|
||||
// TESTRESPONSE[s/FjlmbndxNmJjU0RPdExBTGg0elNOOEEaQk9xSjJBQzBRMldZa1VVQ2pPa01YUToxMDY=/$body.id/]
|
||||
// TESTRESPONSE[s/"took": 52/"took": $body.took/]
|
||||
// TESTRESPONSE[s/"hits": \.\.\./"hits": $body.hits/]
|
||||
|
||||
You can use the search ID and the <<get-async-eql-search-api,get async EQL
|
||||
search API>> to retrieve the same results later.
|
||||
|
||||
[source,console]
|
||||
----
|
||||
GET /_eql/search/FjlmbndxNmJjU0RPdExBTGg0elNOOEEaQk9xSjJBQzBRMldZa1VVQ2pPa01YUToxMDY=
|
||||
----
|
||||
// TEST[skip: no access to search ID]
|
||||
|
||||
Saved synchronous searches are still subject to the storage retention period set
|
||||
by the `keep_alive` parameter. After this period, the search and its saved
|
||||
results are deleted.
|
||||
|
||||
You can also manually delete saved synchronous searches using the
|
||||
<<delete-async-eql-search-api,delete async EQL search API>>.
|
||||
|
||||
[discrete]
|
||||
[[eql-search-case-sensitive]]
|
||||
=== Run a case-sensitive EQL search
|
||||
|
||||
By default, matching for EQL queries is case-insensitive. You can use the EQL
|
||||
search API's `case_sensitive` parameter to toggle case sensitivity on or off.
|
||||
|
||||
The following search request contains a query that matches `process` events
|
||||
with a `process.executable` containing `System32`.
|
||||
|
||||
Because the `case_sensitive` parameter is `true`, this query only matches
|
||||
`process.executable` values containing `System32` with the exact same capitalization.
|
||||
A `process.executable` value containing `system32` or `SYSTEM32` would not match this
|
||||
query.
|
||||
|
||||
[source,console]
|
||||
----
|
||||
GET /sec_logs/_eql/search
|
||||
{
|
||||
"keep_on_completion": true,
|
||||
"case_sensitive": true,
|
||||
"query": """
|
||||
process where stringContains(process.executable, "System32")
|
||||
"""
|
||||
}
|
||||
----
|
||||
|
|
@ -8,10 +8,7 @@
|
|||
|
||||
experimental::[]
|
||||
|
||||
[IMPORTANT]
|
||||
====
|
||||
{es} supports a subset of EQL syntax. See <<eql-limitations>>.
|
||||
====
|
||||
[IMPORTANT: {es} supports a subset of EQL syntax. See <<eql-syntax-limitations>.
|
||||
|
||||
[discrete]
|
||||
[[eql-basic-syntax]]
|
||||
|
|
@ -683,3 +680,43 @@ You can pass the output of a pipe to another pipe. This lets you use multiple
|
|||
pipes with a single query.
|
||||
|
||||
For a list of supported pipes, see <<eql-pipe-ref>>.
|
||||
|
||||
[discrete]
|
||||
[[eql-syntax-limitations]]
|
||||
=== Limitations
|
||||
|
||||
{es} EQL does not support the following features and syntax.
|
||||
|
||||
[discrete]
|
||||
[[eql-nested-fields]]
|
||||
==== EQL search on nested fields
|
||||
|
||||
You cannot use EQL to search the values of a <<nested,`nested`>> field or the
|
||||
sub-fields of a `nested` field. However, data streams and indices containing
|
||||
`nested` field mappings are otherwise supported.
|
||||
|
||||
[discrete]
|
||||
[[eql-unsupported-syntax]]
|
||||
==== Unsupported syntax
|
||||
|
||||
{es} supports a subset of {eql-ref}/index.html[EQL syntax]. {es} cannot run EQL
|
||||
queries that contain:
|
||||
|
||||
* Array functions:
|
||||
** {eql-ref}/functions.html#arrayContains[`arrayContains`]
|
||||
** {eql-ref}/functions.html#arrayCount[`arrayCount`]
|
||||
** {eql-ref}/functions.html#arraySearch[`arraySearch`]
|
||||
|
||||
* {eql-ref}/joins.html[Joins]
|
||||
|
||||
* {eql-ref}/basic-syntax.html#event-relationships[Lineage-related keywords]:
|
||||
** `child of`
|
||||
** `descendant of`
|
||||
** `event of`
|
||||
|
||||
* The following {eql-ref}/pipes.html[pipes]:
|
||||
** {eql-ref}/pipes.html#count[`count`]
|
||||
** {eql-ref}/pipes.html#filter[`filter`]
|
||||
** {eql-ref}/pipes.html#sort[`sort`]
|
||||
** {eql-ref}/pipes.html#unique[`unique`]
|
||||
** {eql-ref}/pipes.html#unique-count[`unique_count`]
|
||||
|
|
|
|||
|
|
@ -28,7 +28,7 @@ include::search/search-your-data.asciidoc[]
|
|||
|
||||
include::query-dsl.asciidoc[]
|
||||
|
||||
include::eql/index.asciidoc[]
|
||||
include::eql/eql.asciidoc[]
|
||||
|
||||
include::sql/index.asciidoc[]
|
||||
|
||||
|
|
|
|||
|
|
@ -959,6 +959,21 @@ See <<run-an-es-search>>.
|
|||
|
||||
See <<how-es-highlighters-work-internally>>.
|
||||
|
||||
[role="exclude",id="eql-search"]
|
||||
=== Run an EQL search
|
||||
|
||||
See <<run-an-eql-search>>.
|
||||
|
||||
[role="exclude",id="eql-limitations"]
|
||||
=== EQL limitations
|
||||
|
||||
See <<eql-syntax-limitations>>.
|
||||
|
||||
[role="exclude",id="eql-requirements"]
|
||||
=== EQL requirements
|
||||
|
||||
See <<eql-required-fields>>.
|
||||
|
||||
////
|
||||
[role="exclude",id="search-request-body"]
|
||||
=== Request body search
|
||||
|
|
|
|||
Loading…
Reference in New Issue