From a43f29cfc9a83a337ca22cbc1661db2420980dc4 Mon Sep 17 00:00:00 2001 From: Andrei Stefan Date: Thu, 24 Sep 2020 23:37:46 +0300 Subject: [PATCH] EQL: data streams tests for PIT and EQL sequences (#62850) (#62889) * PIT should run well with data streams (cherry picked from commit 0a89a7db848b015b797c7678874b5c9e33bbd650) --- .../search/action/OpenPointInTimeRequest.java | 5 + .../plugin/data-streams/qa/rest/build.gradle | 2 +- .../10_data_stream_resolvability.yml | 189 +++++++++++++++++- 3 files changed, 194 insertions(+), 2 deletions(-) diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/search/action/OpenPointInTimeRequest.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/search/action/OpenPointInTimeRequest.java index 962fc77b281..99b54cdf9a7 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/search/action/OpenPointInTimeRequest.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/search/action/OpenPointInTimeRequest.java @@ -92,6 +92,11 @@ public final class OpenPointInTimeRequest extends ActionRequest implements Indic return indicesOptions; } + @Override + public boolean includeDataStreams() { + return true; + } + public TimeValue keepAlive() { return keepAlive; } diff --git a/x-pack/plugin/data-streams/qa/rest/build.gradle b/x-pack/plugin/data-streams/qa/rest/build.gradle index 8bc751a3a4c..1b65032dd02 100644 --- a/x-pack/plugin/data-streams/qa/rest/build.gradle +++ b/x-pack/plugin/data-streams/qa/rest/build.gradle @@ -3,7 +3,7 @@ apply plugin: 'elasticsearch.yaml-rest-test' restResources { restApi { includeCore 'bulk', 'count', 'search', '_common', 'indices', 'index', 'cluster', 'rank_eval', 'reindex', 'update_by_query', 'delete_by_query' - includeXpack 'eql', 'indices', 'data_stream', 'migration', 'async_search', 'searchable_snapshots', 'rollup', 'graph', 'ilm' + includeXpack 'eql', 'indices', 'data_stream', 'migration', 'async_search', 'searchable_snapshots', 'rollup', 'graph', 'ilm', 'open_point_in_time', 'close_point_in_time' } restTests { includeXpack 'data_stream' diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/test/data_stream/10_data_stream_resolvability.yml b/x-pack/plugin/src/test/resources/rest-api-spec/test/data_stream/10_data_stream_resolvability.yml index 5955ffeeb5a..dd3ddccd049 100644 --- a/x-pack/plugin/src/test/resources/rest-api-spec/test/data_stream/10_data_stream_resolvability.yml +++ b/x-pack/plugin/src/test/resources/rest-api-spec/test/data_stream/10_data_stream_resolvability.yml @@ -476,7 +476,7 @@ - is_true: acknowledged --- -"Verify data stream resolvability in EQL search API": +"Verify data stream resolvability in EQL search API with no sequences": - skip: version: " - 7.9.1" reason: "EQL data stream resolution only properly works from 7.9.2" @@ -538,3 +538,190 @@ indices.delete_data_stream: name: simple-data-stream1 - is_true: acknowledged + +--- +"Verify data stream resolvability in EQL search API with sequences": + - skip: + version: " - 7.9.99" + reason: "EQL data stream resolution with queries using sequences only properly works from 7.10.0 (when point-in-time is introduced)" + features: allowed_warnings + + - do: + allowed_warnings: + - "index template [my-template1] has index patterns [simple-data-stream1] matching patterns from existing older templates [global] with patterns (global => [*]); this template [my-template1] will take precedence during new index creation" + indices.put_index_template: + name: my-template1 + body: + index_patterns: [simple-data-stream1] + template: + mappings: + properties: + '@timestamp': + type: date + data_stream: {} + + - do: + indices.create_data_stream: + name: simple-data-stream1 + + - do: + bulk: + refresh: true + body: + - '{"create":{"_index":"simple-data-stream1","_id": "1"}}' + - '{"@timestamp":"2020-02-03T12:34:56Z","event":{"category":"process"},"user":"SYSTEM"}' + - '{"create":{"_index":"simple-data-stream1", "_id": "2"}}' + - '{"@timestamp":"2020-02-03T12:35:00Z","event":{"category":"file"},"user":"SYSTEM","file":{"name":"sample.exe"}}' + + - do: + eql.search: + index: simple-data-stream1 + body: + query: 'sequence by user [process where user == "SYSTEM"] [file where file.name == "sample.exe"]' + + - match: {timed_out: false} + - match: {hits.total.value: 1} + - match: {hits.total.relation: "eq"} + - match: {hits.sequences.0.join_keys.0: "SYSTEM"} + - match: {hits.sequences.0.events.0._id: "1"} + - match: {hits.sequences.0.events.1._id: "2"} + + - do: + eql.search: + index: simple-data-s* + body: + query: 'sequence by user [process where user == "SYSTEM"] [file where file.name == "sample.exe"]' + + - match: {timed_out: false} + - match: {hits.total.value: 1} + - match: {hits.total.relation: "eq"} + - match: {hits.sequences.0.join_keys.0: "SYSTEM"} + - match: {hits.sequences.0.events.0._id: "1"} + - match: {hits.sequences.0.events.1._id: "2"} + + - do: + indices.delete_data_stream: + name: simple-data-stream1 + - is_true: acknowledged + +--- +"Verify data stream resolvability in PIT searches": + - skip: + version: " - 7.9.99" + reason: "Point-In-Time was introduced in 7.10.0" + features: allowed_warnings + + - do: + allowed_warnings: + - "index template [my-template1] has index patterns [simple-data-stream1] matching patterns from existing older templates [global] with patterns (global => [*]); this template [my-template1] will take precedence during new index creation" + indices.put_index_template: + name: my-template1 + body: + index_patterns: [simple-data-stream1] + template: + mappings: + properties: + '@timestamp': + type: date + data_stream: {} + + - do: + indices.create_data_stream: + name: simple-data-stream1 + + - do: + bulk: + refresh: true + body: + - '{"create":{"_index":"simple-data-stream1","_id": "1"}}' + - '{"id": 1, "foo": "bar", "age": 18, "@timestamp":"2020-02-03T12:34:56Z"}' + - '{"create":{"_index":"simple-data-stream1", "_id": "5"}}' + - '{"id": 5, "foo": "bar", "age": 18, "@timestamp":"2020-02-04T12:34:56Z"}' + - '{"create":{"_index":"simple-data-stream1", "_id": "123"}}' + - '{"id": 123, "foo": "bar", "age": 22, "@timestamp":"2020-02-05T12:34:56Z"}' + + - do: + open_point_in_time: + index: simple-data-s* + keep_alive: 5m + - set: {id: point_in_time_id} + + - do: + search: + rest_total_hits_as_int: true + body: + size: 1 + query: + match: + foo: bar + sort: [{ age: desc }, { id: desc }] + pit: + id: "$point_in_time_id" + keep_alive: 1m + + - match: {hits.total: 3 } + - length: {hits.hits: 1 } + - match: {hits.hits.0._index: .ds-simple-data-stream1-000001 } + - match: {hits.hits.0._id: "123" } + - match: {hits.hits.0.sort: [22, 123] } + + - do: + search: + rest_total_hits_as_int: true + body: + size: 1 + query: + match: + foo: bar + sort: [ { age: desc }, { id: desc } ] + search_after: [22, 123] + pit: + id: "$point_in_time_id" + + - match: {hits.total: 3} + - length: {hits.hits: 1 } + - match: {hits.hits.0._index: .ds-simple-data-stream1-000001 } + - match: {hits.hits.0._id: "5" } + - match: {hits.hits.0.sort: [18, 5] } + + - do: + search: + rest_total_hits_as_int: true + body: + size: 1 + query: + match: + foo: bar + sort: [{ age: desc }, { id: desc } ] + search_after: [18, 5] + pit: + id: "$point_in_time_id" + keep_alive: 1m + + - match: {hits.total: 3} + - length: {hits.hits: 1 } + - match: {hits.hits.0._index: .ds-simple-data-stream1-000001 } + - match: {hits.hits.0._id: "1" } + - match: {hits.hits.0.sort: [18, 1] } + + - do: + search: + rest_total_hits_as_int: true + body: + size: 1 + query: + match: + foo: bar + sort: [{ age: desc }, { id: desc } ] + search_after: [18, 1] + pit: + id: "$point_in_time_id" + keep_alive: 1m + + - match: {hits.total: 3} + - length: {hits.hits: 0 } + + - do: + close_point_in_time: + body: + id: "$point_in_time_id"