diff --git a/docs/reference/eql/eql-search-api.asciidoc b/docs/reference/eql/eql-search-api.asciidoc index 124c4ecb7dc..a933034c8fa 100644 --- a/docs/reference/eql/eql-search-api.asciidoc +++ b/docs/reference/eql/eql-search-api.asciidoc @@ -554,7 +554,12 @@ the events in ascending, lexicographic order. }, "sort": [ 1607252647000 - ] + ], + "fields": { + "@timestamp": [ + "1607252647000" + ] + } }, { "_index": "my_index", @@ -585,7 +590,12 @@ the events in ascending, lexicographic order. }, "sort": [ 1607339228000 - ] + ], + "fields": { + "@timestamp": [ + "1607339228000" + ] + } } ] } diff --git a/docs/reference/eql/search.asciidoc b/docs/reference/eql/search.asciidoc index 218974dd5bb..b1aaae83956 100644 --- a/docs/reference/eql/search.asciidoc +++ b/docs/reference/eql/search.asciidoc @@ -100,6 +100,11 @@ https://en.wikipedia.org/wiki/Unix_time[Unix epoch], in ascending order. "path": "C:\\Windows\\System32\\cmd.exe" } }, + "fields": { + "@timestamp": [ + "1607252645000" + ] + }, "sort": [ 1607252645000 ] @@ -124,6 +129,11 @@ https://en.wikipedia.org/wiki/Unix_time[Unix epoch], in ascending order. "path": "C:\\Windows\\System32\\cmd.exe" } }, + "fields": { + "@timestamp": [ + "1607339167000" + ] + }, "sort": [ 1607339167000 ] @@ -171,6 +181,7 @@ GET /sec_logs/_eql/search """ } ---- +// TEST[s/search/search\?filter_path\=\-\*\.sequences\.events\.\*fields/] The API returns the following response. Matching events in the `hits.sequences.events` property are sorted by @@ -219,11 +230,6 @@ the https://en.wikipedia.org/wiki/Unix_time[Unix epoch], in ascending order. "path": "C:\\Windows\\System32\\cmd.exe" } }, - "fields": { - "@timestamp": [ - "1607339228000" - ] - }, "sort": [ 1607339228000 ] @@ -248,11 +254,6 @@ the https://en.wikipedia.org/wiki/Unix_time[Unix epoch], in ascending order. "path": "C:\\Windows\\System32\\regsvr32.exe" } }, - "fields": { - "@timestamp": [ - "1607339229000" - ] - }, "sort": [ 1607339229000 ] @@ -297,6 +298,7 @@ GET /sec_logs/_eql/search """ } ---- +// TEST[s/search/search\?filter_path\=\-\*\.sequences\.events\.\*fields/] The API returns the following response. The `hits.sequences.join_keys` property contains the shared `agent.id` value for each matching event. @@ -346,11 +348,6 @@ contains the shared `agent.id` value for each matching event. "path": "C:\\Windows\\System32\\cmd.exe" } }, - "fields": { - "@timestamp": [ - "1607339228000" - ] - }, "sort": [ 1607339228000 ] @@ -375,11 +372,6 @@ contains the shared `agent.id` value for each matching event. "path": "C:\\Windows\\System32\\regsvr32.exe" } }, - "fields": { - "@timestamp": [ - "1607339229000" - ] - }, "sort": [ 1607339229000 ] @@ -515,11 +507,16 @@ tiebreaker for events with the same timestamp. "path": "C:\\Windows\\System32\\cmd.exe" } }, + "fields": { + "@timestamp": [ + "1607252645000" + ] + }, "sort": [ 1607252645000, <1> "edwCRnyD" <2> - ] - }, + ] + }, { "_index": "sec_logs", "_type": "_doc", @@ -540,6 +537,11 @@ tiebreaker for events with the same timestamp. "path": "C:\\Windows\\System32\\cmd.exe" } }, + "fields": { + "@timestamp": [ + "1607339167000" + ] + }, "sort": [ 1607339167000, <1> "cMyt5SZ2" <2> diff --git a/x-pack/plugin/eql/qa/common/src/main/resources/test_queries_supported.toml b/x-pack/plugin/eql/qa/common/src/main/resources/test_queries_supported.toml index 85abee178b1..ca1350174f3 100644 --- a/x-pack/plugin/eql/qa/common/src/main/resources/test_queries_supported.toml +++ b/x-pack/plugin/eql/qa/common/src/main/resources/test_queries_supported.toml @@ -1,7 +1,6 @@ # This file is populated with additional EQL queries that were not present in the original EQL python implementation # test_queries.toml file in order to keep the original unchanges and easier to sync with the EQL reference implementation tests. - [[queries]] expected_event_ids = [95] query = ''' diff --git a/x-pack/plugin/eql/qa/common/src/main/resources/test_queries_unsupported.toml b/x-pack/plugin/eql/qa/common/src/main/resources/test_queries_unsupported.toml index c0dc68cce52..6f53a93d9e6 100644 --- a/x-pack/plugin/eql/qa/common/src/main/resources/test_queries_unsupported.toml +++ b/x-pack/plugin/eql/qa/common/src/main/resources/test_queries_unsupported.toml @@ -12,6 +12,9 @@ # query = 'process where serial_event_id = 1' # expected_event_ids = [1] +[[queries]] +expected_event_ids = [] +query = 'process where missing_field != null' # fails because of string check - msbuild does not match MSBuild [[queries]] @@ -20,19 +23,10 @@ sequence by unique_pid [process where opcode=1 and process_name == 'msbuild.exe' expected_event_ids = [75273, 75304] description = "test that process sequences are working correctly" - -[[queries]] -query = 'process where true | head 6' -expected_event_ids = [1, 2, 3, 4, 5, 6] - [[queries]] expected_event_ids = [] query = 'process where missing_field != null' -[[queries]] -expected_event_ids = [1, 2, 3, 4, 5] -query = 'process where bad_field == null | head 5' - [[queries]] tags = ["comparisons", "pipes"] query = ''' @@ -65,25 +59,6 @@ process where true ''' expected_event_ids = [9, 10] -[[queries]] -note = "check that comparisons against null values return false" -expected_event_ids = [] -query = ''' -process where not (exit_code > -1) - and serial_event_id in (58, 64, 69, 74, 80, 85, 90, 93, 94) -| head 10 -''' - -[[queries]] -note = "check that comparisons against null values return false" -expected_event_ids = [1, 2, 3, 4, 5, 6, 7] -query = 'process where not (exit_code > -1) | head 7' - -[[queries]] -note = "check that comparisons against null values return false" -expected_event_ids = [1, 2, 3, 4, 5, 6, 7] -query = 'process where not (-1 < exit_code) | head 7' - [[queries]] query = 'process where (serial_event_id<9 and serial_event_id >= 7) or (opcode == pid)' expected_event_ids = [7, 8] @@ -182,12 +157,6 @@ process where opcode=1 and process_name == "smss.exe" ''' expected_event_ids = [78] -[[queries]] -query = ''' -file where true -| tail 3''' -expected_event_ids = [92, 95, 96] - [[queries]] query = ''' process where opcode in (1,3) and process_name in (parent_process_name, "SYSTEM") @@ -256,17 +225,6 @@ sequence expected_event_ids = [1, 2, 2, 3] -[[queries]] -query = ''' -sequence - [file where event_subtype_full == "file_create_event"] by file_path - [process where opcode == 1] by process_path - [process where opcode == 2] by process_path - [file where event_subtype_full == "file_delete_event"] by file_path -| head 4 -| tail 2''' -expected_event_ids = [67, 68, 69, 70, 72, 73, 74, 75] - [[queries]] query = ''' sequence with maxspan=1d @@ -322,14 +280,6 @@ sequence with maxspan=0.5s | tail 2''' expected_event_ids = [] -[[queries]] -query = ''' -sequence - [file where opcode=0] by unique_pid - [file where opcode=0] by unique_pid -| head 1''' -expected_event_ids = [55, 61] - [[queries]] query = ''' sequence @@ -466,6 +416,59 @@ query = ''' registry where length(bad_field) > 0 ''' +[[queries]] +expected_event_ids = [1, 2, 3, 4, 5] +query = 'process where bad_field == null | head 5' + +[[queries]] +note = "check that comparisons against null values return false" +expected_event_ids = [58, 64, 69, 74, 80, 85, 90, 93, 94, 75303] +query = 'process where exit_code >= 0' + +[[queries]] +note = "check that comparisons against null values return false" +expected_event_ids = [58, 64, 69, 74, 80, 85, 90, 93, 94, 75303] +query = 'process where 0 <= exit_code' + +[[queries]] +note = "check that comparisons against null values return false" +expected_event_ids = [58, 64, 69, 74, 80, 85, 90, 93, 94, 75303] +query = 'process where exit_code <= 0' + +[[queries]] +note = "check that comparisons against null values return false" +expected_event_ids = [58, 64, 69, 74, 80, 85, 90, 93, 94, 75303] +query = 'process where exit_code < 1' + +[[queries]] +note = "check that comparisons against null values return false" +expected_event_ids = [58, 64, 69, 74, 80, 85, 90, 93, 94, 75303] +query = 'process where exit_code > -1' + +[[queries]] +note = "check that comparisons against null values return false" +expected_event_ids = [58, 64, 69, 74, 80, 85, 90, 93, 94, 75303] +query = 'process where -1 < exit_code' + +[[queries]] +note = "check that comparisons against null values return false" +expected_event_ids = [] +query = ''' +process where not (exit_code > -1) + and serial_event_id in (58, 64, 69, 74, 80, 85, 90, 93, 94) +| head 10 +''' + +[[queries]] +note = "check that comparisons against null values return false" +expected_event_ids = [1, 2, 3, 4, 5, 6, 7] +query = 'process where not (exit_code > -1) | head 7' + +[[queries]] +note = "check that comparisons against null values return false" +expected_event_ids = [1, 2, 3, 4, 5, 6, 7] +query = 'process where not (-1 < exit_code) | head 7' + [[queries]] expected_event_ids = [1, 55, 57, 63, 75304] query = ''' @@ -539,16 +542,6 @@ expected_event_ids = [55, 95] query = 'process where event of [process where process_name = "python.exe" ]' expected_event_ids = [48, 50, 51, 54, 93] -[[queries]] -query = ''' -sequence by user_name - [file where opcode=0] by file_path - [process where opcode=1] by process_path - [process where opcode=2] by process_path - [file where opcode=2] by file_path -| tail 1''' -expected_event_ids = [88, 89, 90, 91] - [[queries]] query = ''' sequence by user_name @@ -567,16 +560,6 @@ until [process where opcode=5] by ppid,process_path | head 2''' expected_event_ids = [55, 59, 61, 65] -[[queries]] -query = ''' -sequence by pid - [file where opcode=0] by file_path - [process where opcode=1] by process_path - [process where opcode=2] by process_path - [file where opcode=2] by file_path -| tail 1''' -expected_event_ids = [] - [[queries]] query = ''' join by user_name diff --git a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/execution/assembler/Criterion.java b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/execution/assembler/Criterion.java index a271911df70..18294541dac 100644 --- a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/execution/assembler/Criterion.java +++ b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/execution/assembler/Criterion.java @@ -9,25 +9,35 @@ package org.elasticsearch.xpack.eql.execution.assembler; import org.elasticsearch.search.SearchHit; import org.elasticsearch.search.builder.SearchSourceBuilder; import org.elasticsearch.xpack.eql.EqlIllegalArgumentException; +import org.elasticsearch.xpack.eql.execution.search.QueryRequest; import org.elasticsearch.xpack.ql.execution.search.extractor.HitExtractor; import java.util.List; -public class Criterion { +public class Criterion implements QueryRequest { private final SearchSourceBuilder searchSource; private final List keyExtractors; private final HitExtractor timestampExtractor; private final HitExtractor tiebreakerExtractor; + // search after markers + private Object[] startMarker; + private Object[] stopMarker; + + //TODO: should accept QueryRequest instead of another SearchSourceBuilder public Criterion(SearchSourceBuilder searchSource, List searchAfterExractors, HitExtractor timestampExtractor, HitExtractor tiebreakerExtractor) { this.searchSource = searchSource; this.keyExtractors = searchAfterExractors; this.timestampExtractor = timestampExtractor; this.tiebreakerExtractor = tiebreakerExtractor; + + this.startMarker = null; + this.stopMarker = null; } + @Override public SearchSourceBuilder searchSource() { return searchSource; } @@ -64,8 +74,34 @@ public class Criterion { throw new EqlIllegalArgumentException("Expected tiebreaker to be Comparable but got {}", tb); } - public void fromMarkers(Object[] markers) { - // TODO: this is likely to be rewritten afterwards - searchSource.searchAfter(markers); + public Object[] startMarker() { + return startMarker; + } + + public Object[] stopMarker() { + return stopMarker; + } + + private Object[] marker(SearchHit hit) { + long timestamp = timestamp(hit); + Object tiebreaker = null; + if (tiebreakerExtractor() != null) { + tiebreaker = tiebreaker(hit); + } + + return tiebreaker != null ? new Object[] { timestamp, tiebreaker } : new Object[] { timestamp }; + } + + public void startMarker(SearchHit hit) { + startMarker = marker(hit); + } + + public void stopMarker(SearchHit hit) { + stopMarker = marker(hit); + } + + public Criterion useMarker(Object[] marker) { + searchSource.searchAfter(marker); + return this; } } \ No newline at end of file diff --git a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/execution/assembler/Executable.java b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/execution/assembler/Executable.java index 7d819cf0155..33531702650 100644 --- a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/execution/assembler/Executable.java +++ b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/execution/assembler/Executable.java @@ -7,9 +7,9 @@ package org.elasticsearch.xpack.eql.execution.assembler; import org.elasticsearch.action.ActionListener; -import org.elasticsearch.xpack.eql.session.Results; +import org.elasticsearch.xpack.eql.session.Payload; public interface Executable { - void execute(ActionListener resultsListener); + void execute(ActionListener resultsListener); } diff --git a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/execution/assembler/ExecutionManager.java b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/execution/assembler/ExecutionManager.java index 1ce6d46e5ea..9a95ead3e55 100644 --- a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/execution/assembler/ExecutionManager.java +++ b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/execution/assembler/ExecutionManager.java @@ -6,59 +6,44 @@ package org.elasticsearch.xpack.eql.execution.assembler; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; -import org.elasticsearch.action.ActionListener; -import org.elasticsearch.action.search.SearchRequest; -import org.elasticsearch.client.Client; -import org.elasticsearch.common.unit.TimeValue; -import org.elasticsearch.search.SearchHit; -import org.elasticsearch.search.builder.SearchSourceBuilder; -import org.elasticsearch.tasks.TaskCancelledException; import org.elasticsearch.xpack.eql.EqlIllegalArgumentException; -import org.elasticsearch.xpack.eql.execution.listener.RuntimeUtils; -import org.elasticsearch.xpack.eql.execution.payload.Payload; -import org.elasticsearch.xpack.eql.execution.payload.SearchResponsePayload; -import org.elasticsearch.xpack.eql.execution.search.SourceGenerator; +import org.elasticsearch.xpack.eql.execution.search.BasicQueryClient; +import org.elasticsearch.xpack.eql.execution.search.Limit; +import org.elasticsearch.xpack.eql.execution.search.QueryRequest; +import org.elasticsearch.xpack.eql.execution.search.RuntimeUtils; import org.elasticsearch.xpack.eql.execution.search.extractor.FieldHitExtractor; import org.elasticsearch.xpack.eql.execution.search.extractor.TimestampFieldHitExtractor; import org.elasticsearch.xpack.eql.plan.physical.EsQueryExec; import org.elasticsearch.xpack.eql.plan.physical.PhysicalPlan; import org.elasticsearch.xpack.eql.querydsl.container.FieldExtractorRegistry; -import org.elasticsearch.xpack.eql.querydsl.container.QueryContainer; import org.elasticsearch.xpack.eql.session.EqlConfiguration; import org.elasticsearch.xpack.eql.session.EqlSession; import org.elasticsearch.xpack.ql.execution.search.extractor.HitExtractor; import org.elasticsearch.xpack.ql.expression.Attribute; import org.elasticsearch.xpack.ql.expression.Expression; import org.elasticsearch.xpack.ql.expression.Expressions; +import org.elasticsearch.xpack.ql.expression.Order.OrderDirection; import org.elasticsearch.xpack.ql.util.Check; -import org.elasticsearch.xpack.ql.util.StringUtils; import java.util.ArrayList; import java.util.List; -import static org.elasticsearch.action.ActionListener.wrap; -import static org.elasticsearch.xpack.eql.execution.listener.RuntimeUtils.prepareRequest; - -public class ExecutionManager implements QueryClient { - - private static final Logger log = LogManager.getLogger(ExecutionManager.class); +public class ExecutionManager { + private final EqlSession session; private final EqlConfiguration cfg; - private final Client client; - private final TimeValue keepAlive; - private final String indices; public ExecutionManager(EqlSession eqlSession) { + this.session = eqlSession; this.cfg = eqlSession.configuration(); - this.client = eqlSession.client(); - this.keepAlive = cfg.requestTimeout(); - this.indices = cfg.indexAsWildcard(); } - - public Executable assemble(List> listOfKeys, List plans, Attribute timestamp, Attribute tiebreaker) { + public Executable assemble(List> listOfKeys, + List plans, + Attribute timestamp, + Attribute tiebreaker, + OrderDirection direction, + Limit limit) { FieldExtractorRegistry extractorRegistry = new FieldExtractorRegistry(); List criteria = new ArrayList<>(plans.size() - 1); @@ -75,12 +60,10 @@ public class ExecutionManager implements QueryClient { // search query // TODO: this could be generalized into an exec only query Check.isTrue(query instanceof EsQueryExec, "Expected a query but got [{}]", query.getClass()); - QueryContainer container = ((EsQueryExec) query).queryContainer(); - SearchSourceBuilder searchSource = SourceGenerator.sourceBuilder(container, cfg.filter(), cfg.size()); - - criteria.add(new Criterion(searchSource, keyExtractors, tsExtractor, tbExtractor)); + QueryRequest request = ((EsQueryExec) query).queryRequest(session); + criteria.add(new Criterion(request.searchSource(), keyExtractors, tsExtractor, tbExtractor)); } - return new SequenceRuntime(criteria, this); + return new SequenceRuntime(criteria, new BasicQueryClient(session), direction == OrderDirection.DESC, limit); } private HitExtractor timestampExtractor(HitExtractor hitExtractor) { @@ -102,20 +85,4 @@ public class ExecutionManager implements QueryClient { } return extractors; } - - @Override - public void query(SearchSourceBuilder searchSource, ActionListener> listener) { - // set query timeout - searchSource.timeout(cfg.requestTimeout()); - - if (log.isTraceEnabled()) { - log.trace("About to execute query {} on {}", StringUtils.toString(searchSource), indices); - } - if (cfg.isCancelled()) { - throw new TaskCancelledException("cancelled"); - } - - SearchRequest search = prepareRequest(client, searchSource, false, indices); - client.search(search, wrap(sr -> listener.onResponse(new SearchResponsePayload(sr)), listener::onFailure)); - } } \ No newline at end of file diff --git a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/execution/assembler/KeyAndOrdinal.java b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/execution/assembler/KeyAndOrdinal.java index 4d37226a7f0..9dbdd12ae71 100644 --- a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/execution/assembler/KeyAndOrdinal.java +++ b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/execution/assembler/KeyAndOrdinal.java @@ -8,6 +8,8 @@ package org.elasticsearch.xpack.eql.execution.assembler; import org.elasticsearch.xpack.eql.execution.sequence.SequenceKey; +import java.util.Objects; + class KeyAndOrdinal { final SequenceKey key; final long timestamp; @@ -18,4 +20,30 @@ class KeyAndOrdinal { this.timestamp = timestamp; this.tiebreaker = tiebreaker; } + + @Override + public int hashCode() { + return Objects.hash(key, timestamp, tiebreaker); + } + + @Override + public boolean equals(Object obj) { + if (this == obj) { + return true; + } + + if (obj == null || getClass() != obj.getClass()) { + return false; + } + + KeyAndOrdinal other = (KeyAndOrdinal) obj; + return Objects.equals(key, other.key) + && Objects.equals(timestamp, other.timestamp) + && Objects.equals(tiebreaker, other.tiebreaker); + } + + @Override + public String toString() { + return key + "[" + timestamp + "][" + (tiebreaker != null ? Objects.toString(tiebreaker) : "") + "]"; + } } diff --git a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/execution/assembler/SequencePayload.java b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/execution/assembler/SequencePayload.java new file mode 100644 index 00000000000..e14860a280f --- /dev/null +++ b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/execution/assembler/SequencePayload.java @@ -0,0 +1,39 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.eql.execution.assembler; + +import org.elasticsearch.common.unit.TimeValue; +import org.elasticsearch.xpack.eql.execution.payload.AbstractPayload; +import org.elasticsearch.xpack.eql.execution.sequence.Sequence; +import org.elasticsearch.xpack.eql.session.Results.Type; + +import java.util.ArrayList; +import java.util.List; + +class SequencePayload extends AbstractPayload { + + private final List sequences; + + SequencePayload(List seq, boolean timedOut, TimeValue timeTook, Object[] nextKeys) { + super(timedOut, timeTook, nextKeys); + sequences = new ArrayList<>(seq.size()); + for (Sequence s : seq) { + sequences.add(new org.elasticsearch.xpack.eql.action.EqlSearchResponse.Sequence(s.key().asStringList(), s.hits())); + } + } + + @Override + public Type resultType() { + return Type.SEQUENCE; + } + + @SuppressWarnings("unchecked") + @Override + public List values() { + return (List) sequences; + } +} diff --git a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/execution/assembler/SequenceRuntime.java b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/execution/assembler/SequenceRuntime.java index 05c8b8aed63..e459f59c6d7 100644 --- a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/execution/assembler/SequenceRuntime.java +++ b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/execution/assembler/SequenceRuntime.java @@ -6,17 +6,22 @@ package org.elasticsearch.xpack.eql.execution.assembler; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; import org.elasticsearch.action.ActionListener; import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.search.SearchHit; -import org.elasticsearch.xpack.eql.execution.payload.Payload; +import org.elasticsearch.xpack.eql.execution.payload.ReversePayload; +import org.elasticsearch.xpack.eql.execution.search.Limit; +import org.elasticsearch.xpack.eql.execution.search.QueryClient; import org.elasticsearch.xpack.eql.execution.sequence.Sequence; import org.elasticsearch.xpack.eql.execution.sequence.SequenceKey; import org.elasticsearch.xpack.eql.execution.sequence.SequenceStateMachine; -import org.elasticsearch.xpack.eql.session.Results; +import org.elasticsearch.xpack.eql.session.Payload; +import org.elasticsearch.xpack.eql.util.ReversedIterator; import org.elasticsearch.xpack.ql.execution.search.extractor.HitExtractor; -import java.util.ArrayList; +import java.util.Iterator; import java.util.List; import static org.elasticsearch.action.ActionListener.wrap; @@ -26,105 +31,101 @@ import static org.elasticsearch.action.ActionListener.wrap; */ class SequenceRuntime implements Executable { + private final Logger log = LogManager.getLogger(SequenceRuntime.class); + private final List criteria; // NB: just like in a list, this represents the total number of stages yet counting starts at 0 private final int numberOfStages; private final SequenceStateMachine stateMachine; private final QueryClient queryClient; + private final boolean descending; + private long startTime; - SequenceRuntime(List criteria, QueryClient queryClient) { + SequenceRuntime(List criteria, QueryClient queryClient, boolean descending, Limit limit) { this.criteria = criteria; this.numberOfStages = criteria.size(); this.queryClient = queryClient; boolean hasTiebreaker = criteria.get(0).tiebreakerExtractor() != null; - this.stateMachine = new SequenceStateMachine(numberOfStages, hasTiebreaker); + this.stateMachine = new SequenceStateMachine(numberOfStages, hasTiebreaker, limit); + + this.descending = descending; } @Override - public void execute(ActionListener resultsListener) { + public void execute(ActionListener listener) { startTime = System.currentTimeMillis(); - startSequencing(resultsListener); + log.info("Starting sequencing"); + queryStage(0, listener); } - private void startSequencing(ActionListener resultsListener) { - Criterion firstStage = criteria.get(0); - queryClient.query(firstStage.searchSource(), wrap(payload -> { - - // 1. execute last stage (find keys) - startTracking(payload, resultsListener); - - // 2. go descending through the rest of the stages, while adjusting the query - inspectStage(1, resultsListener); - - }, resultsListener::onFailure)); - } - - private void startTracking(Payload payload, ActionListener resultsListener) { - Criterion lastCriterion = criteria.get(0); - List hits = payload.values(); - - // nothing matches the first query, bail out early - if (hits.isEmpty()) { - resultsListener.onResponse(assembleResults()); - return; - } - - long tMin = Long.MAX_VALUE; - long tMax = Long.MIN_VALUE; - - Comparable bMin = null; - // we could have extracted that in the hit loop but that if would have been evaluated - // for every document - if (hits.isEmpty() == false) { - tMin = lastCriterion.timestamp(hits.get(0)); - tMax = lastCriterion.timestamp(hits.get(hits.size() - 1)); - - if (lastCriterion.tiebreakerExtractor() != null) { - bMin = lastCriterion.tiebreaker(hits.get(0)); - } - } - - for (SearchHit hit : hits) { - KeyAndOrdinal ko = findKey(hit, lastCriterion); - Sequence seq = new Sequence(ko.key, numberOfStages, ko.timestamp, ko.tiebreaker, hit); - stateMachine.trackSequence(seq, tMin, tMax); - } - stateMachine.setTimestampMarker(0, tMin); - if (bMin != null) { - stateMachine.setTiebreakerMarker(0, bMin); - } - } - - private void inspectStage(int stage, ActionListener resultsListener) { + private void queryStage(int stage, ActionListener listener) { // sequencing is done, return results - if (stage == numberOfStages) { - resultsListener.onResponse(assembleResults()); + if (hasFinished(stage)) { + listener.onResponse(sequencePayload()); return; } + // else continue finding matches Criterion currentCriterion = criteria.get(stage); - // narrow by the previous stage timestamp marker - currentCriterion.fromMarkers(stateMachine.getMarkers(stage - 1)); + if (stage > 0) { + // FIXME: revisit this during pagination since the second criterion need to be limited to the range of the first one + // narrow by the previous stage timestamp marker + + Criterion previous = criteria.get(stage - 1); + // if DESC, flip the markers (the stop becomes the start due to the reverse order), otherwise keep it accordingly + Object[] marker = descending && stage == 1 ? previous.stopMarker() : previous.startMarker(); + currentCriterion.useMarker(marker); + } - queryClient.query(currentCriterion.searchSource(), wrap(payload -> { - findMatches(stage, payload); - inspectStage(stage + 1, resultsListener); - }, resultsListener::onFailure)); + log.info("Querying stage {}", stage); + queryClient.query(currentCriterion, wrap(payload -> { + List hits = payload.values(); + + // nothing matches the query -> bail out + // FIXME: needs to be changed when doing pagination + if (hits.isEmpty()) { + listener.onResponse(sequencePayload()); + return; + } + + findMatches(stage, hits); + queryStage(stage + 1, listener); + }, listener::onFailure)); } - private void findMatches(int currentStage, Payload payload) { - Criterion currentCriterion = criteria.get(currentStage); - List hits = payload.values(); - + // hits are guaranteed to be non-empty + private void findMatches(int currentStage, List hits) { + // update criterion + Criterion criterion = criteria.get(currentStage); + criterion.startMarker(hits.get(0)); + criterion.stopMarker(hits.get(hits.size() - 1)); + // break the results per key - for (SearchHit hit : hits) { - KeyAndOrdinal ko = findKey(hit, currentCriterion); - stateMachine.match(currentStage, ko.key, ko.timestamp, ko.tiebreaker, hit); + // when dealing with descending order, queries outside the base are ASC (search_before) + // so look at the data in reverse (that is DESC) + for (Iterator it = descending ? new ReversedIterator<>(hits) : hits.iterator(); it.hasNext();) { + SearchHit hit = it.next(); + + KeyAndOrdinal ko = key(hit, criterion); + if (currentStage == 0) { + Sequence seq = new Sequence(ko.key, numberOfStages, ko.timestamp, ko.tiebreaker, hit); + long tStart = (long) criterion.startMarker()[0]; + long tStop = (long) criterion.stopMarker()[0]; + stateMachine.trackSequence(seq, tStart, tStop); + } else { + stateMachine.match(currentStage, ko.key, ko.timestamp, ko.tiebreaker, hit); + + // early skip in case of reaching the limit + // check the last stage to avoid calling the state machine in other stages + if (stateMachine.reachedLimit()) { + return; + } + } } } - private KeyAndOrdinal findKey(SearchHit hit, Criterion criterion) { + private KeyAndOrdinal key(SearchHit hit, Criterion criterion) { List keyExtractors = criterion.keyExtractors(); SequenceKey key; @@ -141,14 +142,14 @@ class SequenceRuntime implements Executable { return new KeyAndOrdinal(key, criterion.timestamp(hit), criterion.tiebreaker(hit)); } - private Results assembleResults() { - List done = stateMachine.completeSequences(); - List response = new ArrayList<>(done.size()); - for (Sequence s : done) { - response.add(new org.elasticsearch.xpack.eql.action.EqlSearchResponse.Sequence(s.key().asStringList(), s.hits())); - } - + private Payload sequencePayload() { + List completed = stateMachine.completeSequences(); TimeValue tookTime = new TimeValue(System.currentTimeMillis() - startTime); - return Results.fromSequences(tookTime, response); + SequencePayload payload = new SequencePayload(completed, false, tookTime, null); + return descending ? new ReversePayload(payload) : payload; + } + + private boolean hasFinished(int stage) { + return stage == numberOfStages; } } \ No newline at end of file diff --git a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/execution/payload/SequencePayload.java b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/execution/payload/AbstractPayload.java similarity index 60% rename from x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/execution/payload/SequencePayload.java rename to x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/execution/payload/AbstractPayload.java index 73cc395e80a..9cde6a102a6 100644 --- a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/execution/payload/SequencePayload.java +++ b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/execution/payload/AbstractPayload.java @@ -7,18 +7,15 @@ package org.elasticsearch.xpack.eql.execution.payload; import org.elasticsearch.common.unit.TimeValue; +import org.elasticsearch.xpack.eql.session.Payload; -import java.util.List; +public abstract class AbstractPayload implements Payload { -public class SequencePayload implements Payload { + private final boolean timedOut; + private final TimeValue timeTook; + private final Object[] nextKeys; - private final List seq; - private boolean timedOut; - private TimeValue timeTook; - private Object[] nextKeys; - - public SequencePayload(List seq, boolean timedOut, TimeValue timeTook, Object[] nextKeys) { - this.seq = seq; + protected AbstractPayload(boolean timedOut, TimeValue timeTook, Object[] nextKeys) { this.timedOut = timedOut; this.timeTook = timeTook; this.nextKeys = nextKeys; @@ -26,7 +23,7 @@ public class SequencePayload implements Payload { @Override public boolean timedOut() { - return false; + return timedOut; } @Override @@ -38,9 +35,4 @@ public class SequencePayload implements Payload { public Object[] nextKeys() { return nextKeys; } - - @Override - public List values() { - return seq; - } } diff --git a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/execution/payload/ReversePayload.java b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/execution/payload/ReversePayload.java new file mode 100644 index 00000000000..2f7a18f44ed --- /dev/null +++ b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/execution/payload/ReversePayload.java @@ -0,0 +1,49 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.eql.execution.payload; + +import org.elasticsearch.common.unit.TimeValue; +import org.elasticsearch.xpack.eql.session.Payload; +import org.elasticsearch.xpack.eql.session.Results.Type; + +import java.util.Collections; +import java.util.List; + +public class ReversePayload implements Payload { + + private final Payload delegate; + + public ReversePayload(Payload delegate) { + this.delegate = delegate; + Collections.reverse(delegate.values()); + } + + @Override + public Type resultType() { + return delegate.resultType(); + } + + @Override + public boolean timedOut() { + return delegate.timedOut(); + } + + @Override + public TimeValue timeTook() { + return delegate.timeTook(); + } + + @Override + public Object[] nextKeys() { + return delegate.nextKeys(); + } + + @Override + public List values() { + return delegate.values(); + } +} diff --git a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/execution/payload/SearchResponsePayload.java b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/execution/payload/SearchResponsePayload.java index 1bdef153424..284285de332 100644 --- a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/execution/payload/SearchResponsePayload.java +++ b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/execution/payload/SearchResponsePayload.java @@ -7,37 +7,29 @@ package org.elasticsearch.xpack.eql.execution.payload; import org.elasticsearch.action.search.SearchResponse; -import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.search.SearchHit; +import org.elasticsearch.xpack.eql.session.Results.Type; import java.util.Arrays; import java.util.List; -public class SearchResponsePayload implements Payload { +public class SearchResponsePayload extends AbstractPayload { - private final SearchResponse response; + private final List hits; public SearchResponsePayload(SearchResponse response) { - this.response = response; + super(response.isTimedOut(), response.getTook(), null); + hits = Arrays.asList(response.getHits().getHits()); } @Override - public boolean timedOut() { - return response.isTimedOut(); + public Type resultType() { + return Type.SEARCH_HIT; } + @SuppressWarnings("unchecked") @Override - public TimeValue timeTook() { - return response.getTook(); - } - - @Override - public Object[] nextKeys() { - throw new UnsupportedOperationException(); - } - - @Override - public List values() { - return Arrays.asList(response.getHits().getHits()); + public List values() { + return (List) hits; } } diff --git a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/execution/listener/BasicListener.java b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/execution/search/BasicListener.java similarity index 60% rename from x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/execution/listener/BasicListener.java rename to x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/execution/search/BasicListener.java index f7a9a277021..323328842dc 100644 --- a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/execution/listener/BasicListener.java +++ b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/execution/search/BasicListener.java @@ -4,36 +4,27 @@ * you may not use this file except in compliance with the Elastic License. */ -package org.elasticsearch.xpack.eql.execution.listener; +package org.elasticsearch.xpack.eql.execution.search; -import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.elasticsearch.action.ActionListener; -import org.elasticsearch.action.search.SearchRequest; import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.action.search.ShardSearchFailure; import org.elasticsearch.common.util.CollectionUtils; -import org.elasticsearch.search.SearchHit; import org.elasticsearch.xpack.eql.EqlIllegalArgumentException; -import org.elasticsearch.xpack.eql.session.Results; +import org.elasticsearch.xpack.eql.execution.payload.SearchResponsePayload; +import org.elasticsearch.xpack.eql.session.Payload; -import java.util.Arrays; -import java.util.List; - -import static org.elasticsearch.xpack.eql.execution.listener.RuntimeUtils.logSearchResponse; +import static org.elasticsearch.xpack.eql.execution.search.RuntimeUtils.logSearchResponse; public class BasicListener implements ActionListener { - private static final Logger log = LogManager.getLogger(BasicListener.class); + private static final Logger log = RuntimeUtils.QUERY_LOG; - private final ActionListener listener; - private final SearchRequest request; - - public BasicListener(ActionListener listener, - SearchRequest request) { + private final ActionListener listener; + public BasicListener(ActionListener listener) { this.listener = listener; - this.request = request; } @Override @@ -46,17 +37,15 @@ public class BasicListener implements ActionListener { handleResponse(response, listener); } } catch (Exception ex) { - listener.onFailure(ex); + onFailure(ex); } } - private void handleResponse(SearchResponse response, ActionListener listener) { + private void handleResponse(SearchResponse response, ActionListener listener) { if (log.isTraceEnabled()) { logSearchResponse(response, log); } - - List results = Arrays.asList(response.getHits().getHits()); - listener.onResponse(Results.fromHits(response.getTook(), results)); + listener.onResponse(new SearchResponsePayload(response)); } diff --git a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/execution/search/BasicQueryClient.java b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/execution/search/BasicQueryClient.java new file mode 100644 index 00000000000..7a4ee85dcfd --- /dev/null +++ b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/execution/search/BasicQueryClient.java @@ -0,0 +1,52 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.eql.execution.search; + +import org.apache.logging.log4j.Logger; +import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.search.SearchRequest; +import org.elasticsearch.client.Client; +import org.elasticsearch.search.builder.SearchSourceBuilder; +import org.elasticsearch.tasks.TaskCancelledException; +import org.elasticsearch.xpack.eql.session.EqlConfiguration; +import org.elasticsearch.xpack.eql.session.EqlSession; +import org.elasticsearch.xpack.eql.session.Payload; +import org.elasticsearch.xpack.ql.util.StringUtils; + +import static org.elasticsearch.xpack.eql.execution.search.RuntimeUtils.prepareRequest; + +public class BasicQueryClient implements QueryClient { + + private static final Logger log = RuntimeUtils.QUERY_LOG; + + private final EqlConfiguration cfg; + private final Client client; + private final String indices; + + public BasicQueryClient(EqlSession eqlSession) { + this.cfg = eqlSession.configuration(); + this.client = eqlSession.client(); + this.indices = cfg.indexAsWildcard(); + } + + @Override + public void query(QueryRequest request, ActionListener listener) { + SearchSourceBuilder searchSource = request.searchSource(); + // set query timeout + searchSource.timeout(cfg.requestTimeout()); + + if (log.isTraceEnabled()) { + log.trace("About to execute query {} on {}", StringUtils.toString(searchSource), indices); + } + if (cfg.isCancelled()) { + throw new TaskCancelledException("cancelled"); + } + + SearchRequest search = prepareRequest(client, searchSource, false, indices); + client.search(search, new BasicListener(listener)); + } +} diff --git a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/execution/search/Limit.java b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/execution/search/Limit.java new file mode 100644 index 00000000000..aa36922344d --- /dev/null +++ b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/execution/search/Limit.java @@ -0,0 +1,47 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.eql.execution.search; + +import org.elasticsearch.xpack.eql.util.MathUtils; + +import java.util.Objects; + +public class Limit { + + public final int limit; + public final int offset; + public final int total; + + public Limit(int limit, int offset) { + this.limit = limit; + this.offset = offset; + this.total = MathUtils.abs(limit) + offset; + } + + public int absLimit() { + return MathUtils.abs(limit); + } + + @Override + public int hashCode() { + return Objects.hash(limit, offset); + } + + @Override + public boolean equals(Object obj) { + if (this == obj) { + return true; + } + + if (obj == null || getClass() != obj.getClass()) { + return false; + } + + Limit other = (Limit) obj; + return Objects.equals(limit, other.limit) && Objects.equals(offset, other.offset); + } +} diff --git a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/execution/search/Querier.java b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/execution/search/Querier.java deleted file mode 100644 index 3c87772e30d..00000000000 --- a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/execution/search/Querier.java +++ /dev/null @@ -1,79 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License; - * you may not use this file except in compliance with the Elastic License. - */ - -package org.elasticsearch.xpack.eql.execution.search; - -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; -import org.elasticsearch.action.ActionListener; -import org.elasticsearch.action.search.SearchRequest; -import org.elasticsearch.action.search.SearchResponse; -import org.elasticsearch.client.Client; -import org.elasticsearch.common.Strings; -import org.elasticsearch.common.unit.TimeValue; -import org.elasticsearch.index.query.QueryBuilder; -import org.elasticsearch.search.builder.SearchSourceBuilder; -import org.elasticsearch.tasks.TaskCancelledException; -import org.elasticsearch.xpack.eql.execution.listener.BasicListener; -import org.elasticsearch.xpack.eql.querydsl.container.QueryContainer; -import org.elasticsearch.xpack.eql.session.EqlConfiguration; -import org.elasticsearch.xpack.eql.session.EqlSession; -import org.elasticsearch.xpack.eql.session.Results; -import org.elasticsearch.xpack.ql.index.IndexResolver; -import org.elasticsearch.xpack.ql.util.StringUtils; - -public class Querier { - - private static final Logger log = LogManager.getLogger(Querier.class); - - private final EqlConfiguration cfg; - private final Client client; - private final TimeValue keepAlive; - private final QueryBuilder filter; - - - public Querier(EqlSession eqlSession) { - this.cfg = eqlSession.configuration(); - this.client = eqlSession.client(); - this.keepAlive = cfg.requestTimeout(); - this.filter = cfg.filter(); - } - - - public void query(QueryContainer container, String index, ActionListener listener) { - // prepare the request - SearchSourceBuilder sourceBuilder = SourceGenerator.sourceBuilder(container, filter, cfg.size()); - - // set query timeout - sourceBuilder.timeout(cfg.requestTimeout()); - - if (log.isTraceEnabled()) { - log.trace("About to execute query {} on {}", StringUtils.toString(sourceBuilder), index); - } - if (cfg.isCancelled()) { - throw new TaskCancelledException("cancelled"); - } - SearchRequest search = prepareRequest(client, sourceBuilder, cfg.requestTimeout(), false, - Strings.commaDelimitedListToStringArray(index)); - - ActionListener l = new BasicListener(listener, search); - - client.search(search, l); - } - - public static SearchRequest prepareRequest(Client client, SearchSourceBuilder source, TimeValue timeout, boolean includeFrozen, - String... indices) { - return client.prepareSearch(indices) - // always track total hits accurately - .setTrackTotalHits(true) - .setAllowPartialSearchResults(false) - .setSource(source) - .setTimeout(timeout) - .setIndicesOptions( - includeFrozen ? IndexResolver.FIELD_CAPS_FROZEN_INDICES_OPTIONS : IndexResolver.FIELD_CAPS_INDICES_OPTIONS) - .request(); - } -} diff --git a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/execution/assembler/QueryClient.java b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/execution/search/QueryClient.java similarity index 55% rename from x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/execution/assembler/QueryClient.java rename to x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/execution/search/QueryClient.java index 55f46126362..f05250dcab8 100644 --- a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/execution/assembler/QueryClient.java +++ b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/execution/search/QueryClient.java @@ -4,17 +4,15 @@ * you may not use this file except in compliance with the Elastic License. */ -package org.elasticsearch.xpack.eql.execution.assembler; +package org.elasticsearch.xpack.eql.execution.search; import org.elasticsearch.action.ActionListener; -import org.elasticsearch.search.SearchHit; -import org.elasticsearch.search.builder.SearchSourceBuilder; -import org.elasticsearch.xpack.eql.execution.payload.Payload; +import org.elasticsearch.xpack.eql.session.Payload; /** * Infrastructure interface used to decouple listener consumers from the stateful classes holding client-references and co. */ -interface QueryClient { +public interface QueryClient { - void query(SearchSourceBuilder searchSource, ActionListener> listener); + void query(QueryRequest request, ActionListener listener); } diff --git a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/execution/search/QueryRequest.java b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/execution/search/QueryRequest.java new file mode 100644 index 00000000000..6d4c6621c57 --- /dev/null +++ b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/execution/search/QueryRequest.java @@ -0,0 +1,14 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.eql.execution.search; + +import org.elasticsearch.search.builder.SearchSourceBuilder; + +public interface QueryRequest { + + SearchSourceBuilder searchSource(); +} diff --git a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/execution/search/ReverseListener.java b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/execution/search/ReverseListener.java new file mode 100644 index 00000000000..a0b41553e55 --- /dev/null +++ b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/execution/search/ReverseListener.java @@ -0,0 +1,30 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.eql.execution.search; + +import org.elasticsearch.action.ActionListener; +import org.elasticsearch.xpack.eql.execution.payload.ReversePayload; +import org.elasticsearch.xpack.eql.session.Payload; + +public class ReverseListener implements ActionListener { + + private final ActionListener delegate; + + public ReverseListener(ActionListener delegate) { + this.delegate = delegate; + } + + @Override + public void onResponse(Payload response) { + delegate.onResponse(new ReversePayload(response)); + } + + @Override + public void onFailure(Exception e) { + delegate.onFailure(e); + } +} diff --git a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/execution/listener/RuntimeUtils.java b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/execution/search/RuntimeUtils.java similarity index 96% rename from x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/execution/listener/RuntimeUtils.java rename to x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/execution/search/RuntimeUtils.java index 2430a78af22..2bfa6a3ba13 100644 --- a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/execution/listener/RuntimeUtils.java +++ b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/execution/search/RuntimeUtils.java @@ -4,8 +4,9 @@ * you may not use this file except in compliance with the Elastic License. */ -package org.elasticsearch.xpack.eql.execution.listener; +package org.elasticsearch.xpack.eql.execution.search; +import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.elasticsearch.action.search.SearchRequest; import org.elasticsearch.action.search.SearchResponse; @@ -33,6 +34,8 @@ import java.util.Set; public final class RuntimeUtils { + static final Logger QUERY_LOG = LogManager.getLogger(QueryClient.class); + private RuntimeUtils() {} static void logSearchResponse(SearchResponse response, Logger logger) { @@ -102,4 +105,4 @@ public final class RuntimeUtils { includeFrozen ? IndexResolver.FIELD_CAPS_FROZEN_INDICES_OPTIONS : IndexResolver.FIELD_CAPS_INDICES_OPTIONS) .request(); } -} +} \ No newline at end of file diff --git a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/execution/search/SourceGenerator.java b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/execution/search/SourceGenerator.java index a04d72f8b25..750d0243563 100644 --- a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/execution/search/SourceGenerator.java +++ b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/execution/search/SourceGenerator.java @@ -62,6 +62,12 @@ public abstract class SourceGenerator { // set fetch size if (size != null) { int sz = size; + if (container.limit() != null) { + Limit limit = container.limit(); + // negative limit means DESC order but since the results are ordered ASC + // pagination becomes mute (since all the data needs to be returned) + sz = limit.limit > 0 ? Math.min(limit.total, size) : limit.total; + } if (source.size() == -1) { source.size(sz); diff --git a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/execution/sequence/SequenceFrame.java b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/execution/sequence/SequenceFrame.java index 0b789c542c2..cb4ba211eab 100644 --- a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/execution/sequence/SequenceFrame.java +++ b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/execution/sequence/SequenceFrame.java @@ -109,6 +109,7 @@ public class SequenceFrame { public void trim(int position) { sequences.subList(0, position).clear(); + // update min time if (sequences.isEmpty() == false) { min = sequences.get(0).currentTimestamp(); diff --git a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/execution/sequence/SequenceStateMachine.java b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/execution/sequence/SequenceStateMachine.java index c5411bf835e..df4ff63576a 100644 --- a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/execution/sequence/SequenceStateMachine.java +++ b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/execution/sequence/SequenceStateMachine.java @@ -8,6 +8,7 @@ package org.elasticsearch.xpack.eql.execution.sequence; import org.elasticsearch.common.collect.Tuple; import org.elasticsearch.search.SearchHit; +import org.elasticsearch.xpack.eql.execution.search.Limit; import java.util.LinkedList; import java.util.List; @@ -27,16 +28,20 @@ public class SequenceStateMachine { /** this ignores the key */ private final long[] timestampMarkers; - private final Comparable[] tiebreakerMarkers; + private final Comparable[] tiebreakerMarkers; private final boolean hasTieBreaker; private final int completionStage; /** list of completed sequences - separate to avoid polluting the other stages */ private final List completed; + + private int offset = 0; + private int limit = -1; + private boolean limitReached = false; - @SuppressWarnings({ "rawtypes", "unchecked" }) - public SequenceStateMachine(int stages, boolean hasTiebreaker) { + @SuppressWarnings("rawtypes") + public SequenceStateMachine(int stages, boolean hasTiebreaker, Limit limit) { this.completionStage = stages - 1; this.stageToKeys = new StageToKeys(completionStage); @@ -46,6 +51,12 @@ public class SequenceStateMachine { this.completed = new LinkedList<>(); this.hasTieBreaker = hasTiebreaker; + + // limit && offset + if (limit != null) { + this.offset = limit.offset; + this.limit = limit.absLimit(); + } } public List completeSequences() { @@ -70,16 +81,16 @@ public class SequenceStateMachine { public Object[] getMarkers(int stage) { long ts = timestampMarkers[stage]; - Comparable tb = tiebreakerMarkers[stage]; + Comparable tb = tiebreakerMarkers[stage]; return hasTieBreaker ? new Object[] { ts, tb } : new Object[] { ts }; } - public void trackSequence(Sequence sequence, long tMin, long tMax) { + public void trackSequence(Sequence sequence, long tStart, long tStop) { SequenceKey key = sequence.key(); stageToKeys.keys(0).add(key); SequenceFrame frame = keyToSequences.frame(0, key); - frame.setTimeFrame(tMin, tMax); + frame.setTimeFrame(tStart, tStop); frame.add(sequence); } @@ -94,14 +105,14 @@ public class SequenceStateMachine { if (frame == null || frame.isEmpty()) { return false; } - // pick the sequence with the highest timestamp lower than current match timestamp - Tuple before = frame.before(timestamp, tiebreaker); - if (before == null) { + // pick the sequence with the highest (for ASC) / lowest (for DESC) timestamp lower than current match timestamp + Tuple neighbour = frame.before(timestamp, tiebreaker); + if (neighbour == null) { return false; } - Sequence sequence = before.v1(); + Sequence sequence = neighbour.v1(); // eliminate the match and all previous values from the frame - frame.trim(before.v2() + 1); + frame.trim(neighbour.v2() + 1); // update sequence sequence.putMatch(stage, hit, timestamp, tiebreaker); @@ -112,11 +123,24 @@ public class SequenceStateMachine { // bump the stages if (stage == completionStage) { - completed.add(sequence); + // add the sequence only if needed + if (offset > 0) { + offset--; + } else { + if (limit < 0 || (limit > 0 && completed.size() < limit)) { + completed.add(sequence); + // update the bool lazily + limitReached = limit > 0 && completed.size() == limit; + } + } } else { stageToKeys.keys(stage).add(key); keyToSequences.frame(stage, key).add(sequence); } return true; } + + public boolean reachedLimit() { + return limitReached; + } } \ No newline at end of file diff --git a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/optimizer/Optimizer.java b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/optimizer/Optimizer.java index a86d1fb039e..5054d3c9e66 100644 --- a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/optimizer/Optimizer.java +++ b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/optimizer/Optimizer.java @@ -6,12 +6,19 @@ package org.elasticsearch.xpack.eql.optimizer; +import org.elasticsearch.xpack.eql.EqlIllegalArgumentException; import org.elasticsearch.xpack.eql.plan.logical.Join; import org.elasticsearch.xpack.eql.plan.logical.KeyedFilter; +import org.elasticsearch.xpack.eql.plan.logical.LimitWithOffset; import org.elasticsearch.xpack.eql.plan.physical.LocalRelation; import org.elasticsearch.xpack.eql.session.Results; +import org.elasticsearch.xpack.eql.util.MathUtils; import org.elasticsearch.xpack.eql.util.StringUtils; import org.elasticsearch.xpack.ql.expression.Expression; +import org.elasticsearch.xpack.ql.expression.Literal; +import org.elasticsearch.xpack.ql.expression.Order; +import org.elasticsearch.xpack.ql.expression.Order.NullsPosition; +import org.elasticsearch.xpack.ql.expression.Order.OrderDirection; import org.elasticsearch.xpack.ql.expression.predicate.logical.Not; import org.elasticsearch.xpack.ql.expression.predicate.nulls.IsNotNull; import org.elasticsearch.xpack.ql.expression.predicate.nulls.IsNull; @@ -31,11 +38,19 @@ import org.elasticsearch.xpack.ql.optimizer.OptimizerRules.ReplaceSurrogateFunct import org.elasticsearch.xpack.ql.optimizer.OptimizerRules.SetAsOptimized; import org.elasticsearch.xpack.ql.optimizer.OptimizerRules.TransformDirection; import org.elasticsearch.xpack.ql.plan.logical.Filter; +import org.elasticsearch.xpack.ql.plan.logical.Limit; import org.elasticsearch.xpack.ql.plan.logical.LogicalPlan; +import org.elasticsearch.xpack.ql.plan.logical.OrderBy; +import org.elasticsearch.xpack.ql.plan.logical.Project; import org.elasticsearch.xpack.ql.plan.logical.UnaryPlan; import org.elasticsearch.xpack.ql.rule.RuleExecutor; +import org.elasticsearch.xpack.ql.type.DataTypes; +import java.util.ArrayList; import java.util.Arrays; +import java.util.List; + +import static java.util.Collections.singletonList; public class Optimizer extends RuleExecutor { @@ -45,9 +60,8 @@ public class Optimizer extends RuleExecutor { @Override protected Iterable.Batch> batches() { - Batch substitutions = new Batch("Operator Replacement", Limiter.ONCE, - new ReplaceSurrogateFunction()); - + Batch substitutions = new Batch("Operator Replacement", Limiter.ONCE, new ReplaceSurrogateFunction()); + Batch operators = new Batch("Operator Optimization", new ConstantFolding(), // boolean @@ -61,19 +75,25 @@ public class Optimizer extends RuleExecutor { new CombineBinaryComparisons(), // prune/elimination new PruneFilters(), - new PruneLiteralsInOrderBy() - ); + new PruneLiteralsInOrderBy(), + new CombineLimits()); + + Batch ordering = new Batch("Implicit Order", + new SortByLimit(), + new PushDownOrderBy()); Batch local = new Batch("Skip Elasticsearch", new SkipEmptyFilter(), - new SkipEmptyJoin()); + new SkipEmptyJoin(), + new SkipQueryOnLimitZero()); Batch label = new Batch("Set as Optimized", Limiter.ONCE, new SetAsOptimized()); - return Arrays.asList(substitutions, operators, local, label); + return Arrays.asList(substitutions, operators, ordering, local, label); } + private static class ReplaceWildcards extends OptimizerRule { private static boolean isWildcard(Expression expr) { @@ -135,8 +155,8 @@ public class Optimizer extends RuleExecutor { static class PruneFilters extends org.elasticsearch.xpack.ql.optimizer.OptimizerRules.PruneFilters { @Override - protected LogicalPlan nonMatchingFilter(Filter filter) { - return new LocalRelation(filter.source(), filter.output()); + protected LogicalPlan skipPlan(Filter filter) { + return Optimizer.skipPlan(filter); } } @@ -154,14 +174,167 @@ public class Optimizer extends RuleExecutor { return plan; } } - + + static class SkipQueryOnLimitZero extends org.elasticsearch.xpack.ql.optimizer.OptimizerRules.SkipQueryOnLimitZero { + + @Override + protected LogicalPlan skipPlan(Limit limit) { + return Optimizer.skipPlan(limit); + } + } + + private static LogicalPlan skipPlan(UnaryPlan plan) { + return new LocalRelation(plan.source(), plan.output()); + } + + /** + * Combine tail and head into one limit. + * The rules moves up since the first limit is the one that defines whether it's the head (positive) or + * the tail (negative) limit of the data and the rest simply work in this space. + */ + static final class CombineLimits extends OptimizerRule { + + CombineLimits() { + super(TransformDirection.UP); + } + + @Override + protected LogicalPlan rule(LimitWithOffset limit) { + // bail out early + if (limit.child() instanceof LimitWithOffset == false) { + return limit; + } + + LimitWithOffset primary = (LimitWithOffset) limit.child(); + + int primaryLimit = (Integer) primary.limit().fold(); + int primaryOffset = primary.offset(); + // +1 means ASC, -1 descending and 0 if there are no results + int sign = Integer.signum(primaryLimit); + + int secondaryLimit = (Integer) limit.limit().fold(); + if (limit.offset() != 0) { + throw new EqlIllegalArgumentException("Limits with different offset not implemented yet"); + } + + // for the same direction + if (primaryLimit > 0 && secondaryLimit > 0) { + // consider the minimum + primaryLimit = Math.min(primaryLimit, secondaryLimit); + } else if (primaryLimit < 0 && secondaryLimit < 0) { + primaryLimit = Math.max(primaryLimit, secondaryLimit); + } else { + // the secondary limit cannot go beyond the primary - if it does it gets ignored + if (MathUtils.abs(secondaryLimit) < MathUtils.abs(primaryLimit)) { + primaryOffset += MathUtils.abs(primaryLimit + secondaryLimit); + // preserve order + primaryLimit = MathUtils.abs(secondaryLimit) * sign; + } + } + + Literal literal = new Literal(primary.limit().source(), primaryLimit, DataTypes.INTEGER); + return new LimitWithOffset(primary.source(), literal, primaryOffset, primary.child()); + } + } + + /** + * Align the implicit order with the limit (head means ASC or tail means DESC). + */ + static final class SortByLimit extends OptimizerRule { + + @Override + protected LogicalPlan rule(LimitWithOffset limit) { + if (limit.limit().foldable()) { + LogicalPlan child = limit.child(); + if (child instanceof OrderBy) { + OrderBy ob = (OrderBy) child; + if (PushDownOrderBy.isDefaultOrderBy(ob)) { + int l = (Integer) limit.limit().fold(); + OrderDirection direction = Integer.signum(l) > 0 ? OrderDirection.ASC : OrderDirection.DESC; + ob = new OrderBy(ob.source(), ob.child(), PushDownOrderBy.changeOrderDirection(ob.order(), direction)); + limit = new LimitWithOffset(limit.source(), limit.limit(), limit.offset(), ob); + } + } + } + + return limit; + } + } + + /** + * Push down the OrderBy into the actual queries before translating them. + * There is always an implicit order (timestamp + tiebreaker ascending). + */ + static final class PushDownOrderBy extends OptimizerRule { + + @Override + protected LogicalPlan rule(OrderBy orderBy) { + LogicalPlan plan = orderBy; + if (isDefaultOrderBy(orderBy)) { + LogicalPlan child = orderBy.child(); + // + // When dealing with sequences, the matching needs to happen ascending + // hence why the queries will always be ascending + // but if the order is descending, apply that only to the first query + // which is used to discover the window for which matching is being applied. + // + if (child instanceof Join) { + Join join = (Join) child; + List queries = join.queries(); + + // the main reason ASC is used is the lack of search_before (which is emulated through search_after + ASC) + List ascendingOrders = changeOrderDirection(orderBy.order(), OrderDirection.ASC); + // preserve the order direction as is (can be DESC) for the base query + List orderedQueries = new ArrayList<>(queries.size()); + boolean baseFilter = true; + for (KeyedFilter filter : queries) { + // preserve the order for the base query, everything else needs to be ascending + List pushedOrder = baseFilter ? orderBy.order() : ascendingOrders; + OrderBy order = new OrderBy(filter.source(), filter.child(), pushedOrder); + orderedQueries.add((KeyedFilter) filter.replaceChildren(singletonList(order))); + baseFilter = false; + } + + KeyedFilter until = join.until(); + OrderBy order = new OrderBy(until.source(), until.child(), ascendingOrders); + until = (KeyedFilter) until.replaceChildren(singletonList(order)); + + OrderDirection direction = orderBy.order().get(0).direction(); + plan = join.with(orderedQueries, until, direction); + } + } + return plan; + } + + private static boolean isDefaultOrderBy(OrderBy orderBy) { + LogicalPlan child = orderBy.child(); + // the default order by is the first pipe + // so it has to be on top of a event query or join/sequence + return child instanceof Project || child instanceof Join; + } + + private static List changeOrderDirection(List orders, Order.OrderDirection direction) { + List changed = new ArrayList<>(orders.size()); + boolean hasChanged = false; + for (Order order : orders) { + if (order.direction() != direction) { + order = new Order(order.source(), order.child(), direction, + direction == OrderDirection.ASC ? NullsPosition.FIRST : NullsPosition.LAST); + hasChanged = true; + } + changed.add(order); + } + return hasChanged ? changed : orders; + } + } + static class SkipEmptyJoin extends OptimizerRule { @Override protected LogicalPlan rule(Join plan) { // check for empty filters for (KeyedFilter filter : plan.queries()) { - if (filter.child() instanceof LocalRelation) { + if (filter.anyMatch(LocalRelation.class::isInstance)) { return new LocalRelation(plan.source(), plan.output(), Results.Type.SEQUENCE); } } diff --git a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/parser/EqlParser.java b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/parser/EqlParser.java index 93b46b1409e..ebe529c6bbd 100644 --- a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/parser/EqlParser.java +++ b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/parser/EqlParser.java @@ -134,16 +134,6 @@ public class EqlParser { this.ruleNames = ruleNames; } - @Override - public void exitPipe(EqlBaseParser.PipeContext context) { - Token token = context.PIPE().getSymbol(); - throw new ParsingException( - "Pipes are not supported", - null, - token.getLine(), - token.getCharPositionInLine()); - } - @Override public void exitProcessCheck(EqlBaseParser.ProcessCheckContext context) { Token token = context.relationship; diff --git a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/parser/ExpressionBuilder.java b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/parser/ExpressionBuilder.java index 6aa340eb348..66612d7b347 100644 --- a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/parser/ExpressionBuilder.java +++ b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/parser/ExpressionBuilder.java @@ -75,7 +75,12 @@ public class ExpressionBuilder extends IdentifierBuilder { @Override public List visitJoinKeys(JoinKeysContext ctx) { - return ctx != null ? visitList(ctx.expression(), Attribute.class) : emptyList(); + try { + return ctx != null ? visitList(ctx.expression(), Attribute.class) : emptyList(); + } catch (ClassCastException ex) { + Source source = source(ctx); + throw new ParsingException(source, "Unsupported join key ", source.text()); + } } @Override diff --git a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/parser/LogicalPlanBuilder.java b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/parser/LogicalPlanBuilder.java index 01bf0b66c0d..e6252f548ed 100644 --- a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/parser/LogicalPlanBuilder.java +++ b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/parser/LogicalPlanBuilder.java @@ -7,19 +7,25 @@ package org.elasticsearch.xpack.eql.parser; import org.antlr.v4.runtime.tree.ParseTree; import org.elasticsearch.common.unit.TimeValue; +import org.elasticsearch.common.util.set.Sets; +import org.elasticsearch.xpack.eql.parser.EqlBaseParser.BooleanExpressionContext; import org.elasticsearch.xpack.eql.parser.EqlBaseParser.EventFilterContext; import org.elasticsearch.xpack.eql.parser.EqlBaseParser.IntegerLiteralContext; import org.elasticsearch.xpack.eql.parser.EqlBaseParser.JoinContext; import org.elasticsearch.xpack.eql.parser.EqlBaseParser.JoinKeysContext; import org.elasticsearch.xpack.eql.parser.EqlBaseParser.JoinTermContext; import org.elasticsearch.xpack.eql.parser.EqlBaseParser.NumberContext; +import org.elasticsearch.xpack.eql.parser.EqlBaseParser.PipeContext; import org.elasticsearch.xpack.eql.parser.EqlBaseParser.SequenceContext; import org.elasticsearch.xpack.eql.parser.EqlBaseParser.SequenceParamsContext; import org.elasticsearch.xpack.eql.parser.EqlBaseParser.SequenceTermContext; +import org.elasticsearch.xpack.eql.parser.EqlBaseParser.StatementContext; import org.elasticsearch.xpack.eql.parser.EqlBaseParser.SubqueryContext; +import org.elasticsearch.xpack.eql.plan.logical.Head; import org.elasticsearch.xpack.eql.plan.logical.Join; import org.elasticsearch.xpack.eql.plan.logical.KeyedFilter; import org.elasticsearch.xpack.eql.plan.logical.Sequence; +import org.elasticsearch.xpack.eql.plan.logical.Tail; import org.elasticsearch.xpack.eql.plan.physical.LocalRelation; import org.elasticsearch.xpack.ql.expression.Attribute; import org.elasticsearch.xpack.ql.expression.EmptyAttribute; @@ -27,6 +33,7 @@ import org.elasticsearch.xpack.ql.expression.Expression; import org.elasticsearch.xpack.ql.expression.Expressions; import org.elasticsearch.xpack.ql.expression.Literal; import org.elasticsearch.xpack.ql.expression.Order; +import org.elasticsearch.xpack.ql.expression.Order.OrderDirection; import org.elasticsearch.xpack.ql.expression.UnresolvedAttribute; import org.elasticsearch.xpack.ql.expression.predicate.logical.And; import org.elasticsearch.xpack.ql.expression.predicate.operator.comparison.Equals; @@ -38,15 +45,21 @@ import org.elasticsearch.xpack.ql.plan.logical.UnresolvedRelation; import org.elasticsearch.xpack.ql.tree.Source; import org.elasticsearch.xpack.ql.type.DataTypes; import org.elasticsearch.xpack.ql.util.CollectionUtils; +import org.elasticsearch.xpack.ql.util.StringUtils; import java.util.ArrayList; import java.util.List; +import java.util.Set; import java.util.concurrent.TimeUnit; +import static java.util.Arrays.asList; import static java.util.Collections.emptyList; +import static java.util.Collections.singletonList; public abstract class LogicalPlanBuilder extends ExpressionBuilder { + private static final Set SUPPORTED_PIPES = Sets.newHashSet("count", "filter", "head", "sort", "tail", "unique", "unique_count"); + private final UnresolvedRelation RELATION = new UnresolvedRelation(Source.EMPTY, null, "", false, ""); private final EmptyAttribute UNSPECIFIED_FIELD = new EmptyAttribute(Source.EMPTY); @@ -62,9 +75,34 @@ public abstract class LogicalPlanBuilder extends ExpressionBuilder { return params.fieldTiebreaker() != null ? new UnresolvedAttribute(Source.EMPTY, params.fieldTiebreaker()) : UNSPECIFIED_FIELD; } + private OrderDirection defaultDirection() { + return OrderDirection.ASC; + } + + @Override + public Object visitStatement(StatementContext ctx) { + LogicalPlan plan = plan(ctx.query()); + + // the first pipe will be the implicit order + List orders = new ArrayList<>(2); + Source source = plan.source(); + orders.add(new Order(source, fieldTimestamp(), defaultDirection(), Order.NullsPosition.FIRST)); + // make sure to add the tiebreaker as well + Attribute tiebreaker = fieldTiebreaker(); + if (Expressions.isPresent(tiebreaker)) { + orders.add(new Order(source, tiebreaker, defaultDirection(), Order.NullsPosition.FIRST)); + } + plan = new OrderBy(source, plan, orders); + // add the actual declared pipes + for (PipeContext pipeCtx : ctx.pipe()) { + plan = pipe(pipeCtx, plan); + } + return plan; + } + @Override public LogicalPlan visitEventQuery(EqlBaseParser.EventQueryContext ctx) { - return new Project(source(ctx), visitEventFilter(ctx.eventFilter()), emptyList()); + return new Project(source(ctx), visitEventFilter(ctx.eventFilter()), defaultProjection()); } @Override @@ -83,19 +121,7 @@ public abstract class LogicalPlanBuilder extends ExpressionBuilder { condition = new And(source, eventMatch, condition); } - Filter filter = new Filter(source, RELATION, condition); - List orders = new ArrayList<>(2); - - // TODO: add implicit sorting - when pipes are added, this would better sit there (as a default pipe) - orders.add(new Order(source, fieldTimestamp(), Order.OrderDirection.ASC, Order.NullsPosition.FIRST)); - // make sure to add the tiebreaker as well - Attribute tiebreaker = fieldTiebreaker(); - if (Expressions.isPresent(tiebreaker)) { - orders.add(new Order(source, tiebreaker, Order.OrderDirection.ASC, Order.NullsPosition.FIRST)); - } - - OrderBy orderBy = new OrderBy(source, filter, orders); - return orderBy; + return new Filter(source, RELATION, condition); } @Override @@ -133,7 +159,7 @@ public abstract class LogicalPlanBuilder extends ExpressionBuilder { until = defaultUntil(source); } - return new Join(source, queries, until, fieldTimestamp(), fieldTiebreaker()); + return new Join(source, queries, until, fieldTimestamp(), fieldTiebreaker(), defaultDirection()); } private KeyedFilter defaultUntil(Source source) { @@ -149,14 +175,16 @@ public abstract class LogicalPlanBuilder extends ExpressionBuilder { List keys = CollectionUtils.combine(joinKeys, visitJoinKeys(joinCtx)); LogicalPlan eventQuery = visitEventFilter(subqueryCtx.eventFilter()); - List output = CollectionUtils.combine(keys, fieldTimestamp()); + LogicalPlan child = new Project(source(ctx), eventQuery, CollectionUtils.combine(keys, defaultProjection())); + return new KeyedFilter(source(ctx), child, keys, fieldTimestamp(), fieldTiebreaker()); + } + + private List defaultProjection() { Attribute fieldTieBreaker = fieldTiebreaker(); if (Expressions.isPresent(fieldTieBreaker)) { - output = CollectionUtils.combine(output, fieldTieBreaker); + return asList(fieldTimestamp(), fieldTiebreaker()); } - LogicalPlan child = new Project(source(ctx), eventQuery, output); - - return new KeyedFilter(source(ctx), child, keys, fieldTimestamp(), fieldTiebreaker()); + return singletonList(fieldTimestamp()); } @Override @@ -199,7 +227,7 @@ public abstract class LogicalPlanBuilder extends ExpressionBuilder { until = defaultUntil(source); } - return new Sequence(source, queries, until, maxSpan, fieldTimestamp(), fieldTiebreaker()); + return new Sequence(source, queries, until, maxSpan, fieldTimestamp(), fieldTiebreaker(), defaultDirection()); } public KeyedFilter visitSequenceTerm(SequenceTermContext ctx, List joinKeys) { @@ -262,4 +290,51 @@ public abstract class LogicalPlanBuilder extends ExpressionBuilder { text(numberCtx)); } } -} + + private LogicalPlan pipe(PipeContext ctx, LogicalPlan plan) { + String name = text(ctx.IDENTIFIER()); + + if (SUPPORTED_PIPES.contains(name) == false) { + List potentialMatches = StringUtils.findSimilar(name, SUPPORTED_PIPES); + + String msg = "Unrecognized pipe [{}]"; + if (potentialMatches.isEmpty() == false) { + String matchString = potentialMatches.toString(); + msg += ", did you mean " + (potentialMatches.size() == 1 + ? matchString + : "any of " + matchString) + "?"; + } + throw new ParsingException(source(ctx.IDENTIFIER()), msg, name); + } + + switch (name) { + case "head": + Expression headLimit = pipeIntArgument(source(ctx), name, ctx.booleanExpression()); + return new Head(source(ctx), headLimit, plan); + + case "tail": + Expression tailLimit = pipeIntArgument(source(ctx), name, ctx.booleanExpression()); + // negate the limit + return new Tail(source(ctx), tailLimit, plan); + + default: + throw new ParsingException(source(ctx), "Pipe [{}] is not supported yet", name); + } + } + + private Expression pipeIntArgument(Source source, String pipeName, List exps) { + int size = CollectionUtils.isEmpty(exps) ? 0 : exps.size(); + if (size != 1) { + throw new ParsingException(source, "Pipe [{}] expects exactly one argument but found [{}]", pipeName, size); + } + BooleanExpressionContext limitCtx = exps.get(0); + Expression expression = expression(limitCtx); + + if (expression.dataType().isInteger() == false || expression.foldable() == false || (int) expression.fold() < 0) { + throw new ParsingException(source(limitCtx), "Pipe [{}] expects a positive integer but found [{}]", pipeName, expression + .sourceText()); + } + + return expression; + } +} \ No newline at end of file diff --git a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/plan/logical/Head.java b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/plan/logical/Head.java new file mode 100644 index 00000000000..08079050a6f --- /dev/null +++ b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/plan/logical/Head.java @@ -0,0 +1,30 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.eql.plan.logical; + +import org.elasticsearch.xpack.ql.expression.Expression; +import org.elasticsearch.xpack.ql.plan.logical.Limit; +import org.elasticsearch.xpack.ql.plan.logical.LogicalPlan; +import org.elasticsearch.xpack.ql.tree.NodeInfo; +import org.elasticsearch.xpack.ql.tree.Source; + +public class Head extends LimitWithOffset { + + public Head(Source source, Expression limit, LogicalPlan child) { + super(source, limit, child); + } + + @Override + protected NodeInfo info() { + return NodeInfo.create(this, Head::new, limit(), child()); + } + + @Override + protected Head replaceChild(LogicalPlan newChild) { + return new Head(source(), limit(), newChild); + } +} \ No newline at end of file diff --git a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/plan/logical/Join.java b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/plan/logical/Join.java index 31e2785bb42..7c189a6ee85 100644 --- a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/plan/logical/Join.java +++ b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/plan/logical/Join.java @@ -10,6 +10,7 @@ import org.elasticsearch.xpack.eql.EqlIllegalArgumentException; import org.elasticsearch.xpack.ql.capabilities.Resolvables; import org.elasticsearch.xpack.ql.expression.Attribute; import org.elasticsearch.xpack.ql.expression.Expressions; +import org.elasticsearch.xpack.ql.expression.Order.OrderDirection; import org.elasticsearch.xpack.ql.plan.logical.LogicalPlan; import org.elasticsearch.xpack.ql.tree.NodeInfo; import org.elasticsearch.xpack.ql.tree.Source; @@ -20,7 +21,7 @@ import java.util.ArrayList; import java.util.List; import java.util.Objects; -import static java.util.Collections.emptyList; +import static java.util.Collections.singletonList; public class Join extends LogicalPlan { @@ -28,17 +29,29 @@ public class Join extends LogicalPlan { private final KeyedFilter until; private final Attribute timestamp; private final Attribute tiebreaker; + private final OrderDirection direction; - public Join(Source source, List queries, KeyedFilter until, Attribute timestamp, Attribute tiebreaker) { + public Join(Source source, + List queries, + KeyedFilter until, + Attribute timestamp, + Attribute tiebreaker, + OrderDirection direction) { super(source, CollectionUtils.combine(queries, until)); this.queries = queries; this.until = until; this.timestamp = timestamp; this.tiebreaker = tiebreaker; + this.direction = direction; } - private Join(Source source, List queries, LogicalPlan until, Attribute timestamp, Attribute tiebreaker) { - this(source, asKeyed(queries), asKeyed(until), timestamp, tiebreaker); + private Join(Source source, + List queries, + LogicalPlan until, + Attribute timestamp, + Attribute tiebreaker, + OrderDirection direction) { + this(source, asKeyed(queries), asKeyed(until), timestamp, tiebreaker, direction); } static List asKeyed(List list) { @@ -59,7 +72,7 @@ public class Join extends LogicalPlan { @Override protected NodeInfo info() { - return NodeInfo.create(this, Join::new, queries, until, timestamp, tiebreaker); + return NodeInfo.create(this, Join::new, queries, until, timestamp, tiebreaker, direction); } @Override @@ -68,7 +81,7 @@ public class Join extends LogicalPlan { throw new EqlIllegalArgumentException("expected at least [2] children but received [{}]", newChildren.size()); } int lastIndex = newChildren.size() - 1; - return new Join(source(), newChildren.subList(0, lastIndex), newChildren.get(lastIndex), timestamp, tiebreaker); + return new Join(source(), newChildren.subList(0, lastIndex), newChildren.get(lastIndex), timestamp, tiebreaker, direction); } @Override @@ -107,9 +120,17 @@ public class Join extends LogicalPlan { return tiebreaker; } + public OrderDirection direction() { + return direction; + } + + public Join with(List queries, KeyedFilter until, OrderDirection direction) { + return new Join(source(), queries, until, timestamp, tiebreaker, direction); + } + @Override public int hashCode() { - return Objects.hash(timestamp, tiebreaker, queries, until); + return Objects.hash(direction, timestamp, tiebreaker, queries, until); } @Override @@ -123,7 +144,7 @@ public class Join extends LogicalPlan { Join other = (Join) obj; - return Objects.equals(queries, other.queries) + return Objects.equals(direction, other.direction) && Objects.equals(queries, other.queries) && Objects.equals(until, other.until) && Objects.equals(timestamp, other.timestamp) && Objects.equals(tiebreaker, other.tiebreaker); @@ -131,6 +152,6 @@ public class Join extends LogicalPlan { @Override public List nodeProperties() { - return emptyList(); + return singletonList(direction); } } \ No newline at end of file diff --git a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/plan/logical/LimitWithOffset.java b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/plan/logical/LimitWithOffset.java new file mode 100644 index 00000000000..a2ef07f52c0 --- /dev/null +++ b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/plan/logical/LimitWithOffset.java @@ -0,0 +1,57 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.eql.plan.logical; + +import org.elasticsearch.xpack.ql.expression.Expression; +import org.elasticsearch.xpack.ql.plan.logical.Limit; +import org.elasticsearch.xpack.ql.plan.logical.LogicalPlan; +import org.elasticsearch.xpack.ql.tree.NodeInfo; +import org.elasticsearch.xpack.ql.tree.Source; + +import java.util.Objects; + +public class LimitWithOffset extends org.elasticsearch.xpack.ql.plan.logical.Limit { + + private final int offset; + + public LimitWithOffset(Source source, Expression limit, LogicalPlan child) { + this(source, limit, 0, child); + } + + public LimitWithOffset(Source source, Expression limit, int offset, LogicalPlan child) { + super(source, limit, child); + this.offset = offset; + } + + @Override + protected NodeInfo info() { + return NodeInfo.create(this, LimitWithOffset::new, limit(), offset, child()); + } + + @Override + protected LimitWithOffset replaceChild(LogicalPlan newChild) { + return new LimitWithOffset(source(), limit(), offset, newChild); + } + + public int offset() { + return offset; + } + + @Override + public int hashCode() { + return Objects.hash(super.hashCode(), offset); + } + + @Override + public boolean equals(Object obj) { + if (super.equals(obj)) { + LimitWithOffset other = (LimitWithOffset) obj; + return Objects.equals(offset, other.offset); + } + return false; + } +} diff --git a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/plan/logical/Sequence.java b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/plan/logical/Sequence.java index fa7006ec876..56838ede9d4 100644 --- a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/plan/logical/Sequence.java +++ b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/plan/logical/Sequence.java @@ -9,6 +9,7 @@ package org.elasticsearch.xpack.eql.plan.logical; import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.xpack.eql.EqlIllegalArgumentException; import org.elasticsearch.xpack.ql.expression.Attribute; +import org.elasticsearch.xpack.ql.expression.Order.OrderDirection; import org.elasticsearch.xpack.ql.plan.logical.LogicalPlan; import org.elasticsearch.xpack.ql.tree.NodeInfo; import org.elasticsearch.xpack.ql.tree.Source; @@ -16,27 +17,37 @@ import org.elasticsearch.xpack.ql.tree.Source; import java.util.List; import java.util.Objects; -import static java.util.Collections.singletonList; +import static java.util.Arrays.asList; public class Sequence extends Join { private final TimeValue maxSpan; - public Sequence(Source source, List queries, KeyedFilter until, TimeValue maxSpan, Attribute timestamp, - Attribute tiebreaker) { - super(source, queries, until, timestamp, tiebreaker); + public Sequence(Source source, + List queries, + KeyedFilter until, + TimeValue maxSpan, + Attribute timestamp, + Attribute tiebreaker, + OrderDirection direction) { + super(source, queries, until, timestamp, tiebreaker, direction); this.maxSpan = maxSpan; } - private Sequence(Source source, List queries, LogicalPlan until, TimeValue maxSpan, Attribute timestamp, - Attribute tiebreaker) { - super(source, asKeyed(queries), asKeyed(until), timestamp, tiebreaker); + private Sequence(Source source, + List queries, + LogicalPlan until, + TimeValue maxSpan, + Attribute timestamp, + Attribute tiebreaker, + OrderDirection direction) { + super(source, asKeyed(queries), asKeyed(until), timestamp, tiebreaker, direction); this.maxSpan = maxSpan; } @Override protected NodeInfo info() { - return NodeInfo.create(this, Sequence::new, queries(), until(), maxSpan, timestamp(), tiebreaker()); + return NodeInfo.create(this, Sequence::new, queries(), until(), maxSpan, timestamp(), tiebreaker(), direction()); } @Override @@ -45,13 +56,19 @@ public class Sequence extends Join { throw new EqlIllegalArgumentException("expected at least [2] children but received [{}]", newChildren.size()); } int lastIndex = newChildren.size() - 1; - return new Sequence(source(), newChildren.subList(0, lastIndex), newChildren.get(lastIndex), maxSpan, timestamp(), tiebreaker()); + return new Sequence(source(), newChildren.subList(0, lastIndex), newChildren.get(lastIndex), maxSpan, timestamp(), tiebreaker(), + direction()); } public TimeValue maxSpan() { return maxSpan; } + @Override + public Join with(List queries, KeyedFilter until, OrderDirection direction) { + return new Sequence(source(), queries, until, maxSpan, timestamp(), tiebreaker(), direction); + } + @Override public int hashCode() { return Objects.hash(maxSpan, super.hashCode()); @@ -68,6 +85,6 @@ public class Sequence extends Join { @Override public List nodeProperties() { - return singletonList(maxSpan); + return asList(maxSpan, direction()); } } \ No newline at end of file diff --git a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/plan/logical/Tail.java b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/plan/logical/Tail.java new file mode 100644 index 00000000000..e14da110afd --- /dev/null +++ b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/plan/logical/Tail.java @@ -0,0 +1,38 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.eql.plan.logical; + +import org.elasticsearch.xpack.ql.expression.Expression; +import org.elasticsearch.xpack.ql.expression.predicate.operator.arithmetic.Neg; +import org.elasticsearch.xpack.ql.plan.logical.Limit; +import org.elasticsearch.xpack.ql.plan.logical.LogicalPlan; +import org.elasticsearch.xpack.ql.tree.NodeInfo; +import org.elasticsearch.xpack.ql.tree.Source; + +public class Tail extends LimitWithOffset { + + public Tail(Source source, Expression limit, LogicalPlan child) { + this(source, child, new Neg(limit.source(), limit)); + } + + /** + * Constructor that does not negate the limit expression. + */ + private Tail(Source source, LogicalPlan child, Expression limit) { + super(source, limit, child); + } + + @Override + protected NodeInfo info() { + return NodeInfo.create(this, Tail::new, child(), limit()); + } + + @Override + protected Tail replaceChild(LogicalPlan newChild) { + return new Tail(source(), newChild, limit()); + } +} \ No newline at end of file diff --git a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/plan/physical/EsQueryExec.java b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/plan/physical/EsQueryExec.java index 9dfe924e6a3..72b6fc2c68d 100644 --- a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/plan/physical/EsQueryExec.java +++ b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/plan/physical/EsQueryExec.java @@ -6,10 +6,17 @@ package org.elasticsearch.xpack.eql.plan.physical; import org.elasticsearch.action.ActionListener; -import org.elasticsearch.xpack.eql.execution.search.Querier; +import org.elasticsearch.search.builder.SearchSourceBuilder; +import org.elasticsearch.search.sort.SortBuilder; +import org.elasticsearch.search.sort.SortOrder; +import org.elasticsearch.xpack.eql.execution.search.BasicQueryClient; +import org.elasticsearch.xpack.eql.execution.search.QueryRequest; +import org.elasticsearch.xpack.eql.execution.search.ReverseListener; +import org.elasticsearch.xpack.eql.execution.search.SourceGenerator; import org.elasticsearch.xpack.eql.querydsl.container.QueryContainer; +import org.elasticsearch.xpack.eql.session.EqlConfiguration; import org.elasticsearch.xpack.eql.session.EqlSession; -import org.elasticsearch.xpack.eql.session.Results; +import org.elasticsearch.xpack.eql.session.Payload; import org.elasticsearch.xpack.ql.expression.Attribute; import org.elasticsearch.xpack.ql.tree.NodeInfo; import org.elasticsearch.xpack.ql.tree.Source; @@ -19,28 +26,22 @@ import java.util.Objects; public class EsQueryExec extends LeafExec { - private final String index; private final List output; private final QueryContainer queryContainer; - public EsQueryExec(Source source, String index, List output, QueryContainer queryContainer) { + public EsQueryExec(Source source, List output, QueryContainer queryContainer) { super(source); - this.index = index; this.output = output; this.queryContainer = queryContainer; } @Override protected NodeInfo info() { - return NodeInfo.create(this, EsQueryExec::new, index, output, queryContainer); + return NodeInfo.create(this, EsQueryExec::new, output, queryContainer); } public EsQueryExec with(QueryContainer queryContainer) { - return new EsQueryExec(source(), index, output, queryContainer); - } - - public String index() { - return index; + return new EsQueryExec(source(), output, queryContainer); } @Override @@ -48,14 +49,33 @@ public class EsQueryExec extends LeafExec { return output; } + public QueryRequest queryRequest(EqlSession session) { + EqlConfiguration cfg = session.configuration(); + SearchSourceBuilder sourceBuilder = SourceGenerator.sourceBuilder(queryContainer, cfg.filter(), cfg.size()); + return () -> sourceBuilder; + } + @Override - public void execute(EqlSession session, ActionListener listener) { - new Querier(session).query(queryContainer, index, listener); + public void execute(EqlSession session, ActionListener listener) { + QueryRequest request = queryRequest(session); + listener = shouldReverse(request) ? new ReverseListener(listener) : listener; + new BasicQueryClient(session).query(request, listener); + } + + private boolean shouldReverse(QueryRequest query) { + SearchSourceBuilder searchSource = query.searchSource(); + // since all results need to be ASC, use this hack to figure out whether the results need to be flipped + for (SortBuilder sort : searchSource.sorts()) { + if (sort.order() == SortOrder.DESC) { + return true; + } + } + return false; } @Override public int hashCode() { - return Objects.hash(index, queryContainer, output); + return Objects.hash(queryContainer, output); } @Override @@ -69,14 +89,13 @@ public class EsQueryExec extends LeafExec { } EsQueryExec other = (EsQueryExec) obj; - return Objects.equals(index, other.index) - && Objects.equals(queryContainer, other.queryContainer) + return Objects.equals(queryContainer, other.queryContainer) && Objects.equals(output, other.output); } @Override public String nodeString() { - return nodeName() + "[" + index + "," + queryContainer + "]"; + return nodeName() + "[" + queryContainer + "]"; } public QueryContainer queryContainer() { diff --git a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/plan/physical/LimitExec.java b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/plan/physical/LimitWithOffsetExec.java similarity index 61% rename from x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/plan/physical/LimitExec.java rename to x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/plan/physical/LimitWithOffsetExec.java index ff166851555..d4e0260bf92 100644 --- a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/plan/physical/LimitExec.java +++ b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/plan/physical/LimitWithOffsetExec.java @@ -5,32 +5,32 @@ */ package org.elasticsearch.xpack.eql.plan.physical; -import org.elasticsearch.xpack.ql.expression.Expression; +import org.elasticsearch.xpack.eql.execution.search.Limit; import org.elasticsearch.xpack.ql.tree.NodeInfo; import org.elasticsearch.xpack.ql.tree.Source; import java.util.Objects; -public class LimitExec extends UnaryExec implements Unexecutable { +public class LimitWithOffsetExec extends UnaryExec implements Unexecutable { - private final Expression limit; + private final Limit limit; - public LimitExec(Source source, PhysicalPlan child, Expression limit) { + public LimitWithOffsetExec(Source source, PhysicalPlan child, Limit limit) { super(source, child); this.limit = limit; } @Override - protected NodeInfo info() { - return NodeInfo.create(this, LimitExec::new, child(), limit); + protected NodeInfo info() { + return NodeInfo.create(this, LimitWithOffsetExec::new, child(), limit); } @Override - protected LimitExec replaceChild(PhysicalPlan newChild) { - return new LimitExec(source(), newChild, limit); + protected LimitWithOffsetExec replaceChild(PhysicalPlan newChild) { + return new LimitWithOffsetExec(source(), newChild, limit); } - public Expression limit() { + public Limit limit() { return limit; } @@ -49,7 +49,7 @@ public class LimitExec extends UnaryExec implements Unexecutable { return false; } - LimitExec other = (LimitExec) obj; + LimitWithOffsetExec other = (LimitWithOffsetExec) obj; return Objects.equals(limit, other.limit) && Objects.equals(child(), other.child()); } diff --git a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/plan/physical/LocalExec.java b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/plan/physical/LocalExec.java index 48bd65d1a1c..bc0e71d3a44 100644 --- a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/plan/physical/LocalExec.java +++ b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/plan/physical/LocalExec.java @@ -9,7 +9,7 @@ import org.elasticsearch.action.ActionListener; import org.elasticsearch.xpack.eql.session.EmptyExecutable; import org.elasticsearch.xpack.eql.session.EqlSession; import org.elasticsearch.xpack.eql.session.Executable; -import org.elasticsearch.xpack.eql.session.Results; +import org.elasticsearch.xpack.eql.session.Payload; import org.elasticsearch.xpack.ql.expression.Attribute; import org.elasticsearch.xpack.ql.tree.NodeInfo; import org.elasticsearch.xpack.ql.tree.Source; @@ -45,7 +45,7 @@ public class LocalExec extends LeafExec { } @Override - public void execute(EqlSession session, ActionListener listener) { + public void execute(EqlSession session, ActionListener listener) { executable.execute(session, listener); } @@ -67,4 +67,4 @@ public class LocalExec extends LeafExec { LocalExec other = (LocalExec) obj; return Objects.equals(executable, other.executable); } -} +} \ No newline at end of file diff --git a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/plan/physical/LocalRelation.java b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/plan/physical/LocalRelation.java index a1477eeec62..3ef33912c33 100644 --- a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/plan/physical/LocalRelation.java +++ b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/plan/physical/LocalRelation.java @@ -9,10 +9,12 @@ import org.elasticsearch.action.ActionListener; import org.elasticsearch.xpack.eql.session.EmptyExecutable; import org.elasticsearch.xpack.eql.session.EqlSession; import org.elasticsearch.xpack.eql.session.Executable; +import org.elasticsearch.xpack.eql.session.Payload; import org.elasticsearch.xpack.eql.session.Results; import org.elasticsearch.xpack.ql.expression.Attribute; import org.elasticsearch.xpack.ql.plan.logical.LogicalPlan; import org.elasticsearch.xpack.ql.tree.NodeInfo; +import org.elasticsearch.xpack.ql.tree.NodeUtils; import org.elasticsearch.xpack.ql.tree.Source; import java.util.List; @@ -62,7 +64,7 @@ public class LocalRelation extends LogicalPlan implements Executable { } @Override - public void execute(EqlSession session, ActionListener listener) { + public void execute(EqlSession session, ActionListener listener) { executable.execute(session, listener); } @@ -84,4 +86,10 @@ public class LocalRelation extends LogicalPlan implements Executable { LocalRelation other = (LocalRelation) obj; return Objects.equals(executable, other.executable); } + + + @Override + public String nodeString() { + return nodeName() + NodeUtils.limitedToString(output()); + } } diff --git a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/plan/physical/SequenceExec.java b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/plan/physical/SequenceExec.java index 1621234ee89..8c3dc31193f 100644 --- a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/plan/physical/SequenceExec.java +++ b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/plan/physical/SequenceExec.java @@ -9,11 +9,13 @@ package org.elasticsearch.xpack.eql.plan.physical; import org.elasticsearch.action.ActionListener; import org.elasticsearch.xpack.eql.EqlIllegalArgumentException; import org.elasticsearch.xpack.eql.execution.assembler.ExecutionManager; +import org.elasticsearch.xpack.eql.execution.search.Limit; import org.elasticsearch.xpack.eql.session.EqlSession; -import org.elasticsearch.xpack.eql.session.Results; +import org.elasticsearch.xpack.eql.session.Payload; import org.elasticsearch.xpack.ql.expression.Attribute; import org.elasticsearch.xpack.ql.expression.Expressions; import org.elasticsearch.xpack.ql.expression.NamedExpression; +import org.elasticsearch.xpack.ql.expression.Order.OrderDirection; import org.elasticsearch.xpack.ql.tree.NodeInfo; import org.elasticsearch.xpack.ql.tree.Source; @@ -29,6 +31,8 @@ public class SequenceExec extends PhysicalPlan { private final List> keys; private final Attribute timestamp; private final Attribute tiebreaker; + private final Limit limit; + private final OrderDirection direction; public SequenceExec(Source source, List> keys, @@ -36,20 +40,29 @@ public class SequenceExec extends PhysicalPlan { List untilKeys, PhysicalPlan until, Attribute timestamp, - Attribute tiebreaker) { - this(source, combine(matches, until), combine(keys, singletonList(untilKeys)), timestamp, tiebreaker); + Attribute tiebreaker, + OrderDirection direction) { + this(source, combine(matches, until), combine(keys, singletonList(untilKeys)), timestamp, tiebreaker, null, direction); } - private SequenceExec(Source source, List children, List> keys, Attribute ts, Attribute tb) { + private SequenceExec(Source source, + List children, + List> keys, + Attribute ts, + Attribute tb, + Limit limit, + OrderDirection direction) { super(source, children); this.keys = keys; this.timestamp = ts; this.tiebreaker = tb; + this.limit = limit; + this.direction = direction; } @Override protected NodeInfo info() { - return NodeInfo.create(this, SequenceExec::new, children(), keys, timestamp, tiebreaker); + return NodeInfo.create(this, SequenceExec::new, children(), keys, timestamp, tiebreaker, limit, direction); } @Override @@ -59,7 +72,7 @@ public class SequenceExec extends PhysicalPlan { children().size(), newChildren.size()); } - return new SequenceExec(source(), newChildren, keys, timestamp, tiebreaker); + return new SequenceExec(source(), newChildren, keys, timestamp, tiebreaker, limit, direction); } @Override @@ -87,14 +100,26 @@ public class SequenceExec extends PhysicalPlan { return tiebreaker; } + public Limit limit() { + return limit; + } + + public OrderDirection direction() { + return direction; + } + + public SequenceExec with(Limit limit) { + return new SequenceExec(source(), children(), keys(), timestamp(), tiebreaker(), limit, direction); + } + @Override - public void execute(EqlSession session, ActionListener listener) { - new ExecutionManager(session).assemble(keys(), children(), timestamp(), tiebreaker()).execute(listener); + public void execute(EqlSession session, ActionListener listener) { + new ExecutionManager(session).assemble(keys(), children(), timestamp(), tiebreaker(), direction, limit()).execute(listener); } @Override public int hashCode() { - return Objects.hash(timestamp, tiebreaker, keys, children()); + return Objects.hash(timestamp, tiebreaker, keys, limit, direction, children()); } @Override @@ -110,6 +135,8 @@ public class SequenceExec extends PhysicalPlan { SequenceExec other = (SequenceExec) obj; return Objects.equals(timestamp, other.timestamp) && Objects.equals(tiebreaker, other.tiebreaker) + && Objects.equals(limit, other.limit) + && Objects.equals(direction, other.direction) && Objects.equals(children(), other.children()) && Objects.equals(keys, other.keys); } diff --git a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/plan/physical/Unexecutable.java b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/plan/physical/Unexecutable.java index 69a6ca51271..54978298b72 100644 --- a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/plan/physical/Unexecutable.java +++ b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/plan/physical/Unexecutable.java @@ -9,14 +9,14 @@ import org.elasticsearch.action.ActionListener; import org.elasticsearch.xpack.eql.planner.PlanningException; import org.elasticsearch.xpack.eql.session.EqlSession; import org.elasticsearch.xpack.eql.session.Executable; -import org.elasticsearch.xpack.eql.session.Results; +import org.elasticsearch.xpack.eql.session.Payload; // this is mainly a marker interface to validate a plan before being executed public interface Unexecutable extends Executable { @Override - default void execute(EqlSession session, ActionListener listener) { + default void execute(EqlSession session, ActionListener listener) { throw new PlanningException("Current plan {} is not executable", this); } } diff --git a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/plan/physical/UnplannedExec.java b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/plan/physical/UnplannedExec.java index 45061b3f961..df57d156636 100644 --- a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/plan/physical/UnplannedExec.java +++ b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/plan/physical/UnplannedExec.java @@ -8,7 +8,7 @@ package org.elasticsearch.xpack.eql.plan.physical; import org.elasticsearch.action.ActionListener; import org.elasticsearch.xpack.eql.planner.PlanningException; import org.elasticsearch.xpack.eql.session.EqlSession; -import org.elasticsearch.xpack.eql.session.Results; +import org.elasticsearch.xpack.eql.session.Payload; import org.elasticsearch.xpack.ql.expression.Attribute; import org.elasticsearch.xpack.ql.plan.logical.LogicalPlan; import org.elasticsearch.xpack.ql.tree.NodeInfo; @@ -41,7 +41,7 @@ public class UnplannedExec extends LeafExec implements Unexecutable { } @Override - public void execute(EqlSession session, ActionListener listener) { + public void execute(EqlSession session, ActionListener listener) { throw new PlanningException("Current plan {} is not executable", this); } diff --git a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/planner/Mapper.java b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/planner/Mapper.java index 4f48f31cfe6..1b236bebe30 100644 --- a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/planner/Mapper.java +++ b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/planner/Mapper.java @@ -6,11 +6,13 @@ package org.elasticsearch.xpack.eql.planner; +import org.elasticsearch.xpack.eql.execution.search.Limit; import org.elasticsearch.xpack.eql.plan.logical.KeyedFilter; +import org.elasticsearch.xpack.eql.plan.logical.LimitWithOffset; import org.elasticsearch.xpack.eql.plan.logical.Sequence; import org.elasticsearch.xpack.eql.plan.physical.EsQueryExec; import org.elasticsearch.xpack.eql.plan.physical.FilterExec; -import org.elasticsearch.xpack.eql.plan.physical.LimitExec; +import org.elasticsearch.xpack.eql.plan.physical.LimitWithOffsetExec; import org.elasticsearch.xpack.eql.plan.physical.LocalExec; import org.elasticsearch.xpack.eql.plan.physical.LocalRelation; import org.elasticsearch.xpack.eql.plan.physical.OrderExec; @@ -21,14 +23,16 @@ import org.elasticsearch.xpack.eql.plan.physical.UnplannedExec; import org.elasticsearch.xpack.eql.querydsl.container.QueryContainer; import org.elasticsearch.xpack.ql.expression.Attribute; import org.elasticsearch.xpack.ql.expression.Expressions; +import org.elasticsearch.xpack.ql.expression.Foldables; import org.elasticsearch.xpack.ql.plan.logical.EsRelation; import org.elasticsearch.xpack.ql.plan.logical.Filter; -import org.elasticsearch.xpack.ql.plan.logical.Limit; import org.elasticsearch.xpack.ql.plan.logical.LogicalPlan; import org.elasticsearch.xpack.ql.plan.logical.OrderBy; import org.elasticsearch.xpack.ql.plan.logical.Project; import org.elasticsearch.xpack.ql.rule.Rule; import org.elasticsearch.xpack.ql.rule.RuleExecutor; +import org.elasticsearch.xpack.ql.type.DataTypeConverter; +import org.elasticsearch.xpack.ql.type.DataTypes; import org.elasticsearch.xpack.ql.util.ReflectionUtils; import java.util.ArrayList; @@ -73,7 +77,8 @@ class Mapper extends RuleExecutor { Expressions.asAttributes(s.until().keys()), map(s.until().child()), s.timestamp(), - s.tiebreaker()); + s.tiebreaker(), + s.direction()); } if (p instanceof LocalRelation) { @@ -95,9 +100,10 @@ class Mapper extends RuleExecutor { return new OrderExec(p.source(), map(o.child()), o.order()); } - if (p instanceof Limit) { - Limit l = (Limit) p; - return new LimitExec(p.source(), map(l.child()), l.limit()); + if (p instanceof LimitWithOffset) { + LimitWithOffset l = (LimitWithOffset) p; + int limit = (Integer) DataTypeConverter.convert(Foldables.valueOf(l.limit()), DataTypes.INTEGER); + return new LimitWithOffsetExec(p.source(), map(l.child()), new Limit(limit, l.offset())); } if (p instanceof EsRelation) { @@ -107,7 +113,7 @@ class Mapper extends RuleExecutor { if (c.frozen()) { container = container.withFrozen(); } - return new EsQueryExec(p.source(), c.index().name(), output, container); + return new EsQueryExec(p.source(), output, container); } return planLater(p); diff --git a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/planner/QueryFolder.java b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/planner/QueryFolder.java index c8766b5f0a9..eee4cf889bb 100644 --- a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/planner/QueryFolder.java +++ b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/planner/QueryFolder.java @@ -9,9 +9,12 @@ package org.elasticsearch.xpack.eql.planner; import org.elasticsearch.xpack.eql.EqlIllegalArgumentException; import org.elasticsearch.xpack.eql.plan.physical.EsQueryExec; import org.elasticsearch.xpack.eql.plan.physical.FilterExec; +import org.elasticsearch.xpack.eql.plan.physical.LimitWithOffsetExec; import org.elasticsearch.xpack.eql.plan.physical.OrderExec; import org.elasticsearch.xpack.eql.plan.physical.PhysicalPlan; import org.elasticsearch.xpack.eql.plan.physical.ProjectExec; +import org.elasticsearch.xpack.eql.plan.physical.SequenceExec; +import org.elasticsearch.xpack.eql.plan.physical.UnaryExec; import org.elasticsearch.xpack.eql.querydsl.container.QueryContainer; import org.elasticsearch.xpack.ql.expression.Attribute; import org.elasticsearch.xpack.ql.expression.Expression; @@ -39,7 +42,8 @@ class QueryFolder extends RuleExecutor { Batch fold = new Batch("Fold queries", new FoldProject(), new FoldFilter(), - new FoldOrderBy() + new FoldOrderBy(), + new FoldLimit() ); Batch finish = new Batch("Finish query", Limiter.ONCE, new PlanOutputToQueryRef() @@ -49,67 +53,73 @@ class QueryFolder extends RuleExecutor { } - private static class FoldProject extends FoldingRule { + private static class FoldProject extends QueryFoldingRule { @Override - protected PhysicalPlan rule(ProjectExec project) { - if (project.child() instanceof EsQueryExec) { - EsQueryExec exec = (EsQueryExec) project.child(); - return new EsQueryExec(exec.source(), exec.index(), project.output(), exec.queryContainer()); - } - return project; + protected PhysicalPlan rule(ProjectExec project, EsQueryExec exec) { + return new EsQueryExec(exec.source(), project.output(), exec.queryContainer()); } } - private static class FoldFilter extends FoldingRule { + private static class FoldFilter extends QueryFoldingRule { @Override - protected PhysicalPlan rule(FilterExec plan) { - if (plan.child() instanceof EsQueryExec) { - EsQueryExec exec = (EsQueryExec) plan.child(); - QueryContainer qContainer = exec.queryContainer(); + protected PhysicalPlan rule(FilterExec plan, EsQueryExec exec) { + QueryContainer qContainer = exec.queryContainer(); + Query query = QueryTranslator.toQuery(plan.condition()); - Query query = QueryTranslator.toQuery(plan.condition()); - - if (qContainer.query() != null || query != null) { - query = ExpressionTranslators.and(plan.source(), qContainer.query(), query); - } - - qContainer = qContainer.with(query); - return exec.with(qContainer); + if (qContainer.query() != null || query != null) { + query = ExpressionTranslators.and(plan.source(), qContainer.query(), query); } - return plan; + + qContainer = qContainer.with(query); + return exec.with(qContainer); } } - private static class FoldOrderBy extends FoldingRule { + private static class FoldOrderBy extends QueryFoldingRule { + @Override - protected PhysicalPlan rule(OrderExec plan) { - if (plan.child() instanceof EsQueryExec) { - EsQueryExec exec = (EsQueryExec) plan.child(); - QueryContainer qContainer = exec.queryContainer(); + protected PhysicalPlan rule(OrderExec plan, EsQueryExec query) { + QueryContainer qContainer = query.queryContainer(); - for (Order order : plan.order()) { - Direction direction = Direction.from(order.direction()); - Missing missing = Missing.from(order.nullsPosition()); + for (Order order : plan.order()) { + Direction direction = Direction.from(order.direction()); + Missing missing = Missing.from(order.nullsPosition()); - // check whether sorting is on an group (and thus nested agg) or field - Expression orderExpression = order.child(); + // check whether sorting is on an group (and thus nested agg) or field + Expression orderExpression = order.child(); - String lookup = Expressions.id(orderExpression); + String lookup = Expressions.id(orderExpression); - // field - if (orderExpression instanceof FieldAttribute) { - FieldAttribute fa = (FieldAttribute) orderExpression; - qContainer = qContainer.addSort(lookup, new AttributeSort(fa, direction, missing)); - } - // unknown - else { - throw new EqlIllegalArgumentException("unsupported sorting expression {}", orderExpression); - } + // field + if (orderExpression instanceof FieldAttribute) { + FieldAttribute fa = (FieldAttribute) orderExpression; + qContainer = qContainer.addSort(lookup, new AttributeSort(fa, direction, missing)); } + // unknown + else { + throw new EqlIllegalArgumentException("unsupported sorting expression {}", orderExpression); + } + } - return exec.with(qContainer); + return query.with(qContainer); + } + } + + private static class FoldLimit extends FoldingRule { + + @Override + protected PhysicalPlan rule(LimitWithOffsetExec limit) { + PhysicalPlan plan = limit; + PhysicalPlan child = limit.child(); + if (child instanceof EsQueryExec) { + EsQueryExec query = (EsQueryExec) child; + plan = query.with(query.queryContainer().with(limit.limit())); + } + if (child instanceof SequenceExec) { + SequenceExec exec = (SequenceExec) child; + plan = exec.with(limit.limit()); } return plan; } @@ -139,4 +149,18 @@ class QueryFolder extends RuleExecutor { @Override protected abstract PhysicalPlan rule(SubPlan plan); } -} + + abstract static class QueryFoldingRule extends FoldingRule { + + @Override + protected final PhysicalPlan rule(SubPlan plan) { + PhysicalPlan p = plan; + if (plan.child() instanceof EsQueryExec) { + p = rule(plan, (EsQueryExec) plan.child()); + } + return p; + } + + protected abstract PhysicalPlan rule(SubPlan plan, EsQueryExec query); + } +} \ No newline at end of file diff --git a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/querydsl/container/QueryContainer.java b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/querydsl/container/QueryContainer.java index 06b81121968..3228967def4 100644 --- a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/querydsl/container/QueryContainer.java +++ b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/querydsl/container/QueryContainer.java @@ -11,6 +11,7 @@ import org.elasticsearch.common.xcontent.ToXContent; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.json.JsonXContent; import org.elasticsearch.xpack.eql.EqlIllegalArgumentException; +import org.elasticsearch.xpack.eql.execution.search.Limit; import org.elasticsearch.xpack.eql.execution.search.SourceGenerator; import org.elasticsearch.xpack.ql.execution.search.FieldExtraction; import org.elasticsearch.xpack.ql.expression.Attribute; @@ -45,18 +46,27 @@ public class QueryContainer { private final boolean trackHits; private final boolean includeFrozen; + private final Limit limit; + public QueryContainer() { - this(null, emptyList(), AttributeMap.emptyAttributeMap(), emptyMap(), false, false); + this(null, emptyList(), AttributeMap.emptyAttributeMap(), emptyMap(), false, false, null); } - private QueryContainer(Query query, List> fields, AttributeMap attributes, - Map sort, boolean trackHits, boolean includeFrozen) { + private QueryContainer(Query query, + List> fields, + AttributeMap attributes, + Map sort, + boolean trackHits, + boolean includeFrozen, + Limit limit) { this.query = query; this.fields = fields; this.sort = sort; this.attributes = attributes; this.trackHits = trackHits; this.includeFrozen = includeFrozen; + + this.limit = limit; } public QueryContainer withFrozen() { @@ -79,8 +89,16 @@ public class QueryContainer { return trackHits; } + public Limit limit() { + return limit; + } + public QueryContainer with(Query q) { - return new QueryContainer(q, fields, attributes, sort, trackHits, includeFrozen); + return new QueryContainer(q, fields, attributes, sort, trackHits, includeFrozen, limit); + } + + public QueryContainer with(Limit limit) { + return new QueryContainer(query, fields, attributes, sort, trackHits, includeFrozen, limit); } public QueryContainer addColumn(Attribute attr) { @@ -111,7 +129,7 @@ public class QueryContainer { public QueryContainer addSort(String expressionId, Sort sortable) { Map newSort = new LinkedHashMap<>(this.sort); newSort.put(expressionId, sortable); - return new QueryContainer(query, fields, attributes, newSort, trackHits, includeFrozen); + return new QueryContainer(query, fields, attributes, newSort, trackHits, includeFrozen, limit); } // @@ -119,12 +137,12 @@ public class QueryContainer { // public QueryContainer addColumn(FieldExtraction ref, String id) { - return new QueryContainer(query, combine(fields, new Tuple<>(ref, id)), attributes, sort, trackHits, includeFrozen); + return new QueryContainer(query, combine(fields, new Tuple<>(ref, id)), attributes, sort, trackHits, includeFrozen, limit); } @Override public int hashCode() { - return Objects.hash(query, attributes, fields, trackHits, includeFrozen); + return Objects.hash(query, attributes, fields, trackHits, includeFrozen, limit); } @Override @@ -141,8 +159,9 @@ public class QueryContainer { return Objects.equals(query, other.query) && Objects.equals(attributes, other.attributes) && Objects.equals(fields, other.fields) - && Objects.equals(trackHits, other.trackHits) - && Objects.equals(includeFrozen, other.includeFrozen); + && trackHits == other.trackHits + && includeFrozen == other.includeFrozen + && Objects.equals(limit, other.limit); } @Override diff --git a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/session/EmptyExecutable.java b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/session/EmptyExecutable.java index 03eee0661fc..056061925ee 100644 --- a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/session/EmptyExecutable.java +++ b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/session/EmptyExecutable.java @@ -5,23 +5,19 @@ */ package org.elasticsearch.xpack.eql.session; -import org.apache.lucene.search.TotalHits; -import org.apache.lucene.search.TotalHits.Relation; import org.elasticsearch.action.ActionListener; -import org.elasticsearch.common.unit.TimeValue; +import org.elasticsearch.xpack.eql.session.Results.Type; import org.elasticsearch.xpack.ql.expression.Attribute; import java.util.List; import java.util.Objects; -import static java.util.Collections.emptyList; - public class EmptyExecutable implements Executable { private final List output; - private final Results.Type resultType; + private final Type resultType; - public EmptyExecutable(List output, Results.Type resultType) { + public EmptyExecutable(List output, Type resultType) { this.output = output; this.resultType = resultType; } @@ -32,8 +28,8 @@ public class EmptyExecutable implements Executable { } @Override - public void execute(EqlSession session, ActionListener listener) { - listener.onResponse(new Results(new TotalHits(0, Relation.EQUAL_TO), TimeValue.ZERO, false, emptyList(), resultType)); + public void execute(EqlSession session, ActionListener listener) { + listener.onResponse(new EmptyPayload(resultType)); } @Override diff --git a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/session/EmptyPayload.java b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/session/EmptyPayload.java new file mode 100644 index 00000000000..3f4238accc7 --- /dev/null +++ b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/session/EmptyPayload.java @@ -0,0 +1,48 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.eql.session; + +import org.elasticsearch.common.unit.TimeValue; +import org.elasticsearch.xpack.eql.session.Results.Type; + +import java.util.List; + +import static java.util.Collections.emptyList; + +public class EmptyPayload implements Payload { + + private final Type type; + + public EmptyPayload(Type type) { + this.type = type; + } + + @Override + public Type resultType() { + return type; + } + + @Override + public boolean timedOut() { + return false; + } + + @Override + public TimeValue timeTook() { + return TimeValue.ZERO; + } + + @Override + public Object[] nextKeys() { + return null; + } + + @Override + public List values() { + return emptyList(); + } +} diff --git a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/session/EqlSession.java b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/session/EqlSession.java index b60d9968d33..52ba43bd5b3 100644 --- a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/session/EqlSession.java +++ b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/session/EqlSession.java @@ -61,7 +61,8 @@ public class EqlSession { } public void eql(String eql, ParserParams params, ActionListener listener) { - eqlExecutable(eql, params, wrap(e -> e.execute(this, listener), listener::onFailure)); + eqlExecutable(eql, params, wrap(e -> e.execute(this, wrap(p -> listener.onResponse(Results.fromPayload(p)), listener::onFailure)), + listener::onFailure)); } public void eqlExecutable(String eql, ParserParams params, ActionListener listener) { diff --git a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/session/Executable.java b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/session/Executable.java index 71dc188e492..eeeed3f51fb 100644 --- a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/session/Executable.java +++ b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/session/Executable.java @@ -15,5 +15,5 @@ public interface Executable { List output(); - void execute(EqlSession session, ActionListener listener); + void execute(EqlSession session, ActionListener listener); } diff --git a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/execution/payload/Payload.java b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/session/Payload.java similarity index 58% rename from x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/execution/payload/Payload.java rename to x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/session/Payload.java index 9a5fa3d9302..2791e582974 100644 --- a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/execution/payload/Payload.java +++ b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/session/Payload.java @@ -4,13 +4,19 @@ * you may not use this file except in compliance with the Elastic License. */ -package org.elasticsearch.xpack.eql.execution.payload; +package org.elasticsearch.xpack.eql.session; import org.elasticsearch.common.unit.TimeValue; import java.util.List; -public interface Payload { +/** + * Container for internal results. Can be low-level such as SearchHits or Sequences. + * Generalized to allow reuse and internal pluggability. + */ +public interface Payload { + + Results.Type resultType(); boolean timedOut(); @@ -18,5 +24,5 @@ public interface Payload { Object[] nextKeys(); - List values(); + List values(); } diff --git a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/session/Results.java b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/session/Results.java index 65bb6754603..cd49b2749a3 100644 --- a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/session/Results.java +++ b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/session/Results.java @@ -29,18 +29,11 @@ public class Results { private final TimeValue tookTime; private final Type type; - public static Results fromHits(TimeValue tookTime, List hits) { - return new Results(new TotalHits(hits.size(), Relation.EQUAL_TO), tookTime, false, hits, Type.SEARCH_HIT); + public static Results fromPayload(Payload payload) { + List values = payload.values(); + return new Results(new TotalHits(values.size(), Relation.EQUAL_TO), payload.timeTook(), false, values, payload.resultType()); } - - public static Results fromSequences(TimeValue tookTime, List sequences) { - return new Results(new TotalHits(sequences.size(), Relation.EQUAL_TO), tookTime, false, sequences, Type.SEQUENCE); - } - - public static Results fromCounts(TimeValue tookTime, List counts) { - return new Results(new TotalHits(counts.size(), Relation.EQUAL_TO), tookTime, false, counts, Type.COUNT); - } - + Results(TotalHits totalHits, TimeValue tookTime, boolean timedOut, List results, Type type) { this.totalHits = totalHits; this.tookTime = tookTime; diff --git a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/util/MathUtils.java b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/util/MathUtils.java new file mode 100644 index 00000000000..80a45c4a1de --- /dev/null +++ b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/util/MathUtils.java @@ -0,0 +1,21 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.eql.util; + +import org.elasticsearch.xpack.eql.EqlIllegalArgumentException; + +public class MathUtils { + + public static int abs(int number) { + + if (number == Integer.MIN_VALUE) { + throw new EqlIllegalArgumentException("[" + number + "] cannot be negated since the result is outside the range"); + } + + return number < 0 ? -number : number; + } +} diff --git a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/util/ReversedIterator.java b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/util/ReversedIterator.java new file mode 100644 index 00000000000..694fd74bf8f --- /dev/null +++ b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/util/ReversedIterator.java @@ -0,0 +1,35 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.eql.util; + +import java.util.Iterator; +import java.util.List; +import java.util.ListIterator; + +public class ReversedIterator implements Iterator { + + private final ListIterator delegate; + + public ReversedIterator(List delegate) { + this.delegate = delegate.listIterator(delegate.size()); + } + + @Override + public boolean hasNext() { + return delegate.hasPrevious(); + } + + @Override + public T next() { + return delegate.previous(); + } + + @Override + public void remove() { + delegate.remove(); + } +} diff --git a/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/analysis/VerifierTests.java b/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/analysis/VerifierTests.java index 1b7c5021e66..4f769acfb3b 100644 --- a/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/analysis/VerifierTests.java +++ b/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/analysis/VerifierTests.java @@ -87,10 +87,6 @@ public class VerifierTests extends ESTestCase { assertEquals("1:11: Unknown column [pib], did you mean any of [pid, ppid]?", error("foo where pib == 1")); } - public void testPipesUnsupported() { - assertEquals("1:20: Pipes are not supported", errorParsing("process where true | head 6")); - } - public void testProcessRelationshipsUnsupported() { assertEquals("2:7: Process relationships are not supported", errorParsing("process where opcode=1 and process_name == \"csrss.exe\"\n" + diff --git a/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/execution/assembler/SequenceRuntimeTests.java b/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/execution/assembler/SequenceRuntimeTests.java index 9206a764e67..7a78a899fa9 100644 --- a/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/execution/assembler/SequenceRuntimeTests.java +++ b/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/execution/assembler/SequenceRuntimeTests.java @@ -17,8 +17,9 @@ import org.elasticsearch.search.builder.SearchSourceBuilder; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.xpack.eql.action.EqlSearchResponse.Sequence; import org.elasticsearch.xpack.eql.execution.assembler.SeriesUtils.SeriesSpec; -import org.elasticsearch.xpack.eql.execution.payload.Payload; +import org.elasticsearch.xpack.eql.session.Payload; import org.elasticsearch.xpack.eql.session.Results; +import org.elasticsearch.xpack.eql.session.Results.Type; import org.elasticsearch.xpack.ql.execution.search.extractor.HitExtractor; import java.io.IOException; @@ -112,7 +113,7 @@ public class SequenceRuntimeTests extends ESTestCase { } } - static class TestPayload implements Payload { + static class TestPayload implements Payload { private final List hits; private final Map> events; @@ -128,6 +129,11 @@ public class SequenceRuntimeTests extends ESTestCase { } } + @Override + public Type resultType() { + return Type.SEARCH_HIT; + } + @Override public boolean timedOut() { return false; @@ -143,9 +149,10 @@ public class SequenceRuntimeTests extends ESTestCase { return new Object[0]; } + @SuppressWarnings("unchecked") @Override - public List values() { - return hits; + public List values() { + return (List) hits; } @Override @@ -181,10 +188,10 @@ public class SequenceRuntimeTests extends ESTestCase { } // convert the results through a test specific payload - SequenceRuntime runtime = new SequenceRuntime(criteria, (c, l) -> { - Map> evs = events.get(c.size()); + SequenceRuntime runtime = new SequenceRuntime(criteria, (r, l) -> { + Map> evs = events.get(r.searchSource().size()); l.onResponse(new TestPayload(evs)); - }); + }, false, null); // finally make the assertion at the end of the listener runtime.execute(wrap(this::checkResults, ex -> { @@ -192,8 +199,8 @@ public class SequenceRuntimeTests extends ESTestCase { })); } - private void checkResults(Results results) { - List seq = results.sequences(); + private void checkResults(Payload payload) { + List seq = Results.fromPayload(payload).sequences(); String prefix = "Line " + lineNumber + ":"; assertNotNull(prefix + "no matches found", seq); assertEquals(prefix + "different sequences matched ", matches.size(), seq.size()); diff --git a/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/optimizer/OptimizerTests.java b/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/optimizer/OptimizerTests.java index e4166fdbd9c..3bb3f0fcac8 100644 --- a/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/optimizer/OptimizerTests.java +++ b/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/optimizer/OptimizerTests.java @@ -6,13 +6,25 @@ package org.elasticsearch.xpack.eql.optimizer; +import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.xpack.eql.analysis.Analyzer; import org.elasticsearch.xpack.eql.analysis.PreAnalyzer; import org.elasticsearch.xpack.eql.analysis.Verifier; import org.elasticsearch.xpack.eql.expression.function.EqlFunctionRegistry; import org.elasticsearch.xpack.eql.parser.EqlParser; +import org.elasticsearch.xpack.eql.plan.logical.KeyedFilter; +import org.elasticsearch.xpack.eql.plan.logical.LimitWithOffset; +import org.elasticsearch.xpack.eql.plan.logical.Sequence; +import org.elasticsearch.xpack.eql.plan.logical.Tail; +import org.elasticsearch.xpack.eql.plan.physical.LocalRelation; +import org.elasticsearch.xpack.ql.expression.Attribute; +import org.elasticsearch.xpack.ql.expression.EmptyAttribute; import org.elasticsearch.xpack.ql.expression.FieldAttribute; +import org.elasticsearch.xpack.ql.expression.Literal; +import org.elasticsearch.xpack.ql.expression.Order; +import org.elasticsearch.xpack.ql.expression.Order.NullsPosition; +import org.elasticsearch.xpack.ql.expression.Order.OrderDirection; import org.elasticsearch.xpack.ql.expression.predicate.logical.And; import org.elasticsearch.xpack.ql.expression.predicate.logical.Not; import org.elasticsearch.xpack.ql.expression.predicate.nulls.IsNotNull; @@ -20,10 +32,14 @@ import org.elasticsearch.xpack.ql.expression.predicate.nulls.IsNull; import org.elasticsearch.xpack.ql.expression.predicate.regex.Like; import org.elasticsearch.xpack.ql.index.EsIndex; import org.elasticsearch.xpack.ql.index.IndexResolution; +import org.elasticsearch.xpack.ql.plan.TableIdentifier; import org.elasticsearch.xpack.ql.plan.logical.Filter; import org.elasticsearch.xpack.ql.plan.logical.LogicalPlan; import org.elasticsearch.xpack.ql.plan.logical.OrderBy; import org.elasticsearch.xpack.ql.plan.logical.Project; +import org.elasticsearch.xpack.ql.plan.logical.UnaryPlan; +import org.elasticsearch.xpack.ql.plan.logical.UnresolvedRelation; +import org.elasticsearch.xpack.ql.type.DataTypes; import org.elasticsearch.xpack.ql.type.EsField; import org.elasticsearch.xpack.ql.type.TypesTests; @@ -31,7 +47,12 @@ import java.util.Arrays; import java.util.List; import java.util.Map; +import static java.util.Arrays.asList; +import static java.util.Collections.emptyList; +import static java.util.Collections.emptyMap; +import static java.util.Collections.singletonList; import static org.elasticsearch.xpack.eql.EqlTestUtils.TEST_CFG; +import static org.elasticsearch.xpack.ql.tree.Source.EMPTY; public class OptimizerTests extends ESTestCase { @@ -39,6 +60,7 @@ public class OptimizerTests extends ESTestCase { private static final String INDEX_NAME = "test"; private EqlParser parser = new EqlParser(); private IndexResolution index = loadIndexResolution("mapping-default.json"); + private Optimizer optimizer = new Optimizer(); private static Map loadEqlMapping(String name) { return TypesTests.loadMapping(name); @@ -51,7 +73,6 @@ public class OptimizerTests extends ESTestCase { private LogicalPlan accept(IndexResolution resolution, String eql) { PreAnalyzer preAnalyzer = new PreAnalyzer(); Analyzer analyzer = new Analyzer(TEST_CFG, new EqlFunctionRegistry(), new Verifier()); - Optimizer optimizer = new Optimizer(); return optimizer.optimize(analyzer.analyze(preAnalyzer.preAnalyze(parser.createStatement(eql), resolution))); } @@ -67,10 +88,11 @@ public class OptimizerTests extends ESTestCase { for (String q : tests) { LogicalPlan plan = accept(q); - assertTrue(plan instanceof Project); - plan = ((Project) plan).child(); assertTrue(plan instanceof OrderBy); plan = ((OrderBy) plan).child(); + assertTrue(plan instanceof Project); + plan = ((Project) plan).child(); + assertTrue(plan instanceof Filter); Filter filter = (Filter) plan; @@ -89,10 +111,10 @@ public class OptimizerTests extends ESTestCase { for (String q : tests) { LogicalPlan plan = accept(q); - assertTrue(plan instanceof Project); - plan = ((Project) plan).child(); assertTrue(plan instanceof OrderBy); plan = ((OrderBy) plan).child(); + assertTrue(plan instanceof Project); + plan = ((Project) plan).child(); assertTrue(plan instanceof Filter); Filter filter = (Filter) plan; @@ -112,10 +134,10 @@ public class OptimizerTests extends ESTestCase { for (String q : tests) { LogicalPlan plan = accept(q); - assertTrue(plan instanceof Project); - plan = ((Project) plan).child(); assertTrue(plan instanceof OrderBy); plan = ((OrderBy) plan).child(); + assertTrue(plan instanceof Project); + plan = ((Project) plan).child(); assertTrue(plan instanceof Filter); Filter filter = (Filter) plan; @@ -138,10 +160,11 @@ public class OptimizerTests extends ESTestCase { for (String q : tests) { LogicalPlan plan = accept(q); - assertTrue(plan instanceof Project); - plan = ((Project) plan).child(); assertTrue(plan instanceof OrderBy); plan = ((OrderBy) plan).child(); + assertTrue(plan instanceof Project); + plan = ((Project) plan).child(); + assertTrue(plan instanceof Filter); Filter filter = (Filter) plan; @@ -159,10 +182,10 @@ public class OptimizerTests extends ESTestCase { public void testWildcardEscapes() { LogicalPlan plan = accept("foo where command_line == '* %bar_ * \\\\ \\n \\r \\t'"); - assertTrue(plan instanceof Project); - plan = ((Project) plan).child(); assertTrue(plan instanceof OrderBy); plan = ((OrderBy) plan).child(); + assertTrue(plan instanceof Project); + plan = ((Project) plan).child(); assertTrue(plan instanceof Filter); Filter filter = (Filter) plan; @@ -175,4 +198,112 @@ public class OptimizerTests extends ESTestCase { assertEquals(like.pattern().asLuceneWildcard(), "* %bar_ * \\\\ \n \r \t"); assertEquals(like.pattern().asIndexNameWildcard(), "* %bar_ * \\ \n \r \t"); } -} + + public void testCombineHeadBigHeadSmall() { + checkOffsetAndLimit(accept("process where true | head 10 | head 1"), 0, 1); + } + + public void testCombineHeadSmallHeadBig() { + checkOffsetAndLimit(accept("process where true | head 1 | head 12"), 0, 1); + } + + public void testCombineTailBigTailSmall() { + checkOffsetAndLimit(accept("process where true | tail 10 | tail 1"), 0, -1); + } + + public void testCombineTailSmallTailBig() { + checkOffsetAndLimit(accept("process where true | tail 1 | tail 12"), 0, -1); + } + + public void testCombineHeadBigTailSmall() { + checkOffsetAndLimit(accept("process where true | head 10 | tail 7"), 3, 7); + } + + public void testCombineTailBigHeadSmall() { + checkOffsetAndLimit(accept("process where true | tail 10 | head 7"), 3, -7); + } + + public void testCombineTailSmallHeadBig() { + checkOffsetAndLimit(accept("process where true | tail 7 | head 10"), 0, -7); + } + + public void testCombineHeadBigTailBig() { + checkOffsetAndLimit(accept("process where true | head 1 | tail 7"), 0, 1); + } + + public void testCombineHeadTailWithHeadAndTail() { + checkOffsetAndLimit(accept("process where true | head 10 | tail 7 | head 5 | tail 3"), 5, 3); + } + + public void testCombineTailHeadWithTailAndHead() { + checkOffsetAndLimit(accept("process where true | tail 10 | head 7 | tail 5 | head 3"), 5, -3); + } + + private void checkOffsetAndLimit(LogicalPlan plan, int offset, int limit) { + assertTrue(plan instanceof LimitWithOffset); + LimitWithOffset lo = (LimitWithOffset) plan; + assertEquals("Incorrect offset", offset, lo.offset()); + assertEquals("Incorrect limit", limit, lo.limit().fold()); + } + + private static Attribute timestamp() { + return new FieldAttribute(EMPTY, "test", new EsField("field", DataTypes.INTEGER, emptyMap(), true)); + } + + private static Attribute tiebreaker() { + return new EmptyAttribute(EMPTY); + } + + private static LogicalPlan rel() { + return new UnresolvedRelation(EMPTY, new TableIdentifier(EMPTY, "catalog", "index"), "", false); + } + + private static KeyedFilter keyedFilter(LogicalPlan child) { + return new KeyedFilter(EMPTY, child, emptyList(), timestamp(), tiebreaker()); + } + + public void testSkipQueryOnLimitZero() { + KeyedFilter rule1 = keyedFilter(new LocalRelation(EMPTY, emptyList())); + KeyedFilter rule2 = keyedFilter(new Filter(EMPTY, rel(), new IsNull(EMPTY, Literal.TRUE))); + KeyedFilter until = keyedFilter(new Filter(EMPTY, rel(), Literal.FALSE)); + Sequence s = new Sequence(EMPTY, asList(rule1, rule2), until, TimeValue.MINUS_ONE, timestamp(), tiebreaker(), OrderDirection.ASC); + + LogicalPlan optimized = optimizer.optimize(s); + assertEquals(LocalRelation.class, optimized.getClass()); + } + + public void testSortByLimit() { + Project p = new Project(EMPTY, rel(), emptyList()); + OrderBy o = new OrderBy(EMPTY, p, singletonList(new Order(EMPTY, tiebreaker(), OrderDirection.ASC, NullsPosition.FIRST))); + Tail t = new Tail(EMPTY, new Literal(EMPTY, 1, DataTypes.INTEGER), o); + + LogicalPlan optimized = new Optimizer.SortByLimit().rule(t); + assertEquals(LimitWithOffset.class, optimized.getClass()); + LimitWithOffset l = (LimitWithOffset) optimized; + assertOrder(l, OrderDirection.DESC); + } + + public void testPushdownOrderBy() { + Filter filter = new Filter(EMPTY, rel(), new IsNull(EMPTY, Literal.TRUE)); + KeyedFilter rule1 = keyedFilter(filter); + KeyedFilter rule2 = keyedFilter(filter); + KeyedFilter until = keyedFilter(filter); + Sequence s = new Sequence(EMPTY, asList(rule1, rule2), until, TimeValue.MINUS_ONE, timestamp(), tiebreaker(), OrderDirection.ASC); + OrderBy o = new OrderBy(EMPTY, s, singletonList(new Order(EMPTY, tiebreaker(), OrderDirection.DESC, NullsPosition.FIRST))); + + LogicalPlan optimized = new Optimizer.PushDownOrderBy().rule(o); + assertEquals(Sequence.class, optimized.getClass()); + Sequence seq = (Sequence) optimized; + + assertOrder(seq.until(), OrderDirection.ASC); + assertOrder(seq.queries().get(0), OrderDirection.DESC); + assertOrder(seq.queries().get(1), OrderDirection.ASC); + } + + private void assertOrder(UnaryPlan plan, OrderDirection direction) { + assertEquals(OrderBy.class, plan.child().getClass()); + OrderBy orderBy = (OrderBy) plan.child(); + Order order = orderBy.order().get(0); + assertEquals(direction, order.direction()); + } +} \ No newline at end of file diff --git a/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/parser/LogicalPlanTests.java b/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/parser/LogicalPlanTests.java index 8466a3c622f..9e35ed27f47 100644 --- a/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/parser/LogicalPlanTests.java +++ b/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/parser/LogicalPlanTests.java @@ -12,6 +12,7 @@ import org.elasticsearch.xpack.eql.plan.logical.Join; import org.elasticsearch.xpack.eql.plan.logical.KeyedFilter; import org.elasticsearch.xpack.eql.plan.logical.Sequence; import org.elasticsearch.xpack.eql.plan.physical.LocalRelation; +import org.elasticsearch.xpack.ql.expression.Attribute; import org.elasticsearch.xpack.ql.expression.Expression; import org.elasticsearch.xpack.ql.expression.NamedExpression; import org.elasticsearch.xpack.ql.expression.Order; @@ -28,7 +29,6 @@ import org.elasticsearch.xpack.ql.tree.Source; import java.util.List; import java.util.concurrent.TimeUnit; -import static java.util.Collections.emptyList; import static java.util.Collections.singletonList; import static org.elasticsearch.xpack.ql.type.DateUtils.UTC; @@ -40,26 +40,34 @@ public class LogicalPlanTests extends ESTestCase { return parser.createExpression(source); } + private static Attribute timestamp() { + return new UnresolvedAttribute(Source.EMPTY, "@timestamp"); + } + + private static LogicalPlan relation() { + return new UnresolvedRelation(Source.EMPTY, null, "", false, ""); + } + public void testAnyQuery() { LogicalPlan fullQuery = parser.createStatement("any where process_name == 'net.exe'"); Expression fullExpression = expr("process_name == 'net.exe'"); - LogicalPlan filter = new Filter(Source.EMPTY, new UnresolvedRelation(Source.EMPTY, null, "", false, ""), fullExpression); - Order order = new Order(Source.EMPTY, new UnresolvedAttribute(Source.EMPTY, "@timestamp"), OrderDirection.ASC, NullsPosition.FIRST); - LogicalPlan sorted = new OrderBy(Source.EMPTY, filter, singletonList(order)); - LogicalPlan expected = new Project(Source.EMPTY, sorted, emptyList()); - assertEquals(expected, fullQuery); + LogicalPlan filter = new Filter(Source.EMPTY, relation(), fullExpression); + Order order = new Order(Source.EMPTY, timestamp(), OrderDirection.ASC, NullsPosition.FIRST); + LogicalPlan project = new Project(Source.EMPTY, filter, singletonList(timestamp())); + LogicalPlan sorted = new OrderBy(Source.EMPTY, project, singletonList(order)); + assertEquals(sorted, fullQuery); } public void testEventQuery() { LogicalPlan fullQuery = parser.createStatement("process where process_name == 'net.exe'"); Expression fullExpression = expr("event.category == 'process' and process_name == 'net.exe'"); - LogicalPlan filter = new Filter(Source.EMPTY, new UnresolvedRelation(Source.EMPTY, null, "", false, ""), fullExpression); - Order order = new Order(Source.EMPTY, new UnresolvedAttribute(Source.EMPTY, "@timestamp"), OrderDirection.ASC, NullsPosition.FIRST); - LogicalPlan sorted = new OrderBy(Source.EMPTY, filter, singletonList(order)); - LogicalPlan expected = new Project(Source.EMPTY, sorted, emptyList()); - assertEquals(expected, fullQuery); + LogicalPlan filter = new Filter(Source.EMPTY, relation(), fullExpression); + Order order = new Order(Source.EMPTY, timestamp(), OrderDirection.ASC, NullsPosition.FIRST); + LogicalPlan project = new Project(Source.EMPTY, filter, singletonList(timestamp())); + LogicalPlan sorted = new OrderBy(Source.EMPTY, project, singletonList(order)); + assertEquals(sorted, fullQuery); } public void testParameterizedEventQuery() { @@ -67,11 +75,11 @@ public class LogicalPlanTests extends ESTestCase { LogicalPlan fullQuery = parser.createStatement("process where process_name == 'net.exe'", params); Expression fullExpression = expr("myCustomEvent == 'process' and process_name == 'net.exe'"); - LogicalPlan filter = new Filter(Source.EMPTY, new UnresolvedRelation(Source.EMPTY, null, "", false, ""), fullExpression); - Order order = new Order(Source.EMPTY, new UnresolvedAttribute(Source.EMPTY, "@timestamp"), OrderDirection.ASC, NullsPosition.FIRST); - LogicalPlan sorted = new OrderBy(Source.EMPTY, filter, singletonList(order)); - LogicalPlan expected = new Project(Source.EMPTY, sorted, emptyList()); - assertEquals(expected, fullQuery); + LogicalPlan filter = new Filter(Source.EMPTY, relation(), fullExpression); + Order order = new Order(Source.EMPTY, timestamp(), OrderDirection.ASC, NullsPosition.FIRST); + LogicalPlan project = new Project(Source.EMPTY, filter, singletonList(timestamp())); + LogicalPlan sorted = new OrderBy(Source.EMPTY, project, singletonList(order)); + assertEquals(sorted, fullQuery); } @@ -85,6 +93,9 @@ public class LogicalPlanTests extends ESTestCase { " " + "until [process where event_subtype_full == \"termination_event\"]"); + assertEquals(OrderBy.class, plan.getClass()); + OrderBy ob = (OrderBy) plan; + plan = ob.child(); assertEquals(Join.class, plan.getClass()); Join join = (Join) plan; assertEquals(KeyedFilter.class, join.until().getClass()); @@ -113,6 +124,9 @@ public class LogicalPlanTests extends ESTestCase { " [process where process_name == \"*\" ] " + " [file where file_path == \"*\"]"); + assertEquals(OrderBy.class, plan.getClass()); + OrderBy ob = (OrderBy) plan; + plan = ob.child(); assertEquals(Sequence.class, plan.getClass()); Sequence seq = (Sequence) plan; assertEquals(KeyedFilter.class, seq.until().getClass()); diff --git a/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/planner/QueryFolderOkTests.java b/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/planner/QueryFolderOkTests.java index 835181bdb40..d175e3b08fa 100644 --- a/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/planner/QueryFolderOkTests.java +++ b/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/planner/QueryFolderOkTests.java @@ -120,7 +120,7 @@ public class QueryFolderOkTests extends AbstractQueryFolderTestCase { PhysicalPlan p = plan(query); assertEquals(EsQueryExec.class, p.getClass()); EsQueryExec eqe = (EsQueryExec) p; - assertEquals(0, eqe.output().size()); + assertEquals(1, eqe.output().size()); final String query = eqe.queryContainer().toString().replaceAll("\\s+", ""); diff --git a/x-pack/plugin/eql/src/test/resources/queries-supported.eql b/x-pack/plugin/eql/src/test/resources/queries-supported.eql index 372760adc3f..068149f1066 100644 --- a/x-pack/plugin/eql/src/test/resources/queries-supported.eql +++ b/x-pack/plugin/eql/src/test/resources/queries-supported.eql @@ -564,4 +564,126 @@ sequence by pid with maxspan=500ms sequence by pid with maxspan=2h [process where process_name == "*" ] [file where file_path == "*"] -; \ No newline at end of file +; + +// +// Pipes +// + + +security where event_id == 4624 +| tail 10 +; + +process where true | head 6; +process where bad_field == null | head 5; + +process where not (exit_code > -1) + and serial_event_id in (58, 64, 69, 74, 80, 85, 90, 93, 94) +| head 10 +; + + +process where not (exit_code > -1) | head 7; + +process where not (-1 < exit_code) | head 7; + + +file where true +| tail 3; + +sequence + [file where event_subtype_full == "file_create_event"] by file_path + [process where opcode == 1] by process_path + [process where opcode == 2] by process_path + [file where event_subtype_full == "file_delete_event"] by file_path +| head 4 +| tail 2; + + +sequence + [file where opcode=0] by unique_pid + [file where opcode=0] by unique_pid +| head 1; + + +sequence with maxspan=1d + [file where event_subtype_full == "file_create_event"] by file_path + [process where opcode == 1] by process_path + [process where opcode == 2] by process_path + [file where event_subtype_full == "file_delete_event"] by file_path +| head 4 +| tail 2; + +sequence with maxspan=1h + [file where event_subtype_full == "file_create_event"] by file_path + [process where opcode == 1] by process_path + [process where opcode == 2] by process_path + [file where event_subtype_full == "file_delete_event"] by file_path +| head 4 +| tail 2; + +sequence with maxspan=1m + [file where event_subtype_full == "file_create_event"] by file_path + [process where opcode == 1] by process_path + [process where opcode == 2] by process_path + [file where event_subtype_full == "file_delete_event"] by file_path +| head 4 +| tail 2; + +sequence with maxspan=10s + [file where event_subtype_full == "file_create_event"] by file_path + [process where opcode == 1] by process_path + [process where opcode == 2] by process_path + [file where event_subtype_full == "file_delete_event"] by file_path +| head 4 +| tail 2; + +sequence + [file where opcode=0 and file_name="*.exe"] by unique_pid + [file where opcode=0 and file_name="*.exe"] by unique_pid +until [process where opcode=5000] by unique_ppid +| head 1; + +sequence + [file where opcode=0 and file_name="*.exe"] by unique_pid + [file where opcode=0 and file_name="*.exe"] by unique_pid +until [process where opcode=1] by unique_ppid +| head 1; + +join + [file where opcode=0 and file_name="*.exe"] by unique_pid + [file where opcode=2 and file_name="*.exe"] by unique_pid +until [process where opcode=1] by unique_ppid +| head 1; + +sequence by user_name + [file where opcode=0] by file_path + [process where opcode=1] by process_path + [process where opcode=2] by process_path + [file where opcode=2] by file_path +| tail 1; + +sequence by user_name + [file where opcode=0] by pid,file_path + [file where opcode=2] by pid,file_path +until [process where opcode=5] by ppid,process_path +| head 2; + +sequence by pid + [file where opcode=0] by file_path + [process where opcode=1] by process_path + [process where opcode=2] by process_path + [file where opcode=2] by file_path +| tail 1; + +join by user_name + [file where true] by pid,file_path + [process where true] by ppid,process_path +| head 2; + +process where fake_field != "*" +| head 4; + +process where not (fake_field == "*") +| head 4; \ No newline at end of file diff --git a/x-pack/plugin/eql/src/test/resources/queries-unsupported.eql b/x-pack/plugin/eql/src/test/resources/queries-unsupported.eql index 46bbd6d02ee..1ffc81d483a 100644 --- a/x-pack/plugin/eql/src/test/resources/queries-unsupported.eql +++ b/x-pack/plugin/eql/src/test/resources/queries-unsupported.eql @@ -22,10 +22,6 @@ process where process_name == "powershell.exe" | head 50 ; -security where event_id == 4624 -| tail 10 -; - file where true | sort file_name ; @@ -82,6 +78,12 @@ sequence by unique_pid [process where true] [file where true] fork=true; // no longer supported //sequence by unique_pid [process where true] [file where true] fork=1; +sequence + [process where true] by unique_pid + [file where true] fork=true by unique_pid + [process where true] by unique_ppid +| head 4; + sequence by unique_pid [process where true] [file where true] fork=false; // no longer supported @@ -120,9 +122,6 @@ sequence by pid with maxspan=1.0075d * https://raw.githubusercontent.com/endgameinc/eql/master/eql/etc/test_queries.toml */ -process where true | head 6; -process where bad_field == null | head 5; - process where serial_event_id <= 8 and serial_event_id > 7 | filter serial_event_id == 8; @@ -143,15 +142,6 @@ process where true ; -process where not (exit_code > -1) - and serial_event_id in (58, 64, 69, 74, 80, 85, 90, 93, 94) -| head 10 -; - - -process where not (exit_code > -1) | head 7; - -process where not (-1 < exit_code) | head 7; process where process_name == "VMACTHLP.exe" and unique_pid == 12 | filter true; @@ -212,10 +202,6 @@ process where opcode=1 and process_name == "smss.exe" -file where true -| tail 3; - - file where true | tail 4 @@ -243,46 +229,6 @@ process where true | sort md5, event_subtype_full, null_field, process_name | sort serial_event_id; -sequence - [file where event_subtype_full == "file_create_event"] by file_path - [process where opcode == 1] by process_path - [process where opcode == 2] by process_path - [file where event_subtype_full == "file_delete_event"] by file_path -| head 4 -| tail 2; - -sequence with maxspan=1d - [file where event_subtype_full == "file_create_event"] by file_path - [process where opcode == 1] by process_path - [process where opcode == 2] by process_path - [file where event_subtype_full == "file_delete_event"] by file_path -| head 4 -| tail 2; - -sequence with maxspan=1h - [file where event_subtype_full == "file_create_event"] by file_path - [process where opcode == 1] by process_path - [process where opcode == 2] by process_path - [file where event_subtype_full == "file_delete_event"] by file_path -| head 4 -| tail 2; - -sequence with maxspan=1m - [file where event_subtype_full == "file_create_event"] by file_path - [process where opcode == 1] by process_path - [process where opcode == 2] by process_path - [file where event_subtype_full == "file_delete_event"] by file_path -| head 4 -| tail 2; - -sequence with maxspan=10s - [file where event_subtype_full == "file_create_event"] by file_path - [process where opcode == 1] by process_path - [process where opcode == 2] by process_path - [file where event_subtype_full == "file_delete_event"] by file_path -| head 4 -| tail 2; - sequence with maxspan=0.5s [file where event_subtype_full == "file_create_event"] by file_path [process where opcode == 1] by process_path @@ -291,55 +237,14 @@ sequence with maxspan=0.5s | head 4 | tail 2; -sequence - [file where opcode=0] by unique_pid - [file where opcode=0] by unique_pid -| head 1; - sequence [file where opcode=0] by unique_pid [file where opcode=0] by unique_pid | filter events[1].serial_event_id == 92; -sequence - [file where opcode=0 and file_name="*.exe"] by unique_pid - [file where opcode=0 and file_name="*.exe"] by unique_pid -until [process where opcode=5000] by unique_ppid -| head 1; - -sequence - [file where opcode=0 and file_name="*.exe"] by unique_pid - [file where opcode=0 and file_name="*.exe"] by unique_pid -until [process where opcode=1] by unique_ppid -| head 1; - -join - [file where opcode=0 and file_name="*.exe"] by unique_pid - [file where opcode=2 and file_name="*.exe"] by unique_pid -until [process where opcode=1] by unique_ppid -| head 1 -; - -join by string(unique_pid) - [process where opcode=1] - [file where opcode=0 and file_name="svchost.exe"] - [file where opcode == 0 and file_name == "lsass.exe"] -| head 1 -; - -join by string(unique_pid), unique_pid, unique_pid * 2 - [process where opcode=1] - [file where opcode=0 and file_name="svchost.exe"] - [file where opcode == 0 and file_name == "lsass.exe"] -until [file where opcode == 2] -: tail 1 -; - any where true | unique event_type_full; - - process where opcode=1 and process_name in ("services.exe", "smss.exe", "lsass.exe") and descendant of [process where process_name == "cmd.exe" ]; @@ -392,39 +297,6 @@ sequence | tail 1 ; -sequence by user_name - [file where opcode=0] by file_path - [process where opcode=1] by process_path - [process where opcode=2] by process_path - [file where opcode=2] by file_path -| tail 1; - -sequence by user_name - [file where opcode=0] by pid,file_path - [file where opcode=2] by pid,file_path -until [process where opcode=5] by ppid,process_path -| head 2; - -sequence by pid - [file where opcode=0] by file_path - [process where opcode=1] by process_path - [process where opcode=2] by process_path - [file where opcode=2] by file_path -| tail 1; - -join by user_name - [file where true] by pid,file_path - [process where true] by ppid,process_path -| head 2; - -sequence - [process where true] by unique_pid - [file where true] fork=true by unique_pid - [process where true] by unique_ppid -| head 4; - - - process where 'net.EXE' == original_file_name | filter process_name="net*.exe" @@ -443,15 +315,6 @@ process where original_file_name == process_name process where process_name != original_file_name | filter length(original_file_name) > 0; - - - -process where fake_field != "*" -| head 4; - -process where not (fake_field == "*") -| head 4; - any where process_name == "svchost.exe" | unique_count event_type_full, process_name; diff --git a/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/optimizer/OptimizerRules.java b/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/optimizer/OptimizerRules.java index 58e1c19e357..3b9df1d4f56 100644 --- a/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/optimizer/OptimizerRules.java +++ b/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/optimizer/OptimizerRules.java @@ -27,6 +27,7 @@ import org.elasticsearch.xpack.ql.expression.predicate.operator.comparison.LessT import org.elasticsearch.xpack.ql.expression.predicate.operator.comparison.NotEquals; import org.elasticsearch.xpack.ql.expression.predicate.operator.comparison.NullEquals; import org.elasticsearch.xpack.ql.plan.logical.Filter; +import org.elasticsearch.xpack.ql.plan.logical.Limit; import org.elasticsearch.xpack.ql.plan.logical.LogicalPlan; import org.elasticsearch.xpack.ql.plan.logical.OrderBy; import org.elasticsearch.xpack.ql.rule.Rule; @@ -1057,7 +1058,7 @@ public final class OptimizerRules { return filter.child(); } if (FALSE.equals(condition) || Expressions.isNull(condition)) { - return nonMatchingFilter(filter); + return skipPlan(filter); } } @@ -1067,7 +1068,7 @@ public final class OptimizerRules { return filter; } - protected abstract LogicalPlan nonMatchingFilter(Filter filter); + protected abstract LogicalPlan skipPlan(Filter filter); private static Expression foldBinaryLogic(Expression expression) { if (expression instanceof Or) { @@ -1120,6 +1121,21 @@ public final class OptimizerRules { } } + + public abstract static class SkipQueryOnLimitZero extends OptimizerRule { + @Override + protected LogicalPlan rule(Limit limit) { + if (limit.limit().foldable()) { + if (Integer.valueOf(0).equals((limit.limit().fold()))) { + return skipPlan(limit); + } + } + return limit; + } + + protected abstract LogicalPlan skipPlan(Limit limit); + } + public static final class SetAsOptimized extends Rule { @Override diff --git a/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/plan/logical/EsRelation.java b/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/plan/logical/EsRelation.java index c16c5319da5..0368fe08be4 100644 --- a/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/plan/logical/EsRelation.java +++ b/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/plan/logical/EsRelation.java @@ -9,12 +9,11 @@ import org.elasticsearch.xpack.ql.expression.Attribute; import org.elasticsearch.xpack.ql.expression.FieldAttribute; import org.elasticsearch.xpack.ql.index.EsIndex; import org.elasticsearch.xpack.ql.tree.NodeInfo; +import org.elasticsearch.xpack.ql.tree.NodeUtils; import org.elasticsearch.xpack.ql.tree.Source; import org.elasticsearch.xpack.ql.type.EsField; import java.util.ArrayList; -import java.util.Collection; -import java.util.Iterator; import java.util.List; import java.util.Map; import java.util.Map.Entry; @@ -99,36 +98,8 @@ public class EsRelation extends LeafPlan { && frozen == other.frozen; } - private static final int TO_STRING_LIMIT = 52; - - private static String limitedToString(Collection c) { - Iterator it = c.iterator(); - if (!it.hasNext()) { - return "[]"; - } - - // ..] - StringBuilder sb = new StringBuilder(TO_STRING_LIMIT + 4); - sb.append('['); - for (;;) { - E e = it.next(); - String next = e == c ? "(this Collection)" : String.valueOf(e); - if (next.length() + sb.length() > TO_STRING_LIMIT) { - sb.append(next.substring(0, Math.max(0, TO_STRING_LIMIT - sb.length()))); - sb.append('.').append('.').append(']'); - return sb.toString(); - } else { - sb.append(next); - } - if (!it.hasNext()) { - return sb.append(']').toString(); - } - sb.append(',').append(' '); - } - } - @Override public String nodeString() { - return nodeName() + "[" + index + "]" + limitedToString(attrs); + return nodeName() + "[" + index + "]" + NodeUtils.limitedToString(attrs); } } \ No newline at end of file diff --git a/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/tree/NodeUtils.java b/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/tree/NodeUtils.java index 3e2b67a9e79..92086f64585 100644 --- a/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/tree/NodeUtils.java +++ b/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/tree/NodeUtils.java @@ -5,6 +5,9 @@ */ package org.elasticsearch.xpack.ql.tree; +import java.util.Collection; +import java.util.Iterator; + public abstract class NodeUtils { public static , B extends Node> String diffString(A left, B right) { return diffString(left.toString(), right.toString()); @@ -53,4 +56,33 @@ public abstract class NodeUtils { } return sb.toString(); } + + + private static final int TO_STRING_LIMIT = 52; + + public static String limitedToString(Collection c) { + Iterator it = c.iterator(); + if (!it.hasNext()) { + return "[]"; + } + + // ..] + StringBuilder sb = new StringBuilder(TO_STRING_LIMIT + 4); + sb.append('['); + for (;;) { + E e = it.next(); + String next = e == c ? "(this Collection)" : String.valueOf(e); + if (next.length() + sb.length() > TO_STRING_LIMIT) { + sb.append(next.substring(0, Math.max(0, TO_STRING_LIMIT - sb.length()))); + sb.append('.').append('.').append(']'); + return sb.toString(); + } else { + sb.append(next); + } + if (!it.hasNext()) { + return sb.append(']').toString(); + } + sb.append(',').append(' '); + } + } } diff --git a/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/util/StringUtils.java b/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/util/StringUtils.java index 6c74a862f96..638a1b4efbb 100644 --- a/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/util/StringUtils.java +++ b/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/util/StringUtils.java @@ -337,4 +337,4 @@ public final class StringUtils { } } -} +} \ No newline at end of file diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/optimizer/Optimizer.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/optimizer/Optimizer.java index 78ab1d4f808..3bbc125ef0a 100644 --- a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/optimizer/Optimizer.java +++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/optimizer/Optimizer.java @@ -1113,24 +1113,23 @@ public class Optimizer extends RuleExecutor { static class PruneFilters extends org.elasticsearch.xpack.ql.optimizer.OptimizerRules.PruneFilters { @Override - protected LogicalPlan nonMatchingFilter(Filter filter) { - return new LocalRelation(filter.source(), new EmptyExecutable(filter.output())); + protected LogicalPlan skipPlan(Filter filter) { + return Optimizer.skipPlan(filter); } } + static class SkipQueryOnLimitZero extends org.elasticsearch.xpack.ql.optimizer.OptimizerRules.SkipQueryOnLimitZero { - static class SkipQueryOnLimitZero extends OptimizerRule { @Override - protected LogicalPlan rule(Limit limit) { - if (limit.limit() instanceof Literal) { - if (Integer.valueOf(0).equals((limit.limit().fold()))) { - return new LocalRelation(limit.source(), new EmptyExecutable(limit.output())); - } - } - return limit; + protected LogicalPlan skipPlan(Limit limit) { + return Optimizer.skipPlan(limit); } } + private static LogicalPlan skipPlan(UnaryPlan plan) { + return new LocalRelation(plan.source(), new EmptyExecutable(plan.output())); + } + static class SkipQueryIfFoldingProjection extends OptimizerRule { @Override protected LogicalPlan rule(LogicalPlan plan) { diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/plan/logical/LocalRelation.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/plan/logical/LocalRelation.java index 442ef99e377..5b4f94e8da9 100644 --- a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/plan/logical/LocalRelation.java +++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/plan/logical/LocalRelation.java @@ -9,6 +9,7 @@ import org.elasticsearch.action.ActionListener; import org.elasticsearch.xpack.ql.expression.Attribute; import org.elasticsearch.xpack.ql.plan.logical.LogicalPlan; import org.elasticsearch.xpack.ql.tree.NodeInfo; +import org.elasticsearch.xpack.ql.tree.NodeUtils; import org.elasticsearch.xpack.ql.tree.Source; import org.elasticsearch.xpack.sql.session.Cursor.Page; import org.elasticsearch.xpack.sql.session.Executable; @@ -75,4 +76,9 @@ public class LocalRelation extends LogicalPlan implements Executable { LocalRelation other = (LocalRelation) obj; return Objects.equals(executable, other.executable); } -} + + @Override + public String nodeString() { + return nodeName() + NodeUtils.limitedToString(output()); + } +} \ No newline at end of file