EQL: Avoid filtering on tiebreakers (#63215)
Do not filter by tiebreaker while searching sequence matches as it's not monotonic and thus can filter out valid data. Fix #62781 (cherry picked from commit 4d62198df70f3b70f8b6e7730e888057652c18a8)
This commit is contained in:
parent
4f593bdd69
commit
efd2243886
|
@ -6,7 +6,7 @@ query = '''
|
||||||
[ ERROR where true ]
|
[ ERROR where true ]
|
||||||
[ STAT where true ]
|
[ STAT where true ]
|
||||||
'''
|
'''
|
||||||
expected_event_ids = [1,2,3]
|
expected_event_ids = [1,2,3,1,2,3]
|
||||||
|
|
||||||
[[queries]]
|
[[queries]]
|
||||||
name = "basicWithFilter"
|
name = "basicWithFilter"
|
||||||
|
|
|
@ -7,9 +7,7 @@
|
||||||
package org.elasticsearch.xpack.eql;
|
package org.elasticsearch.xpack.eql;
|
||||||
|
|
||||||
import org.elasticsearch.test.eql.EqlExtraSpecTestCase;
|
import org.elasticsearch.test.eql.EqlExtraSpecTestCase;
|
||||||
import org.elasticsearch.test.junit.annotations.TestLogging;
|
|
||||||
|
|
||||||
@TestLogging(value = "org.elasticsearch.xpack.eql:TRACE", reason = "results logging")
|
|
||||||
public class EqlExtraIT extends EqlExtraSpecTestCase {
|
public class EqlExtraIT extends EqlExtraSpecTestCase {
|
||||||
|
|
||||||
public EqlExtraIT(String query, String name, long[] eventIds, boolean caseSensitive) {
|
public EqlExtraIT(String query, String name, long[] eventIds, boolean caseSensitive) {
|
||||||
|
|
|
@ -21,29 +21,22 @@ import static org.elasticsearch.index.query.QueryBuilders.rangeQuery;
|
||||||
*
|
*
|
||||||
* Note that the range is not set at once on purpose since each query tends to have
|
* Note that the range is not set at once on purpose since each query tends to have
|
||||||
* its own number of results separate from the others.
|
* its own number of results separate from the others.
|
||||||
* As such, each query starts where it lefts to reach the current in-progress window
|
* As such, each query starts from where it left off to reach the current in-progress window
|
||||||
* as oppose to always operating with the exact same window.
|
* as oppose to always operating with the exact same window.
|
||||||
*/
|
*/
|
||||||
public class BoxedQueryRequest implements QueryRequest {
|
public class BoxedQueryRequest implements QueryRequest {
|
||||||
|
|
||||||
private final RangeQueryBuilder timestampRange;
|
private final RangeQueryBuilder timestampRange;
|
||||||
private final RangeQueryBuilder tiebreakerRange;
|
|
||||||
|
|
||||||
private final SearchSourceBuilder searchSource;
|
private final SearchSourceBuilder searchSource;
|
||||||
|
|
||||||
private Ordinal from, to;
|
private Ordinal from, to;
|
||||||
private Ordinal after;
|
private Ordinal after;
|
||||||
|
|
||||||
public BoxedQueryRequest(QueryRequest original, String timestamp, String tiebreaker) {
|
public BoxedQueryRequest(QueryRequest original, String timestamp) {
|
||||||
// setup range queries and preserve their reference to simplify the update
|
// setup range queries and preserve their reference to simplify the update
|
||||||
timestampRange = rangeQuery(timestamp).timeZone("UTC").format("epoch_millis");
|
timestampRange = rangeQuery(timestamp).timeZone("UTC").format("epoch_millis");
|
||||||
BoolQueryBuilder filter = boolQuery().filter(timestampRange);
|
BoolQueryBuilder filter = boolQuery().filter(timestampRange);
|
||||||
if (tiebreaker != null) {
|
|
||||||
tiebreakerRange = rangeQuery(tiebreaker);
|
|
||||||
filter.filter(tiebreakerRange);
|
|
||||||
} else {
|
|
||||||
tiebreakerRange = null;
|
|
||||||
}
|
|
||||||
|
|
||||||
searchSource = original.searchSource();
|
searchSource = original.searchSource();
|
||||||
// combine with existing query (if it exists)
|
// combine with existing query (if it exists)
|
||||||
|
@ -72,9 +65,6 @@ public class BoxedQueryRequest implements QueryRequest {
|
||||||
public BoxedQueryRequest from(Ordinal begin) {
|
public BoxedQueryRequest from(Ordinal begin) {
|
||||||
from = begin;
|
from = begin;
|
||||||
timestampRange.gte(begin != null ? begin.timestamp() : null);
|
timestampRange.gte(begin != null ? begin.timestamp() : null);
|
||||||
if (tiebreakerRange != null) {
|
|
||||||
tiebreakerRange.gte(begin != null ? begin.tiebreaker() : null);
|
|
||||||
}
|
|
||||||
return this;
|
return this;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -93,9 +83,6 @@ public class BoxedQueryRequest implements QueryRequest {
|
||||||
public BoxedQueryRequest to(Ordinal end) {
|
public BoxedQueryRequest to(Ordinal end) {
|
||||||
to = end;
|
to = end;
|
||||||
timestampRange.lte(end != null ? end.timestamp() : null);
|
timestampRange.lte(end != null ? end.timestamp() : null);
|
||||||
if (tiebreakerRange != null) {
|
|
||||||
tiebreakerRange.lte(end != null ? end.tiebreaker() : null);
|
|
||||||
}
|
|
||||||
return this;
|
return this;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -72,7 +72,7 @@ public class ExecutionManager {
|
||||||
if (query instanceof EsQueryExec) {
|
if (query instanceof EsQueryExec) {
|
||||||
SearchSourceBuilder source = ((EsQueryExec) query).source(session);
|
SearchSourceBuilder source = ((EsQueryExec) query).source(session);
|
||||||
QueryRequest original = () -> source;
|
QueryRequest original = () -> source;
|
||||||
BoxedQueryRequest boxedRequest = new BoxedQueryRequest(original, timestampName, tiebreakerName);
|
BoxedQueryRequest boxedRequest = new BoxedQueryRequest(original, timestampName);
|
||||||
Criterion<BoxedQueryRequest> criterion =
|
Criterion<BoxedQueryRequest> criterion =
|
||||||
new Criterion<>(i, boxedRequest, keyExtractors, tsExtractor, tbExtractor, i == 0 && descending);
|
new Criterion<>(i, boxedRequest, keyExtractors, tsExtractor, tbExtractor, i == 0 && descending);
|
||||||
criteria.add(criterion);
|
criteria.add(criterion);
|
||||||
|
|
|
@ -95,7 +95,7 @@ public class SequenceSpecTests extends ESTestCase {
|
||||||
|
|
||||||
TestCriterion(final int ordinal) {
|
TestCriterion(final int ordinal) {
|
||||||
super(ordinal,
|
super(ordinal,
|
||||||
new BoxedQueryRequest(() -> SearchSourceBuilder.searchSource().query(matchAllQuery()).size(ordinal), "timestamp", null),
|
new BoxedQueryRequest(() -> SearchSourceBuilder.searchSource().query(matchAllQuery()).size(ordinal), "timestamp"),
|
||||||
keyExtractors,
|
keyExtractors,
|
||||||
tsExtractor, tbExtractor, false);
|
tsExtractor, tbExtractor, false);
|
||||||
this.ordinal = ordinal;
|
this.ordinal = ordinal;
|
||||||
|
|
Loading…
Reference in New Issue