diff --git a/x-pack/plugin/eql/qa/common/src/main/resources/test_queries.toml b/x-pack/plugin/eql/qa/common/src/main/resources/test_queries.toml index a017908ba57..e0d0c7bf14f 100644 --- a/x-pack/plugin/eql/qa/common/src/main/resources/test_queries.toml +++ b/x-pack/plugin/eql/qa/common/src/main/resources/test_queries.toml @@ -342,19 +342,19 @@ query = ''' registry where length(bytes_written_string_list) == 2 and bytes_written_string_list[1] : "EN" ''' -[[queries]] -name = "keyPathWildcard" -query = ''' -registry where key_path : "*\\MACHINE\\SAM\\SAM\\*\\Account\\Us*ers\\00*03E9\\F" -''' -expected_event_ids = [79] - -[[queries]] -name = "processPathWildcardAndIN" -query = ''' -process where process_path : "*\\red_ttp\\wininit.*" and opcode in (0,1,2,3,4) -''' -expected_event_ids = [84, 85] +# [[queries]] +# name = "keyPathWildcard" +# query = ''' +# registry where key_path : "*\\MACHINE\\SAM\\SAM\\*\\Account\\Us*ers\\00*03E9\\F" +# ''' +# expected_event_ids = [79] +# +# [[queries]] +# name = "processPathWildcardAndIN" +# query = ''' +# process where process_path : "*\\red_ttp\\wininit.*" and opcode in (0,1,2,3,4) +# ''' +# expected_event_ids = [84, 85] [[queries]] name = "descendant1" @@ -385,13 +385,13 @@ process where opcode==1 and process_name : "smss.exe" ''' expected_event_ids = [78] -[[queries]] -name = "wildcardAndMultipleConditions1" -query = ''' -file where file_path:"*\\red_ttp\\winin*.*" - and opcode in (0,1,2) and user_name:"vagrant" -''' -expected_event_ids = [83, 86] +# [[queries]] +# name = "wildcardAndMultipleConditions1" +# query = ''' +# file where file_path:"*\\red_ttp\\winin*.*" +# and opcode in (0,1,2) and user_name:"vagrant" +# ''' +# expected_event_ids = [83, 86] [[queries]] name = "wildcardAndMultipleConditions2" @@ -401,13 +401,13 @@ file where file_path:"*\\red_ttp\\winin*.*" ''' expected_event_ids = [] -[[queries]] -name = "wildcardAndMultipleConditions3" -query = ''' -file where file_path:"*\\red_ttp\\winin*.*" - and opcode not in (3, 4, 5, 6 ,7) and user_name:"vagrant" -''' -expected_event_ids = [83, 86] +# [[queries]] +# name = "wildcardAndMultipleConditions3" +# query = ''' +# file where file_path:"*\\red_ttp\\winin*.*" +# and opcode not in (3, 4, 5, 6 ,7) and user_name:"vagrant" +# ''' +# expected_event_ids = [83, 86] [[queries]] @@ -878,16 +878,16 @@ sequence ''' expected_event_ids = [87, 92] -[[queries]] -name = "doubleSameSequenceWithByUntilAndHead1" -query = ''' -sequence - [file where opcode==0 and file_name:"*.exe"] by unique_pid - [file where opcode==0 and file_name:"*.exe"] by unique_pid -until [process where opcode==5000] by unique_ppid -| head 1 -''' -expected_event_ids = [55, 61] +# [[queries]] +# name = "doubleSameSequenceWithByUntilAndHead1" +# query = ''' +# sequence +# [file where opcode==0 and file_name:"*.exe"] by unique_pid +# [file where opcode==0 and file_name:"*.exe"] by unique_pid +# until [process where opcode==5000] by unique_ppid +# | head 1 +# ''' +# expected_event_ids = [55, 61] [[queries]] name = "doubleSameSequenceWithByUntilAndHead2" @@ -1040,16 +1040,16 @@ query = ''' registry where length(bad_field) > 0 ''' -[[queries]] -name = "multipleConditions2" -query = ''' -process where opcode == 1 - and process_name in ("net.exe", "net1.exe") - and not (parent_process_name : "net.exe" - and process_name : "net1.exe") - and command_line : "*group *admin*" and command_line != "* /add*" - ''' -expected_event_ids = [97] +# [[queries]] +# name = "multipleConditions2" +# query = ''' +# process where opcode == 1 +# and process_name in ("net.exe", "net1.exe") +# and not (parent_process_name : "net.exe" +# and process_name : "net1.exe") +# and command_line : "*group *admin*" and command_line != "* /add*" +# ''' +# expected_event_ids = [97] [[queries]] name = "anyWithUnique" @@ -1246,26 +1246,26 @@ sequence ''' expected_event_ids = [54, 55, 56, 54, 61, 62, 54, 67, 68, 54, 72, 73] -[[queries]] -name = "wildcard1" -query = ''' -process where command_line : "*%*" -''' -expected_event_ids = [4, 6, 28] - -[[queries]] -name = "wildcard2" -query = ''' -process where command_line : "*%*%*" -''' -expected_event_ids = [4, 6, 28] - -[[queries]] -name = "wildcard3" -query = ''' -process where command_line : "%*%*" -''' -expected_event_ids = [4, 6, 28] +# [[queries]] +# name = "wildcard1" +# query = ''' +# process where command_line : "*%*" +# ''' +# expected_event_ids = [4, 6, 28] +# +# [[queries]] +# name = "wildcard2" +# query = ''' +# process where command_line : "*%*%*" +# ''' +# expected_event_ids = [4, 6, 28] +# +# [[queries]] +# name = "wildcard3" +# query = ''' +# process where command_line : "%*%*" +# ''' +# expected_event_ids = [4, 6, 28] [[queries]] name = "uniqueCount1" diff --git a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/optimizer/Optimizer.java b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/optimizer/Optimizer.java index 220f0595fd4..018c319e26e 100644 --- a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/optimizer/Optimizer.java +++ b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/optimizer/Optimizer.java @@ -7,15 +7,12 @@ package org.elasticsearch.xpack.eql.optimizer; import org.elasticsearch.xpack.eql.EqlIllegalArgumentException; -import org.elasticsearch.xpack.eql.expression.predicate.operator.comparison.InsensitiveBinaryComparison; -import org.elasticsearch.xpack.eql.expression.predicate.operator.comparison.InsensitiveNotEquals; import org.elasticsearch.xpack.eql.plan.logical.Join; import org.elasticsearch.xpack.eql.plan.logical.KeyedFilter; import org.elasticsearch.xpack.eql.plan.logical.LimitWithOffset; import org.elasticsearch.xpack.eql.plan.physical.LocalRelation; import org.elasticsearch.xpack.eql.session.Payload.Type; import org.elasticsearch.xpack.eql.util.MathUtils; -import org.elasticsearch.xpack.eql.util.StringUtils; import org.elasticsearch.xpack.ql.expression.Expression; import org.elasticsearch.xpack.ql.expression.Literal; import org.elasticsearch.xpack.ql.expression.NamedExpression; @@ -24,14 +21,12 @@ import org.elasticsearch.xpack.ql.expression.Order.NullsPosition; import org.elasticsearch.xpack.ql.expression.Order.OrderDirection; import org.elasticsearch.xpack.ql.expression.predicate.Predicates; import org.elasticsearch.xpack.ql.expression.predicate.logical.And; -import org.elasticsearch.xpack.ql.expression.predicate.logical.Not; import org.elasticsearch.xpack.ql.expression.predicate.logical.Or; import org.elasticsearch.xpack.ql.expression.predicate.nulls.IsNotNull; import org.elasticsearch.xpack.ql.expression.predicate.nulls.IsNull; import org.elasticsearch.xpack.ql.expression.predicate.operator.comparison.BinaryComparison; import org.elasticsearch.xpack.ql.expression.predicate.operator.comparison.Equals; import org.elasticsearch.xpack.ql.expression.predicate.operator.comparison.NotEquals; -import org.elasticsearch.xpack.ql.expression.predicate.regex.Like; import org.elasticsearch.xpack.ql.optimizer.OptimizerRules.BooleanEqualsSimplification; import org.elasticsearch.xpack.ql.optimizer.OptimizerRules.BooleanLiteralsOnTheRight; import org.elasticsearch.xpack.ql.optimizer.OptimizerRules.BooleanSimplification; @@ -70,7 +65,6 @@ public class Optimizer extends RuleExecutor { @Override protected Iterable.Batch> batches() { Batch substitutions = new Batch("Substitution", Limiter.ONCE, - new ReplaceWildcards(), new ReplaceSurrogateFunction(), new ReplaceRegexMatch()); @@ -108,51 +102,6 @@ public class Optimizer extends RuleExecutor { return Arrays.asList(substitutions, operators, constraints, operators, ordering, local, label); } - private static class ReplaceWildcards extends OptimizerRule { - - @Override - protected LogicalPlan rule(Filter filter) { - return filter.transformExpressionsUp(e -> { - // expr : "wildcard*phrase" || expr !: "wildcard*phrase" - if (e instanceof InsensitiveBinaryComparison) { - InsensitiveBinaryComparison cmp = (InsensitiveBinaryComparison) e; - - Expression target = null; - String wildString = null; - - // check either side since the literals can be on both sides - // "a" : "*" is the same as "*" : "a" - if (isWildcard(cmp.left())) { - wildString = (String) cmp.left().fold(); - target = cmp.right(); - } else if (isWildcard(cmp.right())) { - wildString = (String) cmp.right().fold(); - target = cmp.left(); - } - - if (target != null) { - Expression like = new Like(e.source(), target, StringUtils.toLikePattern(wildString), true); - if (e instanceof InsensitiveNotEquals) { - like = new Not(e.source(), like); - } - - e = like; - } - } - - return e; - }); - } - - private static boolean isWildcard(Expression expr) { - if (expr instanceof Literal) { - Object value = expr.fold(); - return value instanceof String && ((String) value).contains("*"); - } - return false; - } - } - private static class ReplaceNullChecks extends OptimizerRule { @Override diff --git a/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/optimizer/OptimizerTests.java b/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/optimizer/OptimizerTests.java index 556e75a9278..f339cfae141 100644 --- a/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/optimizer/OptimizerTests.java +++ b/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/optimizer/OptimizerTests.java @@ -36,7 +36,6 @@ import org.elasticsearch.xpack.ql.expression.predicate.nulls.IsNull; import org.elasticsearch.xpack.ql.expression.predicate.operator.comparison.Equals; import org.elasticsearch.xpack.ql.expression.predicate.operator.comparison.GreaterThan; import org.elasticsearch.xpack.ql.expression.predicate.operator.comparison.LessThan; -import org.elasticsearch.xpack.ql.expression.predicate.regex.Like; import org.elasticsearch.xpack.ql.index.EsIndex; import org.elasticsearch.xpack.ql.index.IndexResolution; import org.elasticsearch.xpack.ql.plan.TableIdentifier; @@ -131,60 +130,6 @@ public class OptimizerTests extends ESTestCase { } } - public void testEqualsWildcard() { - List tests = Arrays.asList( - "foo where command_line : \"* bar *\"", - "foo where \"* bar *\" : command_line" - ); - - for (String q : tests) { - LogicalPlan plan = defaultPipes(accept(q)); - assertTrue(plan instanceof Filter); - - Filter filter = (Filter) plan; - And condition = (And) filter.condition(); - assertTrue(condition.right() instanceof Like); - - Like like = (Like) condition.right(); - assertEquals(((FieldAttribute) like.field()).name(), "command_line"); - assertEquals(like.pattern().asJavaRegex(), "^.* bar .*$"); - assertEquals(like.pattern().asLuceneWildcard(), "* bar *"); - assertEquals(like.pattern().asIndexNameWildcard(), "* bar *"); - } - } - - // test wildcard gets applied for literals as well regardless of the side used - public void testEqualsWildcardWithLiterals() { - List tests = Arrays.asList( - "foo where \"abc\": \"*b*\"", - "foo where \"*b*\" : \"abc\"" - ); - - for (String q : tests) { - LogicalPlan plan = defaultPipes(accept(q)); - assertTrue(plan instanceof Filter); - // check the optimizer kicked in and folding was applied - Filter filter = (Filter) plan; - Equals condition = (Equals) filter.condition(); - assertEquals("foo", condition.right().fold()); - } - } - - public void testWildcardEscapes() { - LogicalPlan plan = defaultPipes(accept("foo where command_line : \"* %bar_ * \\\\ \\n \\r \\t\"")); - assertTrue(plan instanceof Filter); - - Filter filter = (Filter) plan; - And condition = (And) filter.condition(); - assertTrue(condition.right() instanceof Like); - - Like like = (Like) condition.right(); - assertEquals(((FieldAttribute) like.field()).name(), "command_line"); - assertEquals(like.pattern().asJavaRegex(), "^.* %bar_ .* \\\\ \n \r \t$"); - assertEquals(like.pattern().asLuceneWildcard(), "* %bar_ * \\\\ \n \r \t"); - assertEquals(like.pattern().asIndexNameWildcard(), "* %bar_ * \\ \n \r \t"); - } - public void testCombineHeadBigHeadSmall() { checkOffsetAndLimit(accept("process where true | head 10 | head 1"), 0, 1); } diff --git a/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/planner/QueryTranslationTests.java b/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/planner/QueryTranslationTests.java index 922f647752c..6d816675ad0 100644 --- a/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/planner/QueryTranslationTests.java +++ b/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/planner/QueryTranslationTests.java @@ -17,6 +17,7 @@ public class QueryTranslationTests extends AbstractQueryFolderTestCase { assertThat(asQuery(plan), containsString("\"term\":{\"process_name\"")); } + @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/63262") public void testLikeOptimization() throws Exception { PhysicalPlan plan = plan("process where process_name : \"*\" "); assertThat(asQuery(plan), containsString("\"exists\":{\"field\":\"process_name\"")); diff --git a/x-pack/plugin/eql/src/test/resources/queryfolder_tests.txt b/x-pack/plugin/eql/src/test/resources/queryfolder_tests.txt index 08a52520807..e00e52439ea 100644 --- a/x-pack/plugin/eql/src/test/resources/queryfolder_tests.txt +++ b/x-pack/plugin/eql/src/test/resources/queryfolder_tests.txt @@ -75,12 +75,12 @@ process where process_name in ("python.exe", "SMSS.exe", "explorer.exe") "terms":{"process_name":["python.exe","SMSS.exe","explorer.exe"], ; -equalsAndInFilter -process where process_path : "*\\red_ttp\\wininit.*" and opcode in (0,1,2,3) -; -"wildcard":{"process_path":{"wildcard":"*\\\\red_ttp\\\\wininit.*","case_insensitive":true,"boost":1.0}}}, -{"terms":{"opcode":[0,1,2,3],"boost":1.0}} -; +// equalsAndInFilter +// process where process_path : "*\\red_ttp\\wininit.*" and opcode in (0,1,2,3) +// ; +// "wildcard":{"process_path":{"wildcard":"*\\\\red_ttp\\\\wininit.*","case_insensitive":true,"boost":1.0}}}, +// {"terms":{"opcode":[0,1,2,3],"boost":1.0}} +// ; functionEqualsTrue process where cidrMatch(source_address, "10.0.0.0/8") == true