EQL: Remove wildcard functionality from : (#63276)

Restrict : operator to only case insensitive matching on strings

Close #63262

(cherry picked from commit bc02e77150cdd85594dfac4f03d8aeb85aaddbb3)

x-pack/plugin/eql/qa/common/src/main/resources/test_queries.toml

@@ -342,19 +342,19 @@ query = '''
 registry where length(bytes_written_string_list) == 2 and bytes_written_string_list[1] : "EN"
 '''
 
-[[queries]]
+# [[queries]]
-name = "keyPathWildcard"
+# name = "keyPathWildcard"
-query = '''
+# query = '''
-registry where key_path : "*\\MACHINE\\SAM\\SAM\\*\\Account\\Us*ers\\00*03E9\\F"
+# registry where key_path : "*\\MACHINE\\SAM\\SAM\\*\\Account\\Us*ers\\00*03E9\\F"
-'''
+# '''
-expected_event_ids  = [79]
+# expected_event_ids  = [79]
-
+#
-[[queries]]
+# [[queries]]
-name = "processPathWildcardAndIN"
+# name = "processPathWildcardAndIN"
-query = '''
+# query = '''
-process where process_path : "*\\red_ttp\\wininit.*" and opcode in (0,1,2,3,4)
+# process where process_path : "*\\red_ttp\\wininit.*" and opcode in (0,1,2,3,4)
-'''
+# '''
-expected_event_ids  = [84, 85]
+# expected_event_ids  = [84, 85]
 
 [[queries]]
 name = "descendant1"
@@ -385,13 +385,13 @@ process where opcode==1 and process_name : "smss.exe"
 '''
 expected_event_ids  = [78]
 
-[[queries]]
+# [[queries]]
-name = "wildcardAndMultipleConditions1"
+# name = "wildcardAndMultipleConditions1"
-query = '''
+# query = '''
-file where file_path:"*\\red_ttp\\winin*.*"
+# file where file_path:"*\\red_ttp\\winin*.*"
-  and opcode in (0,1,2) and user_name:"vagrant"
+#   and opcode in (0,1,2) and user_name:"vagrant"
-'''
+# '''
-expected_event_ids  = [83, 86]
+# expected_event_ids  = [83, 86]
 
 [[queries]]
 name = "wildcardAndMultipleConditions2"
@@ -401,13 +401,13 @@ file where file_path:"*\\red_ttp\\winin*.*"
 '''
 expected_event_ids  = []
 
-[[queries]]
+# [[queries]]
-name = "wildcardAndMultipleConditions3"
+# name = "wildcardAndMultipleConditions3"
-query = '''
+# query = '''
-file where file_path:"*\\red_ttp\\winin*.*"
+# file where file_path:"*\\red_ttp\\winin*.*"
-  and opcode not in (3, 4, 5, 6 ,7) and user_name:"vagrant"
+#   and opcode not in (3, 4, 5, 6 ,7) and user_name:"vagrant"
-'''
+# '''
-expected_event_ids  = [83, 86]
+# expected_event_ids  = [83, 86]
 
 
 [[queries]]
@@ -878,16 +878,16 @@ sequence
 '''
 expected_event_ids  = [87, 92]
 
-[[queries]]
+# [[queries]]
-name = "doubleSameSequenceWithByUntilAndHead1"
+# name = "doubleSameSequenceWithByUntilAndHead1"
-query = '''
+# query = '''
-sequence
+# sequence
-  [file where opcode==0 and file_name:"*.exe"] by unique_pid
+#   [file where opcode==0 and file_name:"*.exe"] by unique_pid
-  [file where opcode==0 and file_name:"*.exe"] by unique_pid
+#   [file where opcode==0 and file_name:"*.exe"] by unique_pid
-until [process where opcode==5000] by unique_ppid
+# until [process where opcode==5000] by unique_ppid
-| head 1
+# | head 1
-'''
+# '''
-expected_event_ids  = [55, 61]
+# expected_event_ids  = [55, 61]
 
 [[queries]]
 name = "doubleSameSequenceWithByUntilAndHead2"
@@ -1040,16 +1040,16 @@ query = '''
 registry where length(bad_field) > 0
 '''
 
-[[queries]]
+# [[queries]]
-name = "multipleConditions2"
+# name = "multipleConditions2"
-query = '''
+# query = '''
-process where opcode == 1
+# process where opcode == 1
-  and process_name in ("net.exe", "net1.exe")
+#   and process_name in ("net.exe", "net1.exe")
-  and not (parent_process_name : "net.exe"
+#   and not (parent_process_name : "net.exe"
-  and process_name : "net1.exe")
+#   and process_name : "net1.exe")
-  and command_line : "*group *admin*" and command_line != "* /add*"
+#   and command_line : "*group *admin*" and command_line != "* /add*"
-  '''
+#   '''
-expected_event_ids  = [97]
+# expected_event_ids  = [97]
 
 [[queries]]
 name = "anyWithUnique"
@@ -1246,26 +1246,26 @@ sequence
 '''
 expected_event_ids  = [54, 55, 56, 54, 61, 62, 54, 67, 68, 54, 72, 73]
 
-[[queries]]
+# [[queries]]
-name = "wildcard1"
+# name = "wildcard1"
-query = '''
+# query = '''
-process where command_line : "*%*"
+# process where command_line : "*%*"
-'''
+# '''
-expected_event_ids  = [4, 6, 28]
+# expected_event_ids  = [4, 6, 28]
-
+#
-[[queries]]
+# [[queries]]
-name = "wildcard2"
+# name = "wildcard2"
-query = '''
+# query = '''
-process where command_line : "*%*%*"
+# process where command_line : "*%*%*"
-'''
+# '''
-expected_event_ids  = [4, 6, 28]
+# expected_event_ids  = [4, 6, 28]
-
+#
-[[queries]]
+# [[queries]]
-name = "wildcard3"
+# name = "wildcard3"
-query = '''
+# query = '''
-process where command_line : "%*%*"
+# process where command_line : "%*%*"
-'''
+# '''
-expected_event_ids  = [4, 6, 28]
+# expected_event_ids  = [4, 6, 28]
 
 [[queries]]
 name = "uniqueCount1"

x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/optimizer/Optimizer.java

@@ -7,15 +7,12 @@
 package org.elasticsearch.xpack.eql.optimizer;
 
 import org.elasticsearch.xpack.eql.EqlIllegalArgumentException;
-import org.elasticsearch.xpack.eql.expression.predicate.operator.comparison.InsensitiveBinaryComparison;
-import org.elasticsearch.xpack.eql.expression.predicate.operator.comparison.InsensitiveNotEquals;
 import org.elasticsearch.xpack.eql.plan.logical.Join;
 import org.elasticsearch.xpack.eql.plan.logical.KeyedFilter;
 import org.elasticsearch.xpack.eql.plan.logical.LimitWithOffset;
 import org.elasticsearch.xpack.eql.plan.physical.LocalRelation;
 import org.elasticsearch.xpack.eql.session.Payload.Type;
 import org.elasticsearch.xpack.eql.util.MathUtils;
-import org.elasticsearch.xpack.eql.util.StringUtils;
 import org.elasticsearch.xpack.ql.expression.Expression;
 import org.elasticsearch.xpack.ql.expression.Literal;
 import org.elasticsearch.xpack.ql.expression.NamedExpression;
@@ -24,14 +21,12 @@ import org.elasticsearch.xpack.ql.expression.Order.NullsPosition;
 import org.elasticsearch.xpack.ql.expression.Order.OrderDirection;
 import org.elasticsearch.xpack.ql.expression.predicate.Predicates;
 import org.elasticsearch.xpack.ql.expression.predicate.logical.And;
-import org.elasticsearch.xpack.ql.expression.predicate.logical.Not;
 import org.elasticsearch.xpack.ql.expression.predicate.logical.Or;
 import org.elasticsearch.xpack.ql.expression.predicate.nulls.IsNotNull;
 import org.elasticsearch.xpack.ql.expression.predicate.nulls.IsNull;
 import org.elasticsearch.xpack.ql.expression.predicate.operator.comparison.BinaryComparison;
 import org.elasticsearch.xpack.ql.expression.predicate.operator.comparison.Equals;
 import org.elasticsearch.xpack.ql.expression.predicate.operator.comparison.NotEquals;
-import org.elasticsearch.xpack.ql.expression.predicate.regex.Like;
 import org.elasticsearch.xpack.ql.optimizer.OptimizerRules.BooleanEqualsSimplification;
 import org.elasticsearch.xpack.ql.optimizer.OptimizerRules.BooleanLiteralsOnTheRight;
 import org.elasticsearch.xpack.ql.optimizer.OptimizerRules.BooleanSimplification;
@@ -70,7 +65,6 @@ public class Optimizer extends RuleExecutor<LogicalPlan> {
     @Override
     protected Iterable<RuleExecutor<LogicalPlan>.Batch> batches() {
         Batch substitutions = new Batch("Substitution", Limiter.ONCE,
-                new ReplaceWildcards(),
                 new ReplaceSurrogateFunction(),
                 new ReplaceRegexMatch());
 
@@ -108,51 +102,6 @@ public class Optimizer extends RuleExecutor<LogicalPlan> {
         return Arrays.asList(substitutions, operators, constraints, operators, ordering, local, label);
     }
 
-    private static class ReplaceWildcards extends OptimizerRule<Filter> {
-
-        @Override
-        protected LogicalPlan rule(Filter filter) {
-            return filter.transformExpressionsUp(e -> {
-                // expr : "wildcard*phrase" || expr !: "wildcard*phrase"
-                if (e instanceof InsensitiveBinaryComparison) {
-                    InsensitiveBinaryComparison cmp = (InsensitiveBinaryComparison) e;
-
-                    Expression target = null;
-                    String wildString = null;
-
-                    // check either side since the literals can be on both sides
-                    // "a" : "*" is the same as "*" : "a"
-                    if (isWildcard(cmp.left())) {
-                        wildString = (String) cmp.left().fold();
-                        target = cmp.right();
-                    } else if (isWildcard(cmp.right())) {
-                        wildString = (String) cmp.right().fold();
-                        target = cmp.left();
-                    }
-
-                    if (target != null) {
-                        Expression like = new Like(e.source(), target, StringUtils.toLikePattern(wildString), true);
-                        if (e instanceof InsensitiveNotEquals) {
-                            like = new Not(e.source(), like);
-                        }
-
-                        e = like;
-                    }
-                }
-
-                return e;
-            });
-        }
-
-        private static boolean isWildcard(Expression expr) {
-            if (expr instanceof Literal) {
-                Object value = expr.fold();
-                return value instanceof String && ((String) value).contains("*");
-            }
-            return false;
-        }
-    }
-
     private static class ReplaceNullChecks extends OptimizerRule<Filter> {
 
         @Override

x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/optimizer/OptimizerTests.java

@@ -36,7 +36,6 @@ import org.elasticsearch.xpack.ql.expression.predicate.nulls.IsNull;
 import org.elasticsearch.xpack.ql.expression.predicate.operator.comparison.Equals;
 import org.elasticsearch.xpack.ql.expression.predicate.operator.comparison.GreaterThan;
 import org.elasticsearch.xpack.ql.expression.predicate.operator.comparison.LessThan;
-import org.elasticsearch.xpack.ql.expression.predicate.regex.Like;
 import org.elasticsearch.xpack.ql.index.EsIndex;
 import org.elasticsearch.xpack.ql.index.IndexResolution;
 import org.elasticsearch.xpack.ql.plan.TableIdentifier;
@@ -131,60 +130,6 @@ public class OptimizerTests extends ESTestCase {
         }
     }
 
-    public void testEqualsWildcard() {
-        List<String> tests = Arrays.asList(
-            "foo where command_line : \"* bar *\"",
-            "foo where \"* bar *\" : command_line"
-        );
-
-        for (String q : tests) {
-            LogicalPlan plan = defaultPipes(accept(q));
-            assertTrue(plan instanceof Filter);
-
-            Filter filter = (Filter) plan;
-            And condition = (And) filter.condition();
-            assertTrue(condition.right() instanceof Like);
-
-            Like like = (Like) condition.right();
-            assertEquals(((FieldAttribute) like.field()).name(), "command_line");
-            assertEquals(like.pattern().asJavaRegex(), "^.* bar .*$");
-            assertEquals(like.pattern().asLuceneWildcard(), "* bar *");
-            assertEquals(like.pattern().asIndexNameWildcard(), "* bar *");
-        }
-    }
-
-    // test wildcard gets applied for literals as well regardless of the side used
-    public void testEqualsWildcardWithLiterals() {
-        List<String> tests = Arrays.asList(
-            "foo where \"abc\": \"*b*\"",
-            "foo where \"*b*\" : \"abc\""
-        );
-
-        for (String q : tests) {
-            LogicalPlan plan = defaultPipes(accept(q));
-            assertTrue(plan instanceof Filter);
-            // check the optimizer kicked in and folding was applied
-            Filter filter = (Filter) plan;
-            Equals condition = (Equals) filter.condition();
-            assertEquals("foo", condition.right().fold());
-        }
-    }
-
-    public void testWildcardEscapes() {
-        LogicalPlan plan = defaultPipes(accept("foo where command_line : \"* %bar_ * \\\\ \\n \\r \\t\""));
-        assertTrue(plan instanceof Filter);
-
-        Filter filter = (Filter) plan;
-        And condition = (And) filter.condition();
-        assertTrue(condition.right() instanceof Like);
-
-        Like like = (Like) condition.right();
-        assertEquals(((FieldAttribute) like.field()).name(), "command_line");
-        assertEquals(like.pattern().asJavaRegex(), "^.* %bar_ .* \\\\ \n \r \t$");
-        assertEquals(like.pattern().asLuceneWildcard(), "* %bar_ * \\\\ \n \r \t");
-        assertEquals(like.pattern().asIndexNameWildcard(), "* %bar_ * \\ \n \r \t");
-    }
-
     public void testCombineHeadBigHeadSmall() {
         checkOffsetAndLimit(accept("process where true | head 10 | head 1"), 0, 1);
     }

x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/planner/QueryTranslationTests.java

@@ -17,6 +17,7 @@ public class QueryTranslationTests extends AbstractQueryFolderTestCase {
         assertThat(asQuery(plan), containsString("\"term\":{\"process_name\""));
     }
 
+    @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/63262")
     public void testLikeOptimization() throws Exception {
         PhysicalPlan plan = plan("process where process_name : \"*\" ");
         assertThat(asQuery(plan), containsString("\"exists\":{\"field\":\"process_name\""));

x-pack/plugin/eql/src/test/resources/queryfolder_tests.txt

@@ -75,12 +75,12 @@ process where process_name in ("python.exe", "SMSS.exe", "explorer.exe")
 "terms":{"process_name":["python.exe","SMSS.exe","explorer.exe"],
 ;
 
-equalsAndInFilter
+// equalsAndInFilter
-process where process_path : "*\\red_ttp\\wininit.*" and opcode in (0,1,2,3)
+// process where process_path : "*\\red_ttp\\wininit.*" and opcode in (0,1,2,3)
-;
+// ;
-"wildcard":{"process_path":{"wildcard":"*\\\\red_ttp\\\\wininit.*","case_insensitive":true,"boost":1.0}}},
+// "wildcard":{"process_path":{"wildcard":"*\\\\red_ttp\\\\wininit.*","case_insensitive":true,"boost":1.0}}},
-{"terms":{"opcode":[0,1,2,3],"boost":1.0}}
+// {"terms":{"opcode":[0,1,2,3],"boost":1.0}}
-;
+// ;
 
 functionEqualsTrue
 process where cidrMatch(source_address, "10.0.0.0/8") == true