From d027e24b31fb21b86b7d8eac649a8158b7d0f2c9 Mon Sep 17 00:00:00 2001 From: Costin Leau Date: Mon, 5 Oct 2020 22:51:10 +0300 Subject: [PATCH] EQL: Remove match functions (#63275) Since match (for matching regex) is not currently in use remove it for now. Close #63263 (cherry picked from commit 6abd531cf457f3c5686f59709647bed3276e3c6b) --- .../resources/additional_test_queries.toml | 81 +++++++++---------- .../resources/test_queries_unsupported.toml | 1 - .../function/EqlFunctionRegistry.java | 4 +- .../eql/planner/QueryFolderFailTests.java | 6 +- .../eql/planner/QueryTranslationTests.java | 2 +- .../src/test/resources/queries-supported.eql | 15 ---- .../test/resources/queries-unsupported.eql | 17 ++-- .../src/test/resources/queryfolder_tests.txt | 49 ++++++----- .../test/resources/test_string_functions.toml | 41 ---------- .../src/test/resources/test_unsupported.toml | 40 +++++++++ 10 files changed, 116 insertions(+), 140 deletions(-) diff --git a/x-pack/plugin/eql/qa/common/src/main/resources/additional_test_queries.toml b/x-pack/plugin/eql/qa/common/src/main/resources/additional_test_queries.toml index f3fb75c0996..1f7d6aba5e3 100644 --- a/x-pack/plugin/eql/qa/common/src/main/resources/additional_test_queries.toml +++ b/x-pack/plugin/eql/qa/common/src/main/resources/additional_test_queries.toml @@ -131,48 +131,45 @@ name = "numberStringConversion5" query = 'any where number(string(serial_event_id), 16) == 17' expected_event_ids = [11] - -[[queries]] -name = "matchWithCharacterClasses1" -expected_event_ids = [98] -notes = "regexp doesn't support character classes" -query = ''' -// -// """.*?net1\s+localgroup.*?""") -process where match(command_line, """.*?net1[ ]+localgroup.*?""") -''' - -[[queries]] -name = "matchLiteAdditional" -expected_event_ids = [98] -query = ''' -process where matchLite(command_line, """.*?net1.*?""") -''' - -[[queries]] -name = "matchWithCharacterClasses2" -expected_event_ids = [98] -notes = "regexp doesn't support predefined character classes (like \\s)" -query = ''' -// """.*?net1\s+\w{4,15}\s+.*?""" -process where match(command_line, """.*?net1[ ]+[a-z]{4,15}[ ]+.*?""") -''' - - - -[[queries]] -name = "multiPatternMatch" -expected_event_ids = [50, 97, 98] -query = ''' -process where match(command_line, ".*?net[1]? localgroup.*?", ".*? myappserver.py .*?") -''' - -[[queries]] -name = "matchWithSubstring" -expected_event_ids = [50, 98] -query = ''' -process where match(substring(command_line, 5), ".*?net[1]? localgroup.*?", ".*? myappserver.py .*?") -''' +# [[queries]] +# name = "matchWithCharacterClasses1" +# expected_event_ids = [98] +# notes = "regexp doesn't support character classes" +# query = ''' +# // +# // """.*?net1\s+localgroup.*?""") +# process where match(command_line, """.*?net1[ ]+localgroup.*?""") +# ''' +# +# [[queries]] +# name = "matchLiteAdditional" +# expected_event_ids = [98] +# query = ''' +# process where matchLite(command_line, """.*?net1.*?""") +# ''' +# +# [[queries]] +# name = "matchWithCharacterClasses2" +# expected_event_ids = [98] +# notes = "regexp doesn't support predefined character classes (like \\s)" +# query = ''' +# // """.*?net1\s+\w{4,15}\s+.*?""" +# process where match(command_line, """.*?net1[ ]+[a-z]{4,15}[ ]+.*?""") +# ''' +# +# [[queries]] +# name = "multiPatternMatch" +# expected_event_ids = [50, 97, 98] +# query = ''' +# process where match(command_line, ".*?net[1]? localgroup.*?", ".*? myappserver.py .*?") +# ''' +# +# [[queries]] +# name = "matchWithSubstring" +# expected_event_ids = [50, 98] +# query = ''' +# process where match(substring(command_line, 5), ".*?net[1]? localgroup.*?", ".*? myappserver.py .*?") +# ''' [[queries]] name = "moduloEqualsField" diff --git a/x-pack/plugin/eql/qa/common/src/main/resources/test_queries_unsupported.toml b/x-pack/plugin/eql/qa/common/src/main/resources/test_queries_unsupported.toml index 72a2914d515..b450cad34b5 100644 --- a/x-pack/plugin/eql/qa/common/src/main/resources/test_queries_unsupported.toml +++ b/x-pack/plugin/eql/qa/common/src/main/resources/test_queries_unsupported.toml @@ -1149,4 +1149,3 @@ process where length(between(process_name, "g", "e")) > 0 #query = ''' #process where length(between(process_name, "g", "e")) > 0 #''' - diff --git a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/expression/function/EqlFunctionRegistry.java b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/expression/function/EqlFunctionRegistry.java index bf90766758e..04c9f732136 100644 --- a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/expression/function/EqlFunctionRegistry.java +++ b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/expression/function/EqlFunctionRegistry.java @@ -6,17 +6,16 @@ package org.elasticsearch.xpack.eql.expression.function; +import org.elasticsearch.xpack.eql.expression.function.scalar.math.ToNumber; import org.elasticsearch.xpack.eql.expression.function.scalar.string.Between; import org.elasticsearch.xpack.eql.expression.function.scalar.string.CIDRMatch; import org.elasticsearch.xpack.eql.expression.function.scalar.string.Concat; import org.elasticsearch.xpack.eql.expression.function.scalar.string.EndsWith; import org.elasticsearch.xpack.eql.expression.function.scalar.string.IndexOf; import org.elasticsearch.xpack.eql.expression.function.scalar.string.Length; -import org.elasticsearch.xpack.eql.expression.function.scalar.string.Match; import org.elasticsearch.xpack.eql.expression.function.scalar.string.StartsWith; import org.elasticsearch.xpack.eql.expression.function.scalar.string.StringContains; import org.elasticsearch.xpack.eql.expression.function.scalar.string.Substring; -import org.elasticsearch.xpack.eql.expression.function.scalar.math.ToNumber; import org.elasticsearch.xpack.eql.expression.function.scalar.string.ToString; import org.elasticsearch.xpack.eql.expression.function.scalar.string.Wildcard; import org.elasticsearch.xpack.ql.expression.function.FunctionDefinition; @@ -46,7 +45,6 @@ public class EqlFunctionRegistry extends FunctionRegistry { def(EndsWith.class, EndsWith::new, "endswith"), def(IndexOf.class, IndexOf::new, "indexof"), def(Length.class, Length::new, "length"), - def(Match.class, Match::new, "match", "matchlite"), def(StartsWith.class, StartsWith::new, "startswith"), def(ToString.class, ToString::new, "string"), def(StringContains.class, StringContains::new, "stringcontains"), diff --git a/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/planner/QueryFolderFailTests.java b/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/planner/QueryFolderFailTests.java index 7e53a07c0dc..f3e0bf4e8bd 100644 --- a/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/planner/QueryFolderFailTests.java +++ b/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/planner/QueryFolderFailTests.java @@ -126,7 +126,7 @@ public class QueryFolderFailTests extends AbstractQueryFolderTestCase { assertEquals("Found 1 problem\nline 1:15: [length(plain_text)] cannot operate on field of data type [text]: No keyword/multi-field " + "defined exact matches for [plain_text]; define one or use MATCH/QUERY instead", msg); } - + @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/63263") public void testMatchWithText() { VerificationException e = expectThrows(VerificationException.class, () -> plan("process where match(plain_text, \"foo.*\")")); @@ -135,7 +135,7 @@ public class QueryFolderFailTests extends AbstractQueryFolderTestCase { "line 1:15: [match(plain_text, \"foo.*\")] cannot operate on first argument field of data type [text]: " + "No keyword/multi-field defined exact matches for [plain_text]; define one or use MATCH/QUERY instead", msg); } - + @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/63263") public void testMatchWithNonString() { VerificationException e = expectThrows(VerificationException.class, () -> plan("process where match(process_name, parent_process_name)")); @@ -144,7 +144,7 @@ public class QueryFolderFailTests extends AbstractQueryFolderTestCase { "line 1:15: second argument of [match(process_name, parent_process_name)] " + "must be a constant, received [parent_process_name]", msg); } - + @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/63263") public void testMatchWithNonRegex() { VerificationException e = expectThrows(VerificationException.class, () -> plan("process where match(process_name, 1)")); diff --git a/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/planner/QueryTranslationTests.java b/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/planner/QueryTranslationTests.java index 6d816675ad0..5f82691b40e 100644 --- a/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/planner/QueryTranslationTests.java +++ b/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/planner/QueryTranslationTests.java @@ -22,7 +22,7 @@ public class QueryTranslationTests extends AbstractQueryFolderTestCase { PhysicalPlan plan = plan("process where process_name : \"*\" "); assertThat(asQuery(plan), containsString("\"exists\":{\"field\":\"process_name\"")); } - + @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/63263") public void testMatchOptimization() throws Exception { PhysicalPlan plan = plan("process where match(process_name, \".*\") "); assertThat(asQuery(plan), containsString("\"exists\":{\"field\":\"process_name\"")); diff --git a/x-pack/plugin/eql/src/test/resources/queries-supported.eql b/x-pack/plugin/eql/src/test/resources/queries-supported.eql index 684372755a2..2fac393e3bf 100644 --- a/x-pack/plugin/eql/src/test/resources/queries-supported.eql +++ b/x-pack/plugin/eql/src/test/resources/queries-supported.eql @@ -182,21 +182,6 @@ process where command_line : "*%*%*" ; process where command_line : "%*%*" ; -process where match(""".*?net1\s+localgroup\s+.*?""", command_line) -; - -process where match(""".*?net1\s+\w+\s+.*?""", command_line) -; - -process where match(""".*?net1\s+\w{4,15}\s+.*?""", command_line) -; - -process where match(""".*?net1\s+\w{4,15}\s+.*?""", command_line) -; - -process where match(""".*?net1\s+[localgrup]{4,15}\s+.*?""", command_line) -; - file where opcode:0 and startsWith(file_name, "exploRER.") ; diff --git a/x-pack/plugin/eql/src/test/resources/queries-unsupported.eql b/x-pack/plugin/eql/src/test/resources/queries-unsupported.eql index e7fe5828912..544be6d59f0 100644 --- a/x-pack/plugin/eql/src/test/resources/queries-unsupported.eql +++ b/x-pack/plugin/eql/src/test/resources/queries-unsupported.eql @@ -1,4 +1,3 @@ - // // Pipes // @@ -96,23 +95,23 @@ file where descendant of [registry where true]; //sequence by unique_pid [process where true] [file where true] fork==true; -sequence with maxspan=2.5m - [process where x == x] by pid +sequence with maxspan=2.5m + [process where x == x] by pid [file where file_path == "*"] by ppid ; -sequence by pid with maxspan=2.0h - [process where process_name == "*"] +sequence by pid with maxspan=2.0h + [process where process_name == "*"] [file where file_path == "*"] ; -sequence by pid with maxspan=2.0h - [process where process_name == "*"] +sequence by pid with maxspan=2.0h + [process where process_name == "*"] [file where file_path == "*"] ; -sequence by pid with maxspan=1.0075d - [process where process_name == "*"] +sequence by pid with maxspan=1.0075d + [process where process_name == "*"] [file where file_path == "*"] ; diff --git a/x-pack/plugin/eql/src/test/resources/queryfolder_tests.txt b/x-pack/plugin/eql/src/test/resources/queryfolder_tests.txt index e00e52439ea..949759a67a1 100644 --- a/x-pack/plugin/eql/src/test/resources/queryfolder_tests.txt +++ b/x-pack/plugin/eql/src/test/resources/queryfolder_tests.txt @@ -304,23 +304,31 @@ InternalEqlScriptUtils.cidrMatch(InternalQlScriptUtils.docValue(doc,params.v0),p "params":{"v0":"source_address","v1":["10.6.48.157/8"],"v2":"true"} ; -matchFunctionOne -process where match(command_line, "^.*?net.exe") -; -"regexp":{"command_line":{"value":"^.*?net.exe" -; +//matchFunctionOne +//process where match(command_line, "^.*?net.exe") +//; +//"regexp":{"command_line":{"value":"^.*?net.exe" +//; -matchFunctionTwo -process where match(command_line, "^.*?net.exe", "net\\.exe") -; -"regexp":{"command_line":{"value":"^.*?net.exe|net\\.exe" -; - -matchFunctionThree -process where match(command_line, "^.*?net.exe", "net\\.exe", "C:\\\\Windows\\\\system32\\\\net1\\s+") -; -"regexp":{"command_line":{"value":"^.*?net.exe|net\\.exe|C:\\\\Windows\\\\system32\\\\net1\\s+" -; +//matchFunctionTwo +//process where match(command_line, "^.*?net.exe", "net\\.exe") +//; +//"regexp":{"command_line":{"value":"^.*?net.exe|net\\.exe" +//; +// +//matchFunctionThree +//process where match(command_line, "^.*?net.exe", "net\\.exe", "C:\\\\Windows\\\\system32\\\\net1\\s+") +//; +//"regexp":{"command_line":{"value":"^.*?net.exe|net\\.exe|C:\\\\Windows\\\\system32\\\\net1\\s+" +//; +// +//matchFunctionScalar +//process where match(substring(command_line, 5), "^.*?net.exe", "net\\.exe", "C:\\\\Windows\\\\system32\\\\net1\\s+") +//; +//"script":{"source":"InternalQlScriptUtils.nullSafeFilter(InternalSqlScriptUtils.regex(InternalEqlScriptUtils.substring( +//InternalQlScriptUtils.docValue(doc,params.v0),params.v1,params.v2),params.v3))", +//"params":{"v0":"command_line","v1":5,"v2":null,"v3":"^.*?net.exe|net\\.exe|C:\\\\Windows\\\\system32\\\\net1\\s+"}} +//; numberFunctionSingleArgument process where number(process_name) == 1; @@ -328,15 +336,6 @@ InternalEqlScriptUtils.number(InternalQlScriptUtils.docValue(doc,params.v0),para "params":{"v0":"process_name","v1":null,"v2":1} ; -matchFunctionScalar -process where match(substring(command_line, 5), "^.*?net.exe", "net\\.exe", "C:\\\\Windows\\\\system32\\\\net1\\s+") -; -"script":{"source":"InternalQlScriptUtils.nullSafeFilter(InternalSqlScriptUtils.regex(InternalEqlScriptUtils.substring( -InternalQlScriptUtils.docValue(doc,params.v0),params.v1,params.v2),params.v3))", -"params":{"v0":"command_line","v1":5,"v2":null,"v3":"^.*?net.exe|net\\.exe|C:\\\\Windows\\\\system32\\\\net1\\s+"}} -; - - numberFunctionTwoFieldArguments process where number(process_name, pid) != null; InternalEqlScriptUtils.number(InternalQlScriptUtils.docValue(doc,params.v0),InternalQlScriptUtils.docValue(doc,params.v1))))", diff --git a/x-pack/plugin/eql/src/test/resources/test_string_functions.toml b/x-pack/plugin/eql/src/test/resources/test_string_functions.toml index 72cd4ebfd87..9624ec6b8c9 100644 --- a/x-pack/plugin/eql/src/test/resources/test_string_functions.toml +++ b/x-pack/plugin/eql/src/test/resources/test_string_functions.toml @@ -182,47 +182,6 @@ description = "Test the folding of the `length` function." expected = 3 -[match] -description = "Test the `match` function" - - [match.verifier] - - [[match.verifier.failures]] - expression = 'match(1, "*")' - - [[match.verifier.failures]] - expression = 'match(1, "*")' - - [[match.verifier.failures]] - expression = 'match("eql", 1)' - - [match.fold] - - [[match.fold.tests]] - expression = 'match(null, "[a-z]{3}")' - # expected = null - - [[match.fold.tests]] - expression = 'match("foo", "[a-z]{3}")' - expected = true - - [[match.fold.tests]] - expression = 'match("foo\nbarbaz", "[a-z]{3}\n[a-z]{6}")' - expected = true - - [[match.fold.tests]] - expression = 'match("999", "[a-z]{3}")' - expected = false - - [[match.fold.tests]] - expression = 'match("999", "[a-z]{3}", "[0-9]{5}")' - expected = false - - [[match.fold.tests]] - expression = 'match("999", "[a-z]{3}", "[0-9]{5}", "[9][9][9]")' - expected = true - - [number] description = "Test the `number` function" diff --git a/x-pack/plugin/eql/src/test/resources/test_unsupported.toml b/x-pack/plugin/eql/src/test/resources/test_unsupported.toml index 27972065c47..2707b126630 100644 --- a/x-pack/plugin/eql/src/test/resources/test_unsupported.toml +++ b/x-pack/plugin/eql/src/test/resources/test_unsupported.toml @@ -132,3 +132,43 @@ case_insensitive = true [[substring.fold.tests]] expression = '''substring("hello world", null, 5)''' expected = "hello" + +[match] +description = "Test the `match` function" + + [match.verifier] + + [[match.verifier.failures]] + expression = 'match(1, "*")' + + [[match.verifier.failures]] + expression = 'match(1, "*")' + + [[match.verifier.failures]] + expression = 'match("eql", 1)' + + [match.fold] + + [[match.fold.tests]] + expression = 'match(null, "[a-z]{3}")' + # expected = null + + [[match.fold.tests]] + expression = 'match("foo", "[a-z]{3}")' + expected = true + + [[match.fold.tests]] + expression = 'match("foo\nbarbaz", "[a-z]{3}\n[a-z]{6}")' + expected = true + + [[match.fold.tests]] + expression = 'match("999", "[a-z]{3}")' + expected = false + + [[match.fold.tests]] + expression = 'match("999", "[a-z]{3}", "[0-9]{5}")' + expected = false + + [[match.fold.tests]] + expression = 'match("999", "[a-z]{3}", "[0-9]{5}", "[9][9][9]")' + expected = true