From 76bba601aba508535775c78f199d853ab4e5565f Mon Sep 17 00:00:00 2001 From: Andrei Stefan Date: Mon, 5 Oct 2020 22:04:42 +0300 Subject: [PATCH] Remove case_sensitive request option (#63218) (#63244) Make EQL case sensitive by default and adapt some of the string functions Remove the case sensitive option from Between string function Add case_insensitive option to term and wildcard queries usage (cherry picked from commit 7550e0664c8c2f1f13519036c759b1e76345551f) --- .../client/eql/EqlSearchRequest.java | 14 -- docs/reference/eql/eql.asciidoc | 28 ---- .../test/eql/BaseEqlSpecTestCase.java | 16 +-- .../test/eql/EqlExtraSpecTestCase.java | 4 +- .../org/elasticsearch/test/eql/EqlSpec.java | 24 +--- .../elasticsearch/test/eql/EqlSpecLoader.java | 14 -- .../test/eql/EqlSpecTestCase.java | 4 +- .../src/main/resources/test_queries.toml | 59 -------- .../resources/test_queries_unsupported.toml | 129 +++++++++++++----- .../elasticsearch/xpack/eql/EqlExtraIT.java | 4 +- .../elasticsearch/xpack/eql/EqlSpecIT.java | 4 +- .../xpack/eql/action/EqlSearchRequest.java | 26 ++-- .../eql/action/EqlSearchRequestBuilder.java | 5 - .../function/EqlFunctionRegistry.java | 2 +- .../function/scalar/string/Between.java | 28 ++-- .../function/scalar/string/EndsWith.java | 3 +- .../function/scalar/string/IndexOf.java | 3 +- .../function/scalar/string/StartsWith.java | 3 +- .../scalar/string/StringContains.java | 2 +- .../xpack/eql/optimizer/Optimizer.java | 2 +- .../xpack/eql/planner/QueryTranslator.java | 2 +- .../eql/plugin/TransportEqlSearchAction.java | 2 +- .../xpack/eql/session/EqlConfiguration.java | 8 +- .../elasticsearch/xpack/eql/EqlTestUtils.java | 17 +-- .../eql/action/EqlRequestParserTests.java | 10 +- .../xpack/eql/analysis/VerifierTests.java | 2 +- .../string/BetweenFunctionPipeTests.java | 3 +- .../EndsWithFunctionProcessorTests.java | 20 +-- .../string/IndexOfFunctionProcessorTests.java | 24 ++-- .../StartsWithFunctionProcessorTests.java | 6 +- .../xpack/eql/optimizer/EqlFoldSpec.java | 18 +-- .../eql/optimizer/EqlFoldSpecLoader.java | 16 +-- .../xpack/eql/optimizer/OptimizerTests.java | 6 +- .../xpack/eql/optimizer/TomlFoldTests.java | 30 +--- .../eql/planner/QueryFolderFailTests.java | 8 +- .../xpack/eql/planner/QueryFolderOkTests.java | 9 -- .../src/test/resources/queries-supported.eql | 4 + .../src/test/resources/queryfolder_tests.txt | 78 +++++------ .../eql/src/test/resources/test_folding.toml | 1 - .../test/resources/test_string_functions.toml | 14 +- .../expression/function/FunctionRegistry.java | 7 +- .../ql/expression/predicate/regex/Like.java | 28 +++- .../ql/planner/ExpressionTranslators.java | 5 +- .../xpack/ql/querydsl/query/TermQuery.java | 20 ++- .../ql/querydsl/query/WildcardQuery.java | 20 ++- 45 files changed, 298 insertions(+), 434 deletions(-) diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/eql/EqlSearchRequest.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/eql/EqlSearchRequest.java index 8cdc935013d..81fbcfedd9a 100644 --- a/client/rest-high-level/src/main/java/org/elasticsearch/client/eql/EqlSearchRequest.java +++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/eql/EqlSearchRequest.java @@ -39,7 +39,6 @@ public class EqlSearchRequest implements Validatable, ToXContentObject { private QueryBuilder filter = null; private String timestampField = "@timestamp"; private String eventCategoryField = "event.category"; - private boolean isCaseSensitive = true; private int size = 10; private int fetchSize = 1000; @@ -55,7 +54,6 @@ public class EqlSearchRequest implements Validatable, ToXContentObject { static final String KEY_TIMESTAMP_FIELD = "timestamp_field"; static final String KEY_TIEBREAKER_FIELD = "tiebreaker_field"; static final String KEY_EVENT_CATEGORY_FIELD = "event_category_field"; - static final String KEY_CASE_SENSITIVE = "case_sensitive"; static final String KEY_SIZE = "size"; static final String KEY_FETCH_SIZE = "fetch_size"; static final String KEY_QUERY = "query"; @@ -81,7 +79,6 @@ public class EqlSearchRequest implements Validatable, ToXContentObject { builder.field(KEY_EVENT_CATEGORY_FIELD, eventCategoryField()); builder.field(KEY_SIZE, size()); builder.field(KEY_FETCH_SIZE, fetchSize()); - builder.field(KEY_CASE_SENSITIVE, isCaseSensitive()); builder.field(KEY_QUERY, query); if (waitForCompletionTimeout != null) { @@ -143,15 +140,6 @@ public class EqlSearchRequest implements Validatable, ToXContentObject { return this; } - public boolean isCaseSensitive() { - return this.isCaseSensitive; - } - - public EqlSearchRequest isCaseSensitive(boolean isCaseSensitive) { - this.isCaseSensitive = isCaseSensitive; - return this; - } - public int size() { return this.size; } @@ -232,7 +220,6 @@ public class EqlSearchRequest implements Validatable, ToXContentObject { Objects.equals(tiebreakerField, that.tiebreakerField) && Objects.equals(eventCategoryField, that.eventCategoryField) && Objects.equals(query, that.query) && - Objects.equals(isCaseSensitive, that.isCaseSensitive) && Objects.equals(waitForCompletionTimeout, that.waitForCompletionTimeout) && Objects.equals(keepAlive, that.keepAlive) && Objects.equals(keepOnCompletion, that.keepOnCompletion); @@ -250,7 +237,6 @@ public class EqlSearchRequest implements Validatable, ToXContentObject { tiebreakerField, eventCategoryField, query, - isCaseSensitive, waitForCompletionTimeout, keepAlive, keepOnCompletion); diff --git a/docs/reference/eql/eql.asciidoc b/docs/reference/eql/eql.asciidoc index 826e709329d..6c7227877a9 100644 --- a/docs/reference/eql/eql.asciidoc +++ b/docs/reference/eql/eql.asciidoc @@ -486,34 +486,6 @@ GET /my-index-000001/_eql/search ---- // TEST[setup:sec_logs] -[discrete] -[[eql-search-case-sensitive]] -=== Run a case-sensitive EQL search - -By default, matching for EQL queries is case-insensitive. You can use the -`case_sensitive` parameter to toggle case sensitivity on or off. - -The following search request contains a query that matches `process` events -with a `process.executable` containing `System32`. - -Because `case_sensitive` is `true`, this query only matches `process.executable` -values containing `System32` with the exact same capitalization. A -`process.executable` value containing `system32` or `SYSTEM32` would not match -this query. - -[source,console] ----- -GET /my-index-000001/_eql/search -{ - "keep_on_completion": true, - "case_sensitive": true, - "query": """ - process where stringContains(process.executable, "System32") - """ -} ----- -// TEST[setup:sec_logs] - [discrete] [[eql-search-async]] === Run an async EQL search diff --git a/x-pack/plugin/eql/qa/common/src/main/java/org/elasticsearch/test/eql/BaseEqlSpecTestCase.java b/x-pack/plugin/eql/qa/common/src/main/java/org/elasticsearch/test/eql/BaseEqlSpecTestCase.java index 798eb868659..64b7c6792dc 100644 --- a/x-pack/plugin/eql/qa/common/src/main/java/org/elasticsearch/test/eql/BaseEqlSpecTestCase.java +++ b/x-pack/plugin/eql/qa/common/src/main/java/org/elasticsearch/test/eql/BaseEqlSpecTestCase.java @@ -40,7 +40,6 @@ public abstract class BaseEqlSpecTestCase extends ESRestTestCase { private final String query; private final String name; private final long[] eventIds; - private final boolean caseSensitive; @Before private void setup() throws Exception { @@ -74,28 +73,22 @@ public abstract class BaseEqlSpecTestCase extends ESRestTestCase { name = "" + (counter); } - boolean[] values = spec.caseSensitive() == null ? new boolean[] { true, false } : new boolean[] { spec.caseSensitive() }; - - for (boolean sensitive : values) { - String prefixed = name + (sensitive ? "-sensitive" : "-insensitive"); - results.add(new Object[] { spec.query(), prefixed, spec.expectedEventIds(), sensitive }); - } + results.add(new Object[] { spec.query(), name, spec.expectedEventIds() }); } return results; } - BaseEqlSpecTestCase(String index, String query, String name, long[] eventIds, boolean caseSensitive) { + BaseEqlSpecTestCase(String index, String query, String name, long[] eventIds) { this.index = index; this.query = query; this.name = name; this.eventIds = eventIds; - this.caseSensitive = caseSensitive; } public void test() throws Exception { - assertResponse(runQuery(index, query, caseSensitive)); + assertResponse(runQuery(index, query)); } protected void assertResponse(EqlSearchResponse response) { @@ -111,9 +104,8 @@ public abstract class BaseEqlSpecTestCase extends ESRestTestCase { } } - protected EqlSearchResponse runQuery(String index, String query, boolean isCaseSensitive) throws Exception { + protected EqlSearchResponse runQuery(String index, String query) throws Exception { EqlSearchRequest request = new EqlSearchRequest(index, query); - request.isCaseSensitive(isCaseSensitive); request.tiebreakerField("event.sequence"); // some queries return more than 10 results request.size(50); diff --git a/x-pack/plugin/eql/qa/common/src/main/java/org/elasticsearch/test/eql/EqlExtraSpecTestCase.java b/x-pack/plugin/eql/qa/common/src/main/java/org/elasticsearch/test/eql/EqlExtraSpecTestCase.java index a7d69167992..f023bf7a502 100644 --- a/x-pack/plugin/eql/qa/common/src/main/java/org/elasticsearch/test/eql/EqlExtraSpecTestCase.java +++ b/x-pack/plugin/eql/qa/common/src/main/java/org/elasticsearch/test/eql/EqlExtraSpecTestCase.java @@ -20,8 +20,8 @@ public abstract class EqlExtraSpecTestCase extends BaseEqlSpecTestCase { return asArray(EqlSpecLoader.load("/test_extra.toml", true, new HashSet<>())); } - public EqlExtraSpecTestCase(String query, String name, long[] eventIds, boolean caseSensitive) { - super(TEST_EXTRA_INDEX, query, name, eventIds, caseSensitive); + public EqlExtraSpecTestCase(String query, String name, long[] eventIds) { + super(TEST_EXTRA_INDEX, query, name, eventIds); } @Override diff --git a/x-pack/plugin/eql/qa/common/src/main/java/org/elasticsearch/test/eql/EqlSpec.java b/x-pack/plugin/eql/qa/common/src/main/java/org/elasticsearch/test/eql/EqlSpec.java index f548107a659..cd9e3faffff 100644 --- a/x-pack/plugin/eql/qa/common/src/main/java/org/elasticsearch/test/eql/EqlSpec.java +++ b/x-pack/plugin/eql/qa/common/src/main/java/org/elasticsearch/test/eql/EqlSpec.java @@ -19,12 +19,6 @@ public class EqlSpec { private String query; private long[] expectedEventIds; - // flag to dictate which modes are supported for the test - // null -> apply the test to both modes (case sensitive and case insensitive) - // TRUE -> case sensitive - // FALSE -> case insensitive - private Boolean caseSensitive = null; - public String name() { return name; } @@ -73,14 +67,6 @@ public class EqlSpec { this.expectedEventIds = expectedEventIds; } - public void caseSensitive(Boolean caseSensitive) { - this.caseSensitive = caseSensitive; - } - - public Boolean caseSensitive() { - return this.caseSensitive; - } - public EqlSpec withSensitivity(boolean caseSensitive) { EqlSpec spec = new EqlSpec(); spec.name = name; @@ -90,7 +76,6 @@ public class EqlSpec { spec.query = query; spec.expectedEventIds = expectedEventIds; - spec.caseSensitive = caseSensitive; return spec; } @@ -102,10 +87,6 @@ public class EqlSpec { str = appendWithComma(str, "description", description); str = appendWithComma(str, "note", note); - if (caseSensitive != null) { - str = appendWithComma(str, "case_sensitive", Boolean.toString(caseSensitive)); - } - if (tags != null) { str = appendWithComma(str, "tags", Arrays.toString(tags)); } @@ -128,13 +109,12 @@ public class EqlSpec { EqlSpec that = (EqlSpec) other; - return Objects.equals(this.query(), that.query()) - && Objects.equals(this.caseSensitive, that.caseSensitive); + return Objects.equals(this.query(), that.query()); } @Override public int hashCode() { - return Objects.hash(this.query, this.caseSensitive); + return Objects.hash(this.query); } private static String appendWithComma(String str, String name, String append) { diff --git a/x-pack/plugin/eql/qa/common/src/main/java/org/elasticsearch/test/eql/EqlSpecLoader.java b/x-pack/plugin/eql/qa/common/src/main/java/org/elasticsearch/test/eql/EqlSpecLoader.java index 523e3a6133a..8ac22479dc1 100644 --- a/x-pack/plugin/eql/qa/common/src/main/java/org/elasticsearch/test/eql/EqlSpecLoader.java +++ b/x-pack/plugin/eql/qa/common/src/main/java/org/elasticsearch/test/eql/EqlSpecLoader.java @@ -73,20 +73,6 @@ public class EqlSpecLoader { spec.note(getTrimmedString(table, "note")); spec.description(getTrimmedString(table, "description")); - Boolean caseSensitive = table.getBoolean("case_sensitive"); - Boolean caseInsensitive = table.getBoolean("case_insensitive"); - // if case_sensitive is TRUE and case_insensitive is not TRUE (FALSE or NULL), then the test is case sensitive only - if (Boolean.TRUE.equals(caseSensitive)) { - if (Boolean.FALSE.equals(caseInsensitive) || caseInsensitive == null) { - spec.caseSensitive(true); - } - } - // if case_sensitive is not TRUE (FALSE or NULL) and case_insensitive is TRUE, then the test is case insensitive only - else if (Boolean.TRUE.equals(caseInsensitive)) { - spec.caseSensitive(false); - } - // in all other cases, the test should run no matter the case sensitivity (should test both scenarios) - List arr = table.getList("tags"); if (arr != null) { String tags[] = new String[arr.size()]; diff --git a/x-pack/plugin/eql/qa/common/src/main/java/org/elasticsearch/test/eql/EqlSpecTestCase.java b/x-pack/plugin/eql/qa/common/src/main/java/org/elasticsearch/test/eql/EqlSpecTestCase.java index 087a61a4ae9..922b722e78d 100644 --- a/x-pack/plugin/eql/qa/common/src/main/java/org/elasticsearch/test/eql/EqlSpecTestCase.java +++ b/x-pack/plugin/eql/qa/common/src/main/java/org/elasticsearch/test/eql/EqlSpecTestCase.java @@ -51,7 +51,7 @@ public abstract class EqlSpecTestCase extends BaseEqlSpecTestCase { return "serial_event_id"; } - public EqlSpecTestCase(String query, String name, long[] eventIds, boolean caseSensitive) { - super(TEST_INDEX, query, name, eventIds, caseSensitive); + public EqlSpecTestCase(String query, String name, long[] eventIds) { + super(TEST_INDEX, query, name, eventIds); } } diff --git a/x-pack/plugin/eql/qa/common/src/main/resources/test_queries.toml b/x-pack/plugin/eql/qa/common/src/main/resources/test_queries.toml index 2004bdcb413..a017908ba57 100644 --- a/x-pack/plugin/eql/qa/common/src/main/resources/test_queries.toml +++ b/x-pack/plugin/eql/qa/common/src/main/resources/test_queries.toml @@ -177,13 +177,11 @@ expected_event_ids = [1, 2] [[queries]] name = "compareTwoFields1" # test that it works for comparing field <--> field -case_insensitive = true query = 'process where serial_event_id < 5 and process_name <= parent_process_name' expected_event_ids = [2, 3] [[queries]] name = "compareTwoFields2" -case_sensitive = true query = 'process where serial_event_id < 5 and process_name <= parent_process_name' expected_event_ids = [2] @@ -219,39 +217,33 @@ expected_event_ids = [1] [[queries]] name = "processWithStringComparisonCaseSensitive1" -case_sensitive = true notes = "s == 0x73; S == 0x53" query = 'process where process_name >= "system idle process" and process_name <= "System Idle Process"' expected_event_ids = [] [[queries]] name = "processWithStringComparisonCaseInsensitive1" -case_insensitive = true query = 'process where process_name >= "system idle process" and process_name <= "System Idle Process"' expected_event_ids = [1] [[queries]] name = "processWithStringComparisonCaseInsensitive2" -case_insensitive = true query = 'process where process_name >= "SYSTE" and process_name < "systex"' expected_event_ids = [1, 2] [[queries]] name = "processWithStringComparisonCaseInsensitive3" -case_insensitive = true query = 'process where process_name >= "sysT" and process_name < "SYsTeZZZ"' expected_event_ids = [1, 2] [[queries]] name = "processWithStringComparisonCaseInsensitive4" -case_insensitive = true query = 'process where process_name >= "SYSTE" and process_name <= "systex"' expected_event_ids = [1, 2] [[queries]] name = "processWithStringEqualityCaseInsensitive1" -case_insensitive = true query = ''' process where process_name : "VMACTHLP.exe" and unique_pid == 12 | filter true @@ -268,7 +260,6 @@ expected_event_ids = [3, 34, 48] [[queries]] name = "processNameINCaseInsensitive1" -case_insensitive = true query = ''' process where process_name in ("python.exe", "SMSS.exe", "explorer.exe") | unique process_name @@ -277,7 +268,6 @@ expected_event_ids = [3, 34, 48] [[queries]] name = "processNameINCaseInsensitive2" -case_insensitive = true query = ''' process where process_name in ("python.exe", "smss.exe", "Explorer.exe") | unique length(process_name) @@ -294,7 +284,6 @@ expected_event_ids = [3, 48] [[queries]] name = "processNameINWithUnique2" -case_insensitive = true query = ''' process where process_name in ("Python.exe", "smss.exe", "explorer.exe") | unique process_name != "python.exe" @@ -348,7 +337,6 @@ expected_event_ids = [3, 48, 50, 54] [[queries]] name = "lengthCaseInsensitive1" -case_insensitive = true expected_event_ids = [57] query = ''' registry where length(bytes_written_string_list) == 2 and bytes_written_string_list[1] : "EN" @@ -446,7 +434,6 @@ expected_event_ids = [2, 50, 51] [[queries]] name = "twoINsCaseInsensitive" -case_insensitive = true query = ''' process where opcode in (1,3) and process_name in (parent_process_name, "SYSTEM") ''' @@ -454,7 +441,6 @@ expected_event_ids = [2, 50, 51] [[queries]] name = "simpleTailWithSort" -case_insensitive = true expected_event_ids = [92, 95, 96, 91] query = ''' file where true @@ -1319,7 +1305,6 @@ any where process_name : "svchost.exe" [[queries]] name = "arrayContainsCaseInsensitive1" -case_insensitive = true expected_event_ids = [57] query = ''' registry where arrayContains(bytes_written_string_list, "En-uS") @@ -1327,7 +1312,6 @@ registry where arrayContains(bytes_written_string_list, "En-uS") [[queries]] name = "arrayContainsCaseInsensitive2" -case_insensitive = true expected_event_ids = [57] query = ''' registry where arrayContains(bytes_written_string_list, "En") @@ -1335,7 +1319,6 @@ registry where arrayContains(bytes_written_string_list, "En") [[queries]] name = "lengthCaseInsensitive2" -case_insensitive = true expected_event_ids = [57] query = ''' registry where length(bytes_written_string_list) > 0 and bytes_written_string_list[0] : "EN-us" @@ -1343,7 +1326,6 @@ registry where length(bytes_written_string_list) > 0 and bytes_written_string_li [[queries]] name = "arrayCaseInsensitive1" -case_insensitive = true expected_event_ids = [57] query = ''' registry where bytes_written_string_list[0] : "EN-us" @@ -1351,7 +1333,6 @@ registry where bytes_written_string_list[0] : "EN-us" [[queries]] name = "arrayCaseInsensitive2" -case_insensitive = true expected_event_ids = [57] query = ''' registry where bytes_written_string_list[1] : "EN" @@ -1360,7 +1341,6 @@ registry where bytes_written_string_list[1] : "EN" [[queries]] name = "arrayContainsCaseInsensitive3" -case_insensitive = true expected_event_ids = [57] query = ''' registry where arrayContains(bytes_written_string_list, "en-US") @@ -1368,7 +1348,6 @@ registry where arrayContains(bytes_written_string_list, "en-US") [[queries]] name = "arrayContainsCaseInsensitive4" -case_insensitive = true expected_event_ids = [57] query = ''' registry where arrayContains(bytes_written_string_list, "en") @@ -1376,7 +1355,6 @@ registry where arrayContains(bytes_written_string_list, "en") [[queries]] name = "arrayCaseInsensitive3" -case_insensitive = true expected_event_ids = [57] query = ''' registry where length(bytes_written_string_list) > 0 and bytes_written_string_list[0] : "en-US" @@ -1433,7 +1411,6 @@ expected_event_ids = [98] [[queries]] name = "stringEqualsCaseInsensitive1" -case_insensitive = true query = ''' process where "net.EXE" == original_file_name | filter process_name:"net*.exe" @@ -1443,7 +1420,6 @@ note = "check that case insensitive comparisons are performed even for lhs strin [[queries]] name = "stringEqualsCaseInsensitive2" -case_insensitive = true query = ''' process where process_name == original_file_name and process_name:"net*.exe" ''' @@ -1452,7 +1428,6 @@ note = "check that case insensitive comparisons are performed for fields." [[queries]] name = "fieldsComparisonCaseInsensitive" -case_insensitive = true query = ''' process where original_file_name == process_name and length(original_file_name) > 0 ''' @@ -1461,17 +1436,14 @@ description = "check that case insensitive comparisons are performed for fields. [[queries]] name = "startsWithCaseSensitive" -case_sensitive = true query = ''' file where opcode==0 and startsWith(file_name, "explorer.") ''' expected_event_ids = [88] description = "check built-in string functions" - [[queries]] name = "startsWithCaseInsensitive1" -case_insensitive = true query = ''' file where opcode==0 and startsWith(file_name, "explorer.") ''' @@ -1481,7 +1453,6 @@ description = "check built-in string functions" [[queries]] name = "startsWithCaseInsensitive2" -case_insensitive = true query = ''' file where opcode==0 and startsWith(file_name, "exploRER.") ''' @@ -1490,7 +1461,6 @@ description = "check built-in string functions" [[queries]] name = "startsWithCaseInsensitive3" -case_insensitive = true query = ''' file where opcode==0 and startsWith(file_name, "expLORER.exe") ''' @@ -1508,7 +1478,6 @@ description = "check built-in string functions" [[queries]] name = "endsWithCaseInsensitive" -case_insensitive = true query = ''' file where opcode==0 and endsWith(file_name, "loREr.exe") ''' @@ -1525,7 +1494,6 @@ description = "check built-in string functions" [[queries]] name = "endsWithAndCondition" -case_insensitive = true query = ''' file where opcode==0 and serial_event_id == 88 and startsWith("explorer.exeaAAAA", "EXPLORER.exe") ''' @@ -1542,7 +1510,6 @@ description = "check built-in string functions" [[queries]] name = "indexOfCaseInsensitive" -case_insensitive = true query = ''' file where opcode==0 and indexOf(file_name, "plore") == 2 and indexOf(file_name, ".pf") == null ''' @@ -1559,7 +1526,6 @@ description = "check built-in string functions" [[queries]] name = "indexOf2" -case_sensitive = true query = ''' file where opcode==0 and indexOf(file_name, "plorer.", 0) == 2 ''' @@ -1568,7 +1534,6 @@ description = "check built-in string functions" [[queries]] name = "indexOf3" -case_insensitive = true query = ''' file where opcode==0 and indexOf(file_name, "plorer.", 0) == 2 ''' @@ -1577,7 +1542,6 @@ description = "check built-in string functions" [[queries]] name = "indexOf4" -case_sensitive = true query = ''' file where opcode==0 and indexOf(file_name, "plorer.", 2) != null ''' @@ -1586,7 +1550,6 @@ description = "check built-in string functions" [[queries]] name = "indexOf5" -case_insensitive = true query = ''' file where opcode==0 and indexOf(file_name, "plorer.", 2) != null ''' @@ -1611,7 +1574,6 @@ description = "check built-in string functions" [[queries]] name = "indexOf8" -case_insensitive = true query = ''' file where opcode==0 and indexOf(file_name, "plorer.", 2) == 2 ''' @@ -1620,7 +1582,6 @@ description = "check substring ranges" [[queries]] name = "indexOf9" -case_sensitive = true query = ''' file where opcode==0 and indexOf(file_name, "plorer.", 2) == 2 ''' @@ -1629,7 +1590,6 @@ description = "check substring ranges" [[queries]] name = "indexOf10" -case_sensitive = true query = ''' file where opcode==0 and indexOf(file_name, "explorer.", 0) == 0 ''' @@ -1638,7 +1598,6 @@ description = "check substring ranges" [[queries]] name = "indexOf11" -case_insensitive = true query = ''' file where opcode==0 and indexOf(file_name, "explorer.", 0) == 0 ''' @@ -1647,7 +1606,6 @@ description = "check substring ranges" [[queries]] name = "substring1" -case_insensitive = true query = ''' file where serial_event_id==88 and substring(file_name, 0, 4) : "expl" ''' @@ -1656,7 +1614,6 @@ description = "check substring ranges" [[queries]] name = "substring2" -case_sensitive = true query = ''' file where substring(file_name, 1, 3) == "xp" ''' @@ -1665,7 +1622,6 @@ description = "check substring ranges" [[queries]] name = "substring3" -case_insensitive = true query = ''' file where substring(file_name, 1, 3) : "xp" ''' @@ -1754,7 +1710,6 @@ description = "test string concatenation" [[queries]] name = "concatCaseInsensitive" -case_insensitive = true query = ''' process where concat(serial_event_id, ":", process_name, opcode) : "5:winINIT.exe3" ''' @@ -1763,7 +1718,6 @@ description = "test string concatenation" [[queries]] name = "fieldComparisonCaseInsensitive" -case_insensitive = true query = ''' process where process_name != original_file_name and length(original_file_name) > 0 ''' @@ -1781,7 +1735,6 @@ registry where arraySearch(bytes_written_string_list, a, a : "en-US") [[queries]] name = "arraySearchCaseSensitive" -case_sensitive = true expected_event_ids = [] description = "test arraySearch functionality for lists of strings, and lists of objects" query = ''' @@ -1790,7 +1743,6 @@ registry where arraySearch(bytes_written_string_list, a, a : "EN-US") [[queries]] name = "arraySearchCaseInsensitive1" -case_insensitive = true expected_event_ids = [57] description = "test arraySearch functionality for lists of strings, and lists of objects" query = ''' @@ -1799,7 +1751,6 @@ registry where arraySearch(bytes_written_string_list, a, a : "en-us") [[queries]] name = "arraySearchCaseInsensitive2" -case_insensitive = true expected_event_ids = [57] description = "test arraySearch functionality for lists of strings, and lists of objects" query = ''' @@ -1889,7 +1840,6 @@ network where safe(divide(process_name, process_name)) [[queries]] name = "nestedSetComparisons" -case_insensitive = true query = ''' file where serial_event_id == 82 and (true == (process_name in ("svchost.EXE", "bad.exe", "bad2.exe"))) ''' @@ -1898,7 +1848,6 @@ description = "nested set comparisons" [[queries]] name = "arrayCount1" -case_insensitive = true expected_event_ids = [57] query = ''' registry where arrayCount(bytes_written_string_list, s, s : "*-us") == 1 @@ -1906,7 +1855,6 @@ registry where arrayCount(bytes_written_string_list, s, s : "*-us") == 1 [[queries]] name = "arrayCount2" -case_insensitive = true expected_event_ids = [57] query = ''' registry where arrayCount(bytes_written_string_list, s, s : "*en*") == 2 @@ -1914,7 +1862,6 @@ registry where arrayCount(bytes_written_string_list, s, s : "*en*") == 2 [[queries]] name = "arrayContainsCaseInsensitive5" -case_insensitive = true expected_event_ids = [57] query = ''' registry where arrayContains(bytes_written_string_list, "missing", "en-US") @@ -1952,7 +1899,6 @@ query = "file where serial_event_id % 40 == 2" [[queries]] name = "betweenCaseInsensitive1" -case_insensitive = true expected_event_ids = [1, 2] query = ''' process where between(process_name, "s", "e") : "yst" @@ -1960,7 +1906,6 @@ process where between(process_name, "s", "e") : "yst" [[queries]] name = "betweenCaseInsensitive2" -case_insensitive = true expected_event_ids = [1, 2] query = ''' process where between(process_name, "s", "e", false) : "yst" @@ -1968,7 +1913,6 @@ process where between(process_name, "s", "e", false) : "yst" [[queries]] name = "betweenCaseSensitive" -case_sensitive = true expected_event_ids = [1, 2, 42] query = ''' process where between(process_name, "s", "e", false) : "t" @@ -1983,7 +1927,6 @@ process where between(process_name, "S", "e", true) : "ystem Idle Proc" [[queries]] name = "betweenCaseInsensitive3" -case_insensitive = true expected_event_ids = [1] query = ''' process where between(process_name, "s", "e", true) : "ystem Idle Proc" @@ -2034,7 +1977,6 @@ network where cidrMatch(source_address, "0.0.0.0/0") [[queries]] name = "lengthCaseSensitive" -case_sensitive = true expected_event_ids = [7, 14, 29, 44] query = ''' process where length(between(process_name, "g", "e")) > 0 @@ -2042,7 +1984,6 @@ process where length(between(process_name, "g", "e")) > 0 [[queries]] name = "lengthCaseInsensitiveAndBetween" -case_insensitive = true expected_event_ids = [7, 14, 22, 29, 44] query = ''' process where length(between(process_name, "g", "e")) > 0 diff --git a/x-pack/plugin/eql/qa/common/src/main/resources/test_queries_unsupported.toml b/x-pack/plugin/eql/qa/common/src/main/resources/test_queries_unsupported.toml index f63b8dea154..72a2914d515 100644 --- a/x-pack/plugin/eql/qa/common/src/main/resources/test_queries_unsupported.toml +++ b/x-pack/plugin/eql/qa/common/src/main/resources/test_queries_unsupported.toml @@ -8,6 +8,99 @@ # in order to allow at least one round-trip test with the test harness. # This will be removed once the EQL implementation is wired and actually supports this query. +[[queries]] +name = "startsWithCaseInsensitive1" +query = ''' +file where opcode==0 and startsWith(file_name, "explorer.") +''' +expected_event_ids = [88, 92] +description = "check built-in string functions" + +[[queries]] +name = "startsWithCaseInsensitive2" +query = ''' +file where opcode==0 and startsWith(file_name, "exploRER.") +''' +expected_event_ids = [88, 92] +description = "check built-in string functions" + +[[queries]] +name = "startsWithCaseInsensitive3" +query = ''' +file where opcode==0 and startsWith(file_name, "expLORER.exe") +''' +expected_event_ids = [88, 92] +description = "check built-in string functions" + +[[queries]] +name = "endsWithCaseInsensitive" +query = ''' +file where opcode==0 and endsWith(file_name, "loREr.exe") +''' +expected_event_ids = [88] +description = "check built-in string functions" + +[[queries]] +name = "endsWithAndCondition" +query = ''' +file where opcode==0 and serial_event_id == 88 and startsWith("explorer.exeaAAAA", "EXPLORER.exe") +''' +expected_event_ids = [88] +description = "check built-in string functions" + +[[queries]] +name = "indexOf3" +query = ''' +file where opcode==0 and indexOf(file_name, "plorer.", 0) == 2 +''' +expected_event_ids = [88, 92] +description = "check built-in string functions" + +[[queries]] +name = "indexOf5" +query = ''' +file where opcode==0 and indexOf(file_name, "plorer.", 2) != null +''' +expected_event_ids = [88, 92] +description = "check built-in string functions" + +[[queries]] +name = "indexOf8" +query = ''' +file where opcode==0 and indexOf(file_name, "plorer.", 2) == 2 +''' +expected_event_ids = [88, 92] +description = "check substring ranges" + +[[queries]] +name = "indexOf11" +query = ''' +file where opcode==0 and indexOf(file_name, "explorer.", 0) == 0 +''' +expected_event_ids = [88, 92] +description = "check substring ranges" + +[[queries]] +name = "betweenCaseInsensitive1" +expected_event_ids = [1, 2] +query = ''' +process where between(process_name, "s", "e") : "yst" +''' + +[[queries]] +name = "betweenCaseInsensitive2" +expected_event_ids = [1, 2] +query = ''' +process where between(process_name, "s", "e", false) : "yst" +''' + +[[queries]] +name = "betweenCaseInsensitive3" +expected_event_ids = [1] +query = ''' +process where between(process_name, "s", "e", true) : "ystem Idle Proc" +''' + [[queries]] name = "twoSequencesWithKeys" query = ''' @@ -93,13 +186,11 @@ expected_event_ids = [7, 8] [[queries]] name = "compareTwoFields1" -case_insensitive = true query = 'process where serial_event_id < 5 and process_name <= parent_process_name' expected_event_ids = [2, 3] [[queries]] name = "compareTwoFields2" -case_sensitive = true query = 'process where serial_event_id < 5 and process_name <= parent_process_name' expected_event_ids = [2] @@ -107,32 +198,27 @@ expected_event_ids = [2] [[queries]] name = "processWithStringComparisonCaseInsensitive1" -case_insensitive = true query = 'process where process_name >= "system idle process" and process_name <= "System Idle Process"' expected_event_ids = [1] [[queries]] name = "processWithStringComparisonCaseInsensitive2" -case_insensitive = true query = 'process where process_name >= "SYSTE" and process_name < "systex"' expected_event_ids = [1, 2] [[queries]] name = "processWithStringComparisonCaseInsensitive3" -case_insensitive = true query = 'process where process_name >= "sysT" and process_name < "SYsTeZZZ"' expected_event_ids = [1, 2] [[queries]] name = "processWithStringComparisonCaseInsensitive4" -case_insensitive = true query = 'process where process_name >= "SYSTE" and process_name <= "systex"' expected_event_ids = [1, 2] [[queries]] name = "processWithStringEqualityCaseInsensitive1" -case_insensitive = true query = ''' process where process_name : "VMACTHLP.exe" and unique_pid == 12 | filter true @@ -149,7 +235,6 @@ expected_event_ids = [3, 34, 48] [[queries]] name = "processNameINCaseInsensitive1" -case_insensitive = true query = ''' process where process_name in ("python.exe", "SMSS.exe", "explorer.exe") | unique process_name @@ -158,7 +243,6 @@ expected_event_ids = [3, 34, 48] [[queries]] name = "processNameINCaseInsensitive2" -case_insensitive = true query = ''' process where process_name in ("python.exe", "smss.exe", "Explorer.exe") | unique length(process_name) @@ -175,7 +259,6 @@ expected_event_ids = [3, 48] [[queries]] name = "processNameINWithUnique2" -case_insensitive = true query = ''' process where process_name in ("Python.exe", "smss.exe", "explorer.exe") | unique process_name != "python.exe" @@ -229,7 +312,6 @@ expected_event_ids = [3, 48, 50, 54] [[queries]] name = "lengthCaseInsensitive" -case_insensitive = true expected_event_ids = [57] query = ''' registry where length(bytes_written_string_list) == 2 and bytes_written_string_list[1] : "EN" @@ -282,7 +364,6 @@ expected_event_ids = [2, 50, 51] [[queries]] name = "twoINsCaseInsensitive" -case_insensitive = true query = ''' process where opcode in (1,3) and process_name in (parent_process_name, "SYSTEM") ''' @@ -290,7 +371,6 @@ expected_event_ids = [2, 50, 51] [[queries]] name = "simpleTailWithSort" -case_insensitive = true expected_event_ids = [92, 95, 96, 91] query = ''' file where true @@ -716,7 +796,6 @@ any where process_name : "svchost.exe" [[queries]] name = "arrayContainsCaseInsensitive1" -case_insensitive = true expected_event_ids = [57] query = ''' registry where arrayContains(bytes_written_string_list, "En-uS") @@ -724,7 +803,6 @@ registry where arrayContains(bytes_written_string_list, "En-uS") [[queries]] name = "arrayContainsCaseInsensitive2" -case_insensitive = true expected_event_ids = [57] query = ''' registry where arrayContains(bytes_written_string_list, "En") @@ -732,7 +810,6 @@ registry where arrayContains(bytes_written_string_list, "En") [[queries]] name = "lengthCaseInsensitive" -case_insensitive = true expected_event_ids = [57] query = ''' registry where length(bytes_written_string_list) > 0 and bytes_written_string_list[0] : "EN-us" @@ -740,7 +817,6 @@ registry where length(bytes_written_string_list) > 0 and bytes_written_string_li [[queries]] name = "arrayCaseInsensitive1" -case_insensitive = true expected_event_ids = [57] query = ''' registry where bytes_written_string_list[0] : "EN-us" @@ -748,7 +824,6 @@ registry where bytes_written_string_list[0] : "EN-us" [[queries]] name = "arrayCaseInsensitive2" -case_insensitive = true expected_event_ids = [57] query = ''' registry where bytes_written_string_list[1] : "EN" @@ -757,7 +832,6 @@ registry where bytes_written_string_list[1] : "EN" [[queries]] name = "arrayContainsCaseInsensitive3" -case_insensitive = true expected_event_ids = [57] query = ''' registry where arrayContains(bytes_written_string_list, "en-US") @@ -765,7 +839,6 @@ registry where arrayContains(bytes_written_string_list, "en-US") [[queries]] name = "arrayContainsCaseInsensitive4" -case_insensitive = true expected_event_ids = [57] query = ''' registry where arrayContains(bytes_written_string_list, "en") @@ -773,7 +846,6 @@ registry where arrayContains(bytes_written_string_list, "en") [[queries]] name = "arrayCaseInsensitive3" -case_insensitive = true expected_event_ids = [57] query = ''' registry where length(bytes_written_string_list) > 0 and bytes_written_string_list[0] : "en-US" @@ -831,7 +903,6 @@ expected_event_ids = [98] [[queries]] name = "stringEqualsCaseInsensitive1" -case_insensitive = true query = ''' process where "net.EXE" == original_file_name | filter process_name:"net*.exe" @@ -841,7 +912,6 @@ note = "check that case insensitive comparisons are performed even for lhs strin [[queries]] name = "stringEqualsCaseInsensitive2" -case_insensitive = true query = ''' process where process_name == original_file_name and process_name:"net*.exe" ''' @@ -850,7 +920,6 @@ note = "check that case insensitive comparisons are performed for fields." [[queries]] name = "fieldsComparisonCaseInsensitive" -case_insensitive = true query = ''' process where original_file_name == process_name and length(original_file_name) > 0 ''' @@ -859,7 +928,6 @@ description = "check that case insensitive comparisons are performed for fields. [[queries]] name = "substring3" -case_insensitive = true query = ''' file where substring(file_name, 1, 3) : "xp" ''' @@ -876,7 +944,6 @@ description = "test built-in math functions" [[queries]] name = "concatCaseInsensitive" -case_insensitive = true query = ''' process where concat(serial_event_id, ":", process_name, opcode) : "5:winINIT.exe3" ''' @@ -885,7 +952,6 @@ description = "test string concatenation" [[queries]] name = "fieldComparisonCaseInsensitive" -case_insensitive = true query = ''' process where process_name != original_file_name and length(original_file_name) > 0 ''' @@ -903,7 +969,6 @@ registry where arraySearch(bytes_written_string_list, a, a : "en-US") [[queries]] name = "arraySearchCaseSensitive" -case_sensitive = true expected_event_ids = [] description = "test arraySearch functionality for lists of strings, and lists of objects" query = ''' @@ -912,7 +977,6 @@ registry where arraySearch(bytes_written_string_list, a, a : "EN-US") [[queries]] name = "arraySearchCaseInsensitive1" -case_insensitive = true expected_event_ids = [57] description = "test arraySearch functionality for lists of strings, and lists of objects" query = ''' @@ -921,7 +985,6 @@ registry where arraySearch(bytes_written_string_list, a, a : "en-us") [[queries]] name = "arraySearchCaseInsensitive2" -case_insensitive = true expected_event_ids = [57] description = "test arraySearch functionality for lists of strings, and lists of objects" query = ''' @@ -1011,7 +1074,6 @@ network where safe(divide(process_name, process_name)) [[queries]] name = "nestedSetComparisons" -case_insensitive = true query = ''' file where serial_event_id == 82 and (true == (process_name in ("svchost.EXE", "bad.exe", "bad2.exe"))) ''' @@ -1020,7 +1082,6 @@ description = "nested set comparisons" [[queries]] name = "arrayCount1" -case_insensitive = true expected_event_ids = [57] query = ''' registry where arrayCount(bytes_written_string_list, s, s : "*-us") == 1 @@ -1028,7 +1089,6 @@ registry where arrayCount(bytes_written_string_list, s, s : "*-us") == 1 [[queries]] name = "arrayCount2" -case_insensitive = true expected_event_ids = [57] query = ''' registry where arrayCount(bytes_written_string_list, s, s : "*en*") == 2 @@ -1036,7 +1096,6 @@ registry where arrayCount(bytes_written_string_list, s, s : "*en*") == 2 [[queries]] name = "arrayContainsCaseInsensitive5" -case_insensitive = true expected_event_ids = [57] query = ''' registry where arrayContains(bytes_written_string_list, "missing", "en-US") @@ -1069,7 +1128,6 @@ expected_event_ids = [1, 2, # TODO: update toggles for this function [[queries]] name = "betweenCaseSensitive" -case_sensitive = true expected_event_ids = [1, 2, 42] query = ''' process where between(process_name, "s", "e", false) : "t" @@ -1078,7 +1136,6 @@ process where between(process_name, "s", "e", false) : "t" # TODO: add toggles to this function so it's not always insensitive [[queries]] name = "lengthCaseSensitive" -case_sensitive = true expected_event_ids = [7, 14, 29, 44] query = ''' process where length(between(process_name, "g", "e")) > 0 diff --git a/x-pack/plugin/eql/qa/rest/src/javaRestTest/java/org/elasticsearch/xpack/eql/EqlExtraIT.java b/x-pack/plugin/eql/qa/rest/src/javaRestTest/java/org/elasticsearch/xpack/eql/EqlExtraIT.java index c804c050bba..313571b415d 100644 --- a/x-pack/plugin/eql/qa/rest/src/javaRestTest/java/org/elasticsearch/xpack/eql/EqlExtraIT.java +++ b/x-pack/plugin/eql/qa/rest/src/javaRestTest/java/org/elasticsearch/xpack/eql/EqlExtraIT.java @@ -12,7 +12,7 @@ import org.elasticsearch.test.junit.annotations.TestLogging; @TestLogging(value = "org.elasticsearch.xpack.eql:TRACE", reason = "results logging") public class EqlExtraIT extends EqlExtraSpecTestCase { - public EqlExtraIT(String query, String name, long[] eventIds, boolean caseSensitive) { - super(query, name, eventIds, caseSensitive); + public EqlExtraIT(String query, String name, long[] eventIds) { + super(query, name, eventIds); } } diff --git a/x-pack/plugin/eql/qa/rest/src/javaRestTest/java/org/elasticsearch/xpack/eql/EqlSpecIT.java b/x-pack/plugin/eql/qa/rest/src/javaRestTest/java/org/elasticsearch/xpack/eql/EqlSpecIT.java index d84668d16c1..fa6786819c1 100644 --- a/x-pack/plugin/eql/qa/rest/src/javaRestTest/java/org/elasticsearch/xpack/eql/EqlSpecIT.java +++ b/x-pack/plugin/eql/qa/rest/src/javaRestTest/java/org/elasticsearch/xpack/eql/EqlSpecIT.java @@ -10,7 +10,7 @@ import org.elasticsearch.test.eql.EqlSpecTestCase; public class EqlSpecIT extends EqlSpecTestCase { - public EqlSpecIT(String query, String name, long[] eventIds, boolean caseSensitive) { - super(query, name, eventIds, caseSensitive); + public EqlSpecIT(String query, String name, long[] eventIds) { + super(query, name, eventIds); } } diff --git a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/action/EqlSearchRequest.java b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/action/EqlSearchRequest.java index 62c691033a9..6f4dd177bd5 100644 --- a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/action/EqlSearchRequest.java +++ b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/action/EqlSearchRequest.java @@ -50,7 +50,6 @@ public class EqlSearchRequest extends ActionRequest implements IndicesRequest.Re private int size = RequestDefaults.SIZE; private int fetchSize = RequestDefaults.FETCH_SIZE; private String query; - private boolean isCaseSensitive = false; // Async settings private TimeValue waitForCompletionTimeout = null; @@ -67,7 +66,6 @@ public class EqlSearchRequest extends ActionRequest implements IndicesRequest.Re static final String KEY_WAIT_FOR_COMPLETION_TIMEOUT = "wait_for_completion_timeout"; static final String KEY_KEEP_ALIVE = "keep_alive"; static final String KEY_KEEP_ON_COMPLETION = "keep_on_completion"; - static final String KEY_CASE_SENSITIVE = "case_sensitive"; static final ParseField FILTER = new ParseField(KEY_FILTER); static final ParseField TIMESTAMP_FIELD = new ParseField(KEY_TIMESTAMP_FIELD); @@ -79,7 +77,6 @@ public class EqlSearchRequest extends ActionRequest implements IndicesRequest.Re static final ParseField WAIT_FOR_COMPLETION_TIMEOUT = new ParseField(KEY_WAIT_FOR_COMPLETION_TIMEOUT); static final ParseField KEEP_ALIVE = new ParseField(KEY_KEEP_ALIVE); static final ParseField KEEP_ON_COMPLETION = new ParseField(KEY_KEEP_ON_COMPLETION); - static final ParseField CASE_SENSITIVE = new ParseField(KEY_CASE_SENSITIVE); private static final ObjectParser PARSER = objectParser(EqlSearchRequest::new); @@ -103,7 +100,9 @@ public class EqlSearchRequest extends ActionRequest implements IndicesRequest.Re this.keepAlive = in.readOptionalTimeValue(); this.keepOnCompletion = in.readBoolean(); } - isCaseSensitive = in.readBoolean(); + if (in.getVersion().before(Version.V_7_10_0)) { + in.readBoolean(); + } } @Override @@ -178,7 +177,6 @@ public class EqlSearchRequest extends ActionRequest implements IndicesRequest.Re builder.field(KEY_KEEP_ALIVE, keepAlive); } builder.field(KEY_KEEP_ON_COMPLETION, keepOnCompletion); - builder.field(KEY_CASE_SENSITIVE, isCaseSensitive); return builder; } @@ -203,7 +201,6 @@ public class EqlSearchRequest extends ActionRequest implements IndicesRequest.Re parser.declareField(EqlSearchRequest::keepAlive, (p, c) -> TimeValue.parseTimeValue(p.text(), KEY_KEEP_ALIVE), KEEP_ALIVE, ObjectParser.ValueType.VALUE); parser.declareBoolean(EqlSearchRequest::keepOnCompletion, KEEP_ON_COMPLETION); - parser.declareBoolean(EqlSearchRequest::isCaseSensitive, CASE_SENSITIVE); return parser; } @@ -293,13 +290,6 @@ public class EqlSearchRequest extends ActionRequest implements IndicesRequest.Re return this; } - public boolean isCaseSensitive() { return this.isCaseSensitive; } - - public EqlSearchRequest isCaseSensitive(boolean isCaseSensitive) { - this.isCaseSensitive = isCaseSensitive; - return this; - } - @Override public void writeTo(StreamOutput out) throws IOException { super.writeTo(out); @@ -317,7 +307,9 @@ public class EqlSearchRequest extends ActionRequest implements IndicesRequest.Re out.writeOptionalTimeValue(keepAlive); out.writeBoolean(keepOnCompletion); } - out.writeBoolean(isCaseSensitive); + if (out.getVersion().onOrAfter(Version.V_7_10_0)) { + out.writeBoolean(true); + } } @Override @@ -341,8 +333,7 @@ public class EqlSearchRequest extends ActionRequest implements IndicesRequest.Re Objects.equals(eventCategoryField, that.eventCategoryField) && Objects.equals(query, that.query) && Objects.equals(waitForCompletionTimeout, that.waitForCompletionTimeout) && - Objects.equals(keepAlive, that.keepAlive) && - Objects.equals(isCaseSensitive, that.isCaseSensitive); + Objects.equals(keepAlive, that.keepAlive); } @@ -359,8 +350,7 @@ public class EqlSearchRequest extends ActionRequest implements IndicesRequest.Re eventCategoryField, query, waitForCompletionTimeout, - keepAlive, - isCaseSensitive); + keepAlive); } @Override diff --git a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/action/EqlSearchRequestBuilder.java b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/action/EqlSearchRequestBuilder.java index 6e7d1326a39..7540b1a16ed 100644 --- a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/action/EqlSearchRequestBuilder.java +++ b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/action/EqlSearchRequestBuilder.java @@ -54,9 +54,4 @@ public class EqlSearchRequestBuilder extends ActionRequestBuilder info() { - return NodeInfo.create(this, Between::new, input, left, right, greedy, caseSensitive); + return NodeInfo.create(this, Between::new, input, left, right, greedy); } @Override @@ -146,10 +146,10 @@ public class Between extends ScalarFunction implements OptionalArgument { @Override public Expression replaceChildren(List newChildren) { - if (newChildren.size() != 5) { - throw new IllegalArgumentException("expected [5] children but received [" + newChildren.size() + "]"); + if (newChildren.size() != 4) { + throw new IllegalArgumentException("expected [4] children but received [" + newChildren.size() + "]"); } - return new Between(source(), newChildren.get(0), newChildren.get(1), newChildren.get(2), newChildren.get(3), newChildren.get(4)); + return new Between(source(), newChildren.get(0), newChildren.get(1), newChildren.get(2), newChildren.get(3)); } } diff --git a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/expression/function/scalar/string/EndsWith.java b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/expression/function/scalar/string/EndsWith.java index 74af8c7bbc5..d8b952e909b 100644 --- a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/expression/function/scalar/string/EndsWith.java +++ b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/expression/function/scalar/string/EndsWith.java @@ -6,7 +6,6 @@ package org.elasticsearch.xpack.eql.expression.function.scalar.string; -import org.elasticsearch.xpack.eql.session.EqlConfiguration; import org.elasticsearch.xpack.ql.expression.Expression; import org.elasticsearch.xpack.ql.expression.Expressions; import org.elasticsearch.xpack.ql.expression.Expressions.ParamOrdinal; @@ -47,7 +46,7 @@ public class EndsWith extends CaseSensitiveScalarFunction { @Override public boolean isCaseSensitive() { - return ((EqlConfiguration) configuration()).isCaseSensitive(); + return true; } @Override diff --git a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/expression/function/scalar/string/IndexOf.java b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/expression/function/scalar/string/IndexOf.java index f2eed8b214f..9c3af54a8b6 100644 --- a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/expression/function/scalar/string/IndexOf.java +++ b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/expression/function/scalar/string/IndexOf.java @@ -6,7 +6,6 @@ package org.elasticsearch.xpack.eql.expression.function.scalar.string; -import org.elasticsearch.xpack.eql.session.EqlConfiguration; import org.elasticsearch.xpack.ql.expression.Expression; import org.elasticsearch.xpack.ql.expression.Expressions; import org.elasticsearch.xpack.ql.expression.Expressions.ParamOrdinal; @@ -50,7 +49,7 @@ public class IndexOf extends CaseSensitiveScalarFunction implements OptionalArgu @Override public boolean isCaseSensitive() { - return ((EqlConfiguration) configuration()).isCaseSensitive(); + return true; } @Override diff --git a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/expression/function/scalar/string/StartsWith.java b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/expression/function/scalar/string/StartsWith.java index ce23c7de50b..e0ff7ead947 100644 --- a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/expression/function/scalar/string/StartsWith.java +++ b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/expression/function/scalar/string/StartsWith.java @@ -6,7 +6,6 @@ package org.elasticsearch.xpack.eql.expression.function.scalar.string; -import org.elasticsearch.xpack.eql.session.EqlConfiguration; import org.elasticsearch.xpack.ql.expression.Expression; import org.elasticsearch.xpack.ql.session.Configuration; import org.elasticsearch.xpack.ql.tree.Source; @@ -19,7 +18,7 @@ public class StartsWith extends org.elasticsearch.xpack.ql.expression.function.s @Override public boolean isCaseSensitive() { - return ((EqlConfiguration) configuration()).isCaseSensitive(); + return true; } } diff --git a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/expression/function/scalar/string/StringContains.java b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/expression/function/scalar/string/StringContains.java index 2f5ed1d4411..1ccccb87042 100644 --- a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/expression/function/scalar/string/StringContains.java +++ b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/expression/function/scalar/string/StringContains.java @@ -132,6 +132,6 @@ public class StringContains extends CaseSensitiveScalarFunction { @Override public boolean isCaseSensitive() { - return eqlConfiguration().isCaseSensitive(); + return true; } } diff --git a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/optimizer/Optimizer.java b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/optimizer/Optimizer.java index 7870712d8ba..220f0595fd4 100644 --- a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/optimizer/Optimizer.java +++ b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/optimizer/Optimizer.java @@ -131,7 +131,7 @@ public class Optimizer extends RuleExecutor { } if (target != null) { - Expression like = new Like(e.source(), target, StringUtils.toLikePattern(wildString)); + Expression like = new Like(e.source(), target, StringUtils.toLikePattern(wildString), true); if (e instanceof InsensitiveNotEquals) { like = new Not(e.source(), like); } diff --git a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/planner/QueryTranslator.java b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/planner/QueryTranslator.java index c9cca060eea..3ee1f16c101 100644 --- a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/planner/QueryTranslator.java +++ b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/planner/QueryTranslator.java @@ -103,7 +103,7 @@ final class QueryTranslator { // (which is important for strings) name = ((FieldAttribute) bc.left()).exactAttribute().name(); } - Query query = new TermQuery(source, name, value); + Query query = new TermQuery(source, name, value, true); if (bc instanceof InsensitiveNotEquals) { query = new NotQuery(source, query); diff --git a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/plugin/TransportEqlSearchAction.java b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/plugin/TransportEqlSearchAction.java index 14c9e598ba2..501955036e4 100644 --- a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/plugin/TransportEqlSearchAction.java +++ b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/plugin/TransportEqlSearchAction.java @@ -119,7 +119,7 @@ public class TransportEqlSearchAction extends HandledTransportAction listener.onResponse(createResponse(r, task.getExecutionId())), listener::onFailure)); } diff --git a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/session/EqlConfiguration.java b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/session/EqlConfiguration.java index 17595416268..3d8ba8761b1 100644 --- a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/session/EqlConfiguration.java +++ b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/session/EqlConfiguration.java @@ -23,14 +23,13 @@ public class EqlConfiguration extends org.elasticsearch.xpack.ql.session.Configu private final boolean includeFrozenIndices; private final TaskId taskId; private final EqlSearchTask task; - private final boolean isCaseSensitive; private final int fetchSize; @Nullable private final QueryBuilder filter; public EqlConfiguration(String[] indices, ZoneId zi, String username, String clusterName, QueryBuilder filter, TimeValue requestTimeout, - boolean includeFrozen, boolean isCaseSensitive, int fetchSize, String clientId, TaskId taskId, + boolean includeFrozen, int fetchSize, String clientId, TaskId taskId, EqlSearchTask task) { super(zi, username, clusterName); @@ -39,7 +38,6 @@ public class EqlConfiguration extends org.elasticsearch.xpack.ql.session.Configu this.requestTimeout = requestTimeout; this.clientId = clientId; this.includeFrozenIndices = includeFrozen; - this.isCaseSensitive = isCaseSensitive; this.taskId = taskId; this.task = task; this.fetchSize = fetchSize; @@ -73,10 +71,6 @@ public class EqlConfiguration extends org.elasticsearch.xpack.ql.session.Configu return includeFrozenIndices; } - public boolean isCaseSensitive() { - return isCaseSensitive; - } - public boolean isCancelled() { return task.isCancelled(); } diff --git a/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/EqlTestUtils.java b/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/EqlTestUtils.java index ab208aad433..48f3a4387a8 100644 --- a/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/EqlTestUtils.java +++ b/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/EqlTestUtils.java @@ -31,23 +31,11 @@ public final class EqlTestUtils { private EqlTestUtils() { } - public static final EqlConfiguration TEST_CFG_CASE_INSENSITIVE = new EqlConfiguration(new String[] {"none"}, - org.elasticsearch.xpack.ql.util.DateUtils.UTC, "nobody", "cluster", null, TimeValue.timeValueSeconds(30), false, false, - 123, "", new TaskId("test", 123), null); - - public static final EqlConfiguration TEST_CFG_CASE_SENSITIVE = new EqlConfiguration(new String[] {"none"}, - org.elasticsearch.xpack.ql.util.DateUtils.UTC, "nobody", "cluster", null, TimeValue.timeValueSeconds(30), false, true, + public static final EqlConfiguration TEST_CFG = new EqlConfiguration(new String[] {"none"}, + org.elasticsearch.xpack.ql.util.DateUtils.UTC, "nobody", "cluster", null, TimeValue.timeValueSeconds(30), false, 123, "", new TaskId("test", 123), null); public static EqlConfiguration randomConfiguration() { - return internalRandomConfiguration(randomBoolean()); - } - - public static EqlConfiguration randomConfigurationWithCaseSensitive(boolean isCaseSensitive) { - return internalRandomConfiguration(isCaseSensitive); - } - - private static EqlConfiguration internalRandomConfiguration(boolean isCaseSensitive) { return new EqlConfiguration(new String[]{randomAlphaOfLength(16)}, randomZone(), randomAlphaOfLength(16), @@ -55,7 +43,6 @@ public final class EqlTestUtils { null, new TimeValue(randomNonNegativeLong()), randomBoolean(), - isCaseSensitive, randomIntBetween(1, 1000), randomAlphaOfLength(16), new TaskId(randomAlphaOfLength(10), randomNonNegativeLong()), diff --git a/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/action/EqlRequestParserTests.java b/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/action/EqlRequestParserTests.java index e7ed40b4704..d39a3f88bc7 100644 --- a/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/action/EqlRequestParserTests.java +++ b/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/action/EqlRequestParserTests.java @@ -42,18 +42,13 @@ public class EqlRequestParserTests extends ESTestCase { EqlSearchRequest::fromXContent); assertParsingErrorMessage("{\"query\" : \"whatever\", \"size\":\"abc\"}", "failed to parse field [size]", EqlSearchRequest::fromXContent); - assertParsingErrorMessage("{\"case_sensitive\" : \"whatever\"}", "failed to parse field [case_sensitive]", - EqlSearchRequest::fromXContent); - boolean setIsCaseSensitive = randomBoolean(); - boolean isCaseSensitive = randomBoolean(); EqlSearchRequest request = generateRequest("endgame-*", "{\"filter\" : {\"match\" : {\"foo\":\"bar\"}}, " + "\"timestamp_field\" : \"tsf\", " + "\"event_category_field\" : \"etf\"," + "\"size\" : \"101\"," - + "\"query\" : \"file where user != 'SYSTEM' by file_path\"" - + (setIsCaseSensitive ? (",\"case_sensitive\" : " + isCaseSensitive) : "") - + "}", EqlSearchRequest::fromXContent); + + "\"query\" : \"file where user != 'SYSTEM' by file_path\"}" + , EqlSearchRequest::fromXContent); assertArrayEquals(new String[]{"endgame-*"}, request.indices()); assertNotNull(request.query()); assertTrue(request.filter() instanceof MatchQueryBuilder); @@ -65,7 +60,6 @@ public class EqlRequestParserTests extends ESTestCase { assertEquals(101, request.size()); assertEquals(1000, request.fetchSize()); assertEquals("file where user != 'SYSTEM' by file_path", request.query()); - assertEquals(setIsCaseSensitive && isCaseSensitive, request.isCaseSensitive()); } private EqlSearchRequest generateRequest(String index, String json, Function fromXContent) diff --git a/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/analysis/VerifierTests.java b/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/analysis/VerifierTests.java index bb4dad8690b..a2fbe2ca986 100644 --- a/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/analysis/VerifierTests.java +++ b/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/analysis/VerifierTests.java @@ -37,7 +37,7 @@ public class VerifierTests extends ESTestCase { private LogicalPlan accept(IndexResolution resolution, String eql) { PreAnalyzer preAnalyzer = new PreAnalyzer(); - Analyzer analyzer = new Analyzer(EqlTestUtils.TEST_CFG_CASE_INSENSITIVE, new EqlFunctionRegistry(), new Verifier(new Metrics())); + Analyzer analyzer = new Analyzer(EqlTestUtils.TEST_CFG, new EqlFunctionRegistry(), new Verifier(new Metrics())); return analyzer.analyze(preAnalyzer.preAnalyze(parser.createStatement(eql), resolution)); } diff --git a/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/expression/function/scalar/string/BetweenFunctionPipeTests.java b/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/expression/function/scalar/string/BetweenFunctionPipeTests.java index 8e93d64f6d3..8a86c4d90d4 100644 --- a/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/expression/function/scalar/string/BetweenFunctionPipeTests.java +++ b/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/expression/function/scalar/string/BetweenFunctionPipeTests.java @@ -39,8 +39,7 @@ public class BetweenFunctionPipeTests extends AbstractNodeTestCase new EndsWith(EMPTY, l(5), l("foo"), caseInsensitive).makePipe().asProcessor().process(null)); + () -> new EndsWith(EMPTY, l(5), l("foo"), config).makePipe().asProcessor().process(null)); assertEquals("A string/char is required; received [5]", siae.getMessage()); siae = expectThrows(QlIllegalArgumentException.class, - () -> new EndsWith(EMPTY, l("bar"), l(false), caseInsensitive).makePipe().asProcessor().process(null)); + () -> new EndsWith(EMPTY, l("bar"), l(false), config).makePipe().asProcessor().process(null)); assertEquals("A string/char is required; received [false]", siae.getMessage()); } public void testEndsWithFunctionWithRandomInvalidDataType() { + Configuration config = randomConfiguration(); Literal literal = randomValueOtherThanMany(v -> v.dataType() == KEYWORD, () -> LiteralTests.randomLiteral()); QlIllegalArgumentException siae = expectThrows(QlIllegalArgumentException.class, - () -> new EndsWith(EMPTY, literal, l("foo"), caseInsensitive).makePipe().asProcessor().process(null)); + () -> new EndsWith(EMPTY, literal, l("foo"), config).makePipe().asProcessor().process(null)); assertThat(siae.getMessage(), startsWith("A string/char is required; received")); siae = expectThrows(QlIllegalArgumentException.class, - () -> new EndsWith(EMPTY, l("foo"), literal, caseInsensitive).makePipe().asProcessor().process(null)); + () -> new EndsWith(EMPTY, l("foo"), literal, config).makePipe().asProcessor().process(null)); assertThat(siae.getMessage(), startsWith("A string/char is required; received")); } } diff --git a/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/expression/function/scalar/string/IndexOfFunctionProcessorTests.java b/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/expression/function/scalar/string/IndexOfFunctionProcessorTests.java index 8b74e94b235..3439f5ba940 100644 --- a/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/expression/function/scalar/string/IndexOfFunctionProcessorTests.java +++ b/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/expression/function/scalar/string/IndexOfFunctionProcessorTests.java @@ -11,8 +11,9 @@ import org.elasticsearch.xpack.ql.QlIllegalArgumentException; import org.elasticsearch.xpack.ql.expression.Literal; import org.elasticsearch.xpack.ql.expression.LiteralTests; import org.elasticsearch.xpack.ql.session.Configuration; +import org.junit.Assume; -import static org.elasticsearch.xpack.eql.EqlTestUtils.randomConfigurationWithCaseSensitive; +import static org.elasticsearch.xpack.eql.EqlTestUtils.randomConfiguration; import static org.elasticsearch.xpack.ql.expression.function.scalar.FunctionTestUtils.l; import static org.elasticsearch.xpack.ql.tree.Source.EMPTY; import static org.elasticsearch.xpack.ql.type.DataTypes.KEYWORD; @@ -20,9 +21,10 @@ import static org.hamcrest.Matchers.startsWith; public class IndexOfFunctionProcessorTests extends ESTestCase { - final Configuration caseInsensitive = randomConfigurationWithCaseSensitive(false); - public void testIndexOfFunctionWithValidInputInsensitive() { + Assume.assumeTrue(false); //TODO: revisit after we decide on functions case sensitivity handling + + final Configuration caseInsensitive = randomConfiguration(); assertEquals(5, new IndexOf(EMPTY, l("foobarbar"), l("r"), l(null), caseInsensitive).makePipe().asProcessor().process(null)); assertEquals(5, new IndexOf(EMPTY, l("foobaRbar"), l("r"), l(null), caseInsensitive).makePipe().asProcessor().process(null)); assertEquals(0, new IndexOf(EMPTY, l("foobar"), l("Foo"), l(null), caseInsensitive).makePipe().asProcessor().process(null)); @@ -44,7 +46,7 @@ public class IndexOfFunctionProcessorTests extends ESTestCase { } public void testIndexOfFunctionWithValidInputSensitive() { - final Configuration caseSensitive = randomConfigurationWithCaseSensitive(true); + final Configuration caseSensitive = randomConfiguration(); assertEquals(5, new IndexOf(EMPTY, l("foobarbar"), l("r"), l(null), caseSensitive).makePipe().asProcessor().process(null)); assertEquals(8, new IndexOf(EMPTY, l("foobaRbar"), l("r"), l(null), caseSensitive).makePipe().asProcessor().process(null)); assertEquals(4, new IndexOf(EMPTY, l("foobARbar"), l("AR"), l(null), caseSensitive).makePipe().asProcessor().process(null)); @@ -67,30 +69,32 @@ public class IndexOfFunctionProcessorTests extends ESTestCase { } public void testIndexOfFunctionInputsValidation() { + Configuration config = randomConfiguration(); QlIllegalArgumentException siae = expectThrows(QlIllegalArgumentException.class, - () -> new IndexOf(EMPTY, l(5), l("foo"), l(null), caseInsensitive).makePipe().asProcessor().process(null)); + () -> new IndexOf(EMPTY, l(5), l("foo"), l(null), config).makePipe().asProcessor().process(null)); assertEquals("A string/char is required; received [5]", siae.getMessage()); siae = expectThrows(QlIllegalArgumentException.class, - () -> new IndexOf(EMPTY, l("bar"), l(false), l(2), caseInsensitive).makePipe().asProcessor().process(null)); + () -> new IndexOf(EMPTY, l("bar"), l(false), l(2), config).makePipe().asProcessor().process(null)); assertEquals("A string/char is required; received [false]", siae.getMessage()); siae = expectThrows(QlIllegalArgumentException.class, - () -> new IndexOf(EMPTY, l("bar"), l("a"), l("1"), caseInsensitive).makePipe().asProcessor().process(null)); + () -> new IndexOf(EMPTY, l("bar"), l("a"), l("1"), config).makePipe().asProcessor().process(null)); assertEquals("A number is required; received [1]", siae.getMessage()); } public void testIndexOfFunctionWithRandomInvalidDataType() { + Configuration config = randomConfiguration(); Literal stringLiteral = randomValueOtherThanMany(v -> v.dataType() == KEYWORD, () -> LiteralTests.randomLiteral()); QlIllegalArgumentException siae = expectThrows(QlIllegalArgumentException.class, - () -> new IndexOf(EMPTY, stringLiteral, l("foo"), l(1), caseInsensitive).makePipe().asProcessor().process(null)); + () -> new IndexOf(EMPTY, stringLiteral, l("foo"), l(1), config).makePipe().asProcessor().process(null)); assertThat(siae.getMessage(), startsWith("A string/char is required; received")); siae = expectThrows(QlIllegalArgumentException.class, - () -> new IndexOf(EMPTY, l("foo"), stringLiteral, l(2), caseInsensitive).makePipe().asProcessor().process(null)); + () -> new IndexOf(EMPTY, l("foo"), stringLiteral, l(2), config).makePipe().asProcessor().process(null)); assertThat(siae.getMessage(), startsWith("A string/char is required; received")); Literal numericLiteral = randomValueOtherThanMany(v -> v.dataType().isNumeric(), () -> LiteralTests.randomLiteral()); siae = expectThrows(QlIllegalArgumentException.class, - () -> new IndexOf(EMPTY, l("foo"), l("o"), numericLiteral, caseInsensitive).makePipe().asProcessor().process(null)); + () -> new IndexOf(EMPTY, l("foo"), l("o"), numericLiteral, config).makePipe().asProcessor().process(null)); assertThat(siae.getMessage(), startsWith("A number is required; received")); } } diff --git a/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/expression/function/scalar/string/StartsWithFunctionProcessorTests.java b/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/expression/function/scalar/string/StartsWithFunctionProcessorTests.java index b3c652648fe..04367df2ae4 100644 --- a/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/expression/function/scalar/string/StartsWithFunctionProcessorTests.java +++ b/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/expression/function/scalar/string/StartsWithFunctionProcessorTests.java @@ -13,18 +13,18 @@ import org.elasticsearch.xpack.ql.tree.Source; import java.util.function.Supplier; -import static org.elasticsearch.xpack.eql.EqlTestUtils.randomConfigurationWithCaseSensitive; +import static org.elasticsearch.xpack.eql.EqlTestUtils.randomConfiguration; public class StartsWithFunctionProcessorTests extends org.elasticsearch.xpack.ql.expression.function.scalar.string.StartsWithProcessorTests{ @Override protected Supplier isCaseSensitiveGenerator() { - return () -> randomBoolean(); + return () -> Boolean.TRUE; } @Override protected Supplier configurationGenerator() { - return () -> randomConfigurationWithCaseSensitive(isCaseSensitive); + return () -> randomConfiguration(); } @Override diff --git a/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/optimizer/EqlFoldSpec.java b/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/optimizer/EqlFoldSpec.java index 85003a2343f..215879eba36 100644 --- a/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/optimizer/EqlFoldSpec.java +++ b/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/optimizer/EqlFoldSpec.java @@ -16,16 +16,10 @@ public class EqlFoldSpec { private final String description; private final String expression; private final Object expected; - // flag to dictate which modes are supported for the test - // null -> apply the test to both modes (case sensitive and case insensitive) - // TRUE -> case sensitive - // FALSE -> case insensitive - private Boolean caseSensitive = null; - EqlFoldSpec(String name, String description, Boolean caseSensitive, String expression, Object expected) { + EqlFoldSpec(String name, String description, String expression, Object expected) { this.name = name; this.description = description; - this.caseSensitive = caseSensitive; this.expression = expression; this.expected = expected; } @@ -38,15 +32,10 @@ public class EqlFoldSpec { return expected; } - public Boolean caseSensitive() { - return caseSensitive; - } - public String toString() { StringBuilder sb = new StringBuilder(); appendWithComma(sb, "name", name); appendWithComma(sb, "expression", expression); - appendWithComma(sb, "case_sensitive", caseSensitive == null ? "null" : caseSensitive); appendWithComma(sb, "expected", expected == null ? "null" : expected); return sb.toString(); } @@ -78,12 +67,11 @@ public class EqlFoldSpec { EqlFoldSpec that = (EqlFoldSpec) other; - return Objects.equals(this.expression, that.expression) - && Objects.equals(this.caseSensitive, that.caseSensitive); + return Objects.equals(this.expression, that.expression); } @Override public int hashCode() { - return Objects.hash(this.expression, this.caseSensitive); + return Objects.hash(this.expression); } } diff --git a/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/optimizer/EqlFoldSpecLoader.java b/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/optimizer/EqlFoldSpecLoader.java index 3bbf8352a59..16709dc8957 100644 --- a/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/optimizer/EqlFoldSpecLoader.java +++ b/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/optimizer/EqlFoldSpecLoader.java @@ -40,20 +40,6 @@ public class EqlFoldSpecLoader { TomlTable fold = table.getTomlTable("fold"); String description = getTrimmedString(table, "description"); - Boolean cs = null; - Boolean caseSensitive = table.getBoolean("case_sensitive"); - Boolean caseInsensitive = table.getBoolean("case_insensitive"); - // if case_sensitive is TRUE and case_insensitive is not TRUE (FALSE or NULL), then the test is case sensitive only - if (Boolean.TRUE.equals(caseSensitive)) { - if (Boolean.FALSE.equals(caseInsensitive) || caseInsensitive == null) { - cs = true; - } - } - // if case_sensitive is not TRUE (FALSE or NULL) and case_insensitive is TRUE, then the test is case insensitive only - else if (Boolean.TRUE.equals(caseInsensitive)) { - cs = false; - } - // in all other cases, the test should run no matter the case sensitivity (should test both scenarios) if (fold != null) { List tests = fold.getArrayTable("tests"); @@ -61,7 +47,7 @@ public class EqlFoldSpecLoader { for (TomlTable test : tests) { String expression = getTrimmedString(test, "expression"); Object expected = test.get("expected"); - EqlFoldSpec spec = new EqlFoldSpec(name, description, cs, expression, expected); + EqlFoldSpec spec = new EqlFoldSpec(name, description, expression, expected); testSpecs.add(spec); } } diff --git a/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/optimizer/OptimizerTests.java b/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/optimizer/OptimizerTests.java index c94d89c9554..556e75a9278 100644 --- a/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/optimizer/OptimizerTests.java +++ b/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/optimizer/OptimizerTests.java @@ -60,7 +60,7 @@ import static java.util.Collections.emptyList; import static java.util.Collections.emptyMap; import static java.util.Collections.singletonList; import static java.util.stream.Collectors.toList; -import static org.elasticsearch.xpack.eql.EqlTestUtils.TEST_CFG_CASE_INSENSITIVE; +import static org.elasticsearch.xpack.eql.EqlTestUtils.TEST_CFG; import static org.elasticsearch.xpack.ql.TestUtils.UTC; import static org.elasticsearch.xpack.ql.expression.Literal.TRUE; import static org.elasticsearch.xpack.ql.tree.Source.EMPTY; @@ -84,9 +84,9 @@ public class OptimizerTests extends ESTestCase { private LogicalPlan accept(IndexResolution resolution, String eql) { PreAnalyzer preAnalyzer = new PreAnalyzer(); PostAnalyzer postAnalyzer = new PostAnalyzer(); - Analyzer analyzer = new Analyzer(TEST_CFG_CASE_INSENSITIVE, new EqlFunctionRegistry(), new Verifier(new Metrics())); + Analyzer analyzer = new Analyzer(TEST_CFG, new EqlFunctionRegistry(), new Verifier(new Metrics())); return optimizer.optimize(postAnalyzer.postAnalyze(analyzer.analyze(preAnalyzer.preAnalyze(parser.createStatement(eql), - resolution)), TEST_CFG_CASE_INSENSITIVE)); + resolution)), TEST_CFG)); } private LogicalPlan accept(String eql) { diff --git a/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/optimizer/TomlFoldTests.java b/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/optimizer/TomlFoldTests.java index 245c87b4f9b..4571cc937ed 100644 --- a/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/optimizer/TomlFoldTests.java +++ b/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/optimizer/TomlFoldTests.java @@ -29,8 +29,7 @@ import java.util.concurrent.atomic.AtomicInteger; import static java.util.Collections.emptyList; import static java.util.Collections.singletonList; import static java.util.stream.Collectors.toList; -import static org.elasticsearch.xpack.eql.EqlTestUtils.TEST_CFG_CASE_INSENSITIVE; -import static org.elasticsearch.xpack.eql.EqlTestUtils.TEST_CFG_CASE_SENSITIVE; +import static org.elasticsearch.xpack.eql.EqlTestUtils.TEST_CFG; import static org.elasticsearch.xpack.ql.tree.Source.EMPTY; public class TomlFoldTests extends ESTestCase { @@ -40,8 +39,7 @@ public class TomlFoldTests extends ESTestCase { private static EqlParser parser = new EqlParser(); private static final EqlFunctionRegistry functionRegistry = new EqlFunctionRegistry(); private static Verifier verifier = new Verifier(new Metrics()); - private static Analyzer caseSensitiveAnalyzer = new Analyzer(TEST_CFG_CASE_SENSITIVE, functionRegistry, verifier); - private static Analyzer caseInsensitiveAnalyzer = new Analyzer(TEST_CFG_CASE_INSENSITIVE, functionRegistry, verifier); + private static Analyzer analyzer = new Analyzer(TEST_CFG, functionRegistry, verifier); private final int num; private final EqlFoldSpec spec; @@ -70,30 +68,6 @@ public class TomlFoldTests extends ESTestCase { } public void test() { - // run both tests if case sensitivity doesn't matter - if (spec.caseSensitive() == null) { - testCaseSensitive(spec); - testCaseInsensitive(spec); - } - // run only the case sensitive test - else if (spec.caseSensitive()) { - testCaseSensitive(spec); - } - // run only the case insensitive test - else { - testCaseInsensitive(spec); - } - } - - private void testCaseSensitive(EqlFoldSpec spec) { - testWithAnalyzer(caseSensitiveAnalyzer, spec); - } - - private void testCaseInsensitive(EqlFoldSpec spec) { - testWithAnalyzer(caseInsensitiveAnalyzer, spec); - } - - private void testWithAnalyzer(Analyzer analyzer, EqlFoldSpec spec) { Expression expr = parser.createExpression(spec.expression()); LogicalPlan logicalPlan = new Project(EMPTY, new LocalRelation(EMPTY, emptyList()), singletonList(new Alias(Source.EMPTY, "test", expr))); diff --git a/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/planner/QueryFolderFailTests.java b/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/planner/QueryFolderFailTests.java index 356036abcf0..7e53a07c0dc 100644 --- a/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/planner/QueryFolderFailTests.java +++ b/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/planner/QueryFolderFailTests.java @@ -33,13 +33,13 @@ public class QueryFolderFailTests extends AbstractQueryFolderTestCase { "process where between(process_name, \"s\") == \"yst\"", "process where between(null) == \"yst\"", "process where between(process_name, null) == \"yst\"", - "process where between(process_name, \"s\", \"e\", false, false, true) == \"yst\"", + "process where between(process_name, \"s\", \"e\", false, false) == \"yst\"", }; for (String query : queries) { ParsingException e = expectThrows(ParsingException.class, () -> plan(query)); - assertEquals("line 1:16: error building [between]: expects between three and five arguments", e.getMessage()); + assertEquals("line 1:16: error building [between]: expects three or four arguments", e.getMessage()); } } @@ -56,10 +56,6 @@ public class QueryFolderFailTests extends AbstractQueryFolderTestCase { assertEquals("1:15: fourth argument of [between(process_name, \"s\", \"e\", \"true\")] must be [boolean], " + "found value [\"true\"] type [keyword]", error("process where between(process_name, \"s\", \"e\", \"true\")")); - - assertEquals("1:15: fifth argument of [between(process_name, \"s\", \"e\", false, 2)] must be [boolean], " + - "found value [2] type [integer]", - error("process where between(process_name, \"s\", \"e\", false, 2)")); } public void testCIDRMatchAgainstField() { diff --git a/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/planner/QueryFolderOkTests.java b/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/planner/QueryFolderOkTests.java index 68f8705f436..187114a1f2a 100644 --- a/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/planner/QueryFolderOkTests.java +++ b/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/planner/QueryFolderOkTests.java @@ -10,14 +10,12 @@ import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; import org.elasticsearch.common.Strings; import org.elasticsearch.xpack.eql.plan.physical.EsQueryExec; import org.elasticsearch.xpack.eql.plan.physical.PhysicalPlan; -import org.junit.Assume; import java.io.BufferedReader; import java.io.InputStreamReader; import java.nio.charset.StandardCharsets; import java.util.ArrayList; import java.util.LinkedHashMap; -import java.util.Locale; import java.util.Map; import static org.hamcrest.Matchers.containsString; @@ -110,13 +108,6 @@ public class QueryFolderOkTests extends AbstractQueryFolderTestCase { } public void test() { - String testName = name.toLowerCase(Locale.ROOT); - // skip tests that do not make sense from case sensitivity point of view - boolean isCaseSensitiveValidTest = testName.endsWith("sensitive") == false - || testName.endsWith("-casesensitive") && configuration.isCaseSensitive() - || testName.endsWith("-caseinsensitive") && configuration.isCaseSensitive() == false; - Assume.assumeTrue(isCaseSensitiveValidTest); - PhysicalPlan p = plan(query); assertEquals(EsQueryExec.class, p.getClass()); EsQueryExec eqe = (EsQueryExec) p; diff --git a/x-pack/plugin/eql/src/test/resources/queries-supported.eql b/x-pack/plugin/eql/src/test/resources/queries-supported.eql index 60db293cb4f..684372755a2 100644 --- a/x-pack/plugin/eql/src/test/resources/queries-supported.eql +++ b/x-pack/plugin/eql/src/test/resources/queries-supported.eql @@ -23,6 +23,8 @@ registry where not pid; process where process_name : "net.exe" and command_line : "* user*.exe"; +process where process_name : "net.EXE" and command_line : "* user*.EXE"; + process where command_line : "~!@#$%^&*();'[]{}\\|<>?,./:\"-= ' "; process where @@ -33,6 +35,8 @@ process where process_name in ("net.exe", "cmd.exe", "at.exe"); process where command_line : "*.exe *admin*" or command_line : "* a b*"; +process where command_line : "*.eXe *aDMin*" or command_line : "* a b*"; + process where pid in (1,2,3,4,5,6,7,8) and abc : 100 and def : 200 and ghi : 300 and jkl : x; process where ppid != pid; diff --git a/x-pack/plugin/eql/src/test/resources/queryfolder_tests.txt b/x-pack/plugin/eql/src/test/resources/queryfolder_tests.txt index ab823ec36bb..08a52520807 100644 --- a/x-pack/plugin/eql/src/test/resources/queryfolder_tests.txt +++ b/x-pack/plugin/eql/src/test/resources/queryfolder_tests.txt @@ -10,6 +10,8 @@ // ... // // ; +// NOTE: do not use whitespaces in the relevant parts of the query itself, as they will be removed. For example 'process where process_name : "System Idle Process"' +// The whitespaces between System Idle Process will be removed and the assertion will fail. basic @@ -21,7 +23,7 @@ null singleNumericFilterEquals process where serial_event_id == 1 ; -"term":{"serial_event_id":{"value":1 +"term":{"serial_event_id":{"value":1,"boost":1.0} ; singleNumericFilterLess @@ -48,10 +50,16 @@ process where serial_event_id >= 4 "range":{"serial_event_id":{"from":4,"to":null,"include_lower":true,"include_upper":false ; +caseInsensitiveEquals +process where process_name : "test" +; +"term":{"process_name":{"value":"test","case_insensitive":true,"boost":1.0} +; + mixedTypeFilter process where process_name : "notepad.exe" or (serial_event_id < 4.5 and serial_event_id >= 3.1) ; -"term":{"process_name":{"value":"notepad.exe" +"term":{"process_name":{"value":"notepad.exe","case_insensitive":true,"boost":1.0} "range":{"serial_event_id":{"from":3.1,"to":4.5,"include_lower":true,"include_upper":false ; @@ -70,35 +78,35 @@ process where process_name in ("python.exe", "SMSS.exe", "explorer.exe") equalsAndInFilter process where process_path : "*\\red_ttp\\wininit.*" and opcode in (0,1,2,3) ; -"wildcard":{"process_path":{"wildcard":"*\\\\red_ttp\\\\wininit.*" -{"terms":{"opcode":[0,1,2,3] +"wildcard":{"process_path":{"wildcard":"*\\\\red_ttp\\\\wininit.*","case_insensitive":true,"boost":1.0}}}, +{"terms":{"opcode":[0,1,2,3],"boost":1.0}} ; functionEqualsTrue process where cidrMatch(source_address, "10.0.0.0/8") == true ; -{"bool":{"must":[{"term":{"event.category":{"value":"process" +{"bool":{"must":[{"term":{"event.category":{"value":"process","boost":1.0} {"terms":{"source_address":["10.0.0.0/8"] ; functionEqualsFalse process where cidrMatch(source_address, "10.0.0.0/8") == false ; -{"bool":{"must":[{"term":{"event.category":{"value":"process" +{"bool":{"must":[{"term":{"event.category":{"value":"process","boost":1.0} {"bool":{"must_not":[{"terms":{"source_address":["10.0.0.0/8"] ; functionNotEqualsTrue process where cidrMatch(source_address, "10.0.0.0/8") != true ; -{"bool":{"must":[{"term":{"event.category":{"value":"process" +{"bool":{"must":[{"term":{"event.category":{"value":"process","boost":1.0} {"bool":{"must_not":[{"terms":{"source_address":["10.0.0.0/8"] ; functionNotEqualsFalse process where cidrMatch(source_address, "10.0.0.0/8") != false ; -{"bool":{"must":[{"term":{"event.category":{"value":"process" +{"bool":{"must":[{"term":{"event.category":{"value":"process","boost":1.0} {"terms":{"source_address":["10.0.0.0/8"] ; @@ -112,13 +120,8 @@ process where endsWith(process_path, "x") == true and endsWith(process_path, "yx twoFunctionsEqualsBooleanLiterals-caseInsensitive process where endsWith(process_path, "x") == true and endsWith(process_path, "yx") != true ; -"bool":{"must":[{"term":{"event.category":{"value":"process" -"must":[{"script":{"script":{"source":"InternalQlScriptUtils.nullSafeFilter( -InternalEqlScriptUtils.endsWith(InternalQlScriptUtils.docValue(doc,params.v0),params.v1,params.v2))","lang":"painless", -"params":{"v0":"process_path","v1":"x","v2":false}} -"script":{"script":{"source":"InternalQlScriptUtils.nullSafeFilter(InternalQlScriptUtils.not( -InternalEqlScriptUtils.endsWith(InternalQlScriptUtils.docValue(doc,params.v0),params.v1,params.v2)))","lang":"painless", -"params":{"v0":"process_path","v1":"yx","v2":false}} +{"bool":{"must":[{"wildcard":{"process_path":{"wildcard":"*x","boost":1.0}}}, +{"bool":{"must_not":[{"wildcard":{"process_path":{"wildcard":"*yx","boost":1.0}}}],"adjust_pure_negative":true,"boost":1.0}}],"adjust_pure_negative":true,"boost":1.0}} ; endsWithKeywordFieldFunction-caseSensitive @@ -138,9 +141,8 @@ process where endsWith(hostname, "c") endsWithFunction-caseInsensitive process where endsWith(user_name, "c") ; -"script":{"source":"InternalQlScriptUtils.nullSafeFilter(InternalEqlScriptUtils.endsWith( -InternalQlScriptUtils.docValue(doc,params.v0),params.v1,params.v2))", -"params":{"v0":"user_name","v1":"c","v2":false}} +{"bool":{"must":[{"term":{"event.category":{"value":"process","boost":1.0}}}, +{"wildcard":{"user_name":{"wildcard":"*c","boost":1.0}}}],"adjust_pure_negative":true,"boost":1.0}} ; lengthFunctionWithExactSubField @@ -170,9 +172,8 @@ InternalEqlScriptUtils.length(InternalQlScriptUtils.docValue(doc,params.v0)),par startsWithFunction-caseInsensitive process where startsWith(user_name, "A") ; -"script":{"source":"InternalQlScriptUtils.nullSafeFilter(InternalQlScriptUtils.startsWith( -InternalQlScriptUtils.docValue(doc,params.v0),params.v1,params.v2))", -"params":{"v0":"user_name","v1":"A","v2":false}} +{"bool":{"must":[{"term":{"event.category":{"value":"process","boost":1.0}}}, +{"prefix":{"user_name":{"value":"A","boost":1.0}}}],"adjust_pure_negative":true,"boost":1.0}} ; startsWithFunctionSimple-caseSensitive @@ -207,9 +208,8 @@ process where stringContains(hostname, "foo") stringContains-caseInsensitive process where stringContains(process_name, "foo") ; -"script":{"source":"InternalQlScriptUtils.nullSafeFilter(InternalEqlScriptUtils.stringContains( -InternalQlScriptUtils.docValue(doc,params.v0),params.v1,params.v2))" -"params":{"v0":"process_name","v1":"foo","v2":false} +{"bool":{"must":[{"term":{"event.category":{"value":"process","boost":1.0}}}, +{"wildcard":{"process_name":{"wildcard":"*foo*","boost":1.0}}}],"adjust_pure_negative":true,"boost":1.0}} ; @@ -234,7 +234,7 @@ process where indexOf(user_name, "A", 2) > 0 ; "script":{"source":"InternalQlScriptUtils.nullSafeFilter(InternalQlScriptUtils.gt( InternalEqlScriptUtils.indexOf(InternalQlScriptUtils.docValue(doc,params.v0),params.v1,params.v2,params.v3),params.v4))", -"params":{"v0":"user_name","v1":"A","v2":2,"v3":false,"v4":0} +"params":{"v0":"user_name","v1":"A","v2":2,"v3":true,"v4":0} ; substringFunction @@ -250,7 +250,7 @@ process where between(process_name, "s", "e") : "yst" ; "script":{"source":"InternalQlScriptUtils.nullSafeFilter(InternalEqlScriptUtils.seq( InternalEqlScriptUtils.between(InternalQlScriptUtils.docValue(doc,params.v0),params.v1,params.v2,params.v3,params.v4),params.v5))", -"params":{"v0":"process_name","v1":"s","v2":"e","v3":false,"v4":false,"v5":"yst"} +"params":{"v0":"process_name","v1":"s","v2":"e","v3":false,"v4":true,"v5":"yst"} ; concatFunction @@ -264,35 +264,35 @@ InternalEqlScriptUtils.concat([InternalQlScriptUtils.docValue(doc,params.v0),par cidrMatchFunctionOne process where cidrMatch(source_address, "10.0.0.0/8") ; -{"bool":{"must":[{"term":{"event.category":{"value":"process" +{"bool":{"must":[{"term":{"event.category":{"value":"process","boost":1.0} {"terms":{"source_address":["10.0.0.0/8"] ; cidrMatchFunctionOneBool process where cidrMatch(source_address, "10.0.0.0/8") == true ; -{"bool":{"must":[{"term":{"event.category":{"value":"process" +{"bool":{"must":[{"term":{"event.category":{"value":"process","boost":1.0} {"terms":{"source_address":["10.0.0.0/8"] ; cidrMatchFunctionTwo process where cidrMatch(source_address, "10.0.0.0/8", "192.168.0.0/16") ; -{"bool":{"must":[{"term":{"event.category":{"value":"process" +{"bool":{"must":[{"term":{"event.category":{"value":"process","boost":1.0} {"terms":{"source_address":["10.0.0.0/8","192.168.0.0/16"] ; cidrMatchFunctionTwoWithOr process where cidrMatch(source_address, "10.0.0.0/8") or cidrMatch(source_address, "192.168.0.0/16") ; -{"bool":{"must":[{"term":{"event.category":{"value":"process" +{"bool":{"must":[{"term":{"event.category":{"value":"process","boost":1.0} {"bool":{"should":[{"terms":{"source_address":["10.0.0.0/8"],"boost":1.0}},{"terms":{"source_address":["192.168.0.0/16"],"boost":1.0}} ; cidrMatchFunctionThree process where cidrMatch(source_address, "10.0.0.0/8", "192.168.0.0/16", "2001:db8::/32") ; -{"bool":{"must":[{"term":{"event.category":{"value":"process" +{"bool":{"must":[{"term":{"event.category":{"value":"process","boost":1.0} {"terms":{"source_address":["10.0.0.0/8","192.168.0.0/16","2001:db8::/32"] ; @@ -351,34 +351,34 @@ InternalEqlScriptUtils.number(InternalQlScriptUtils.docValue(doc,params.v0),para numberFunctionFoldedComparison process where serial_event_id == number("-32.5"); -{"term":{"serial_event_id":{"value":-32.5, +{"term":{"serial_event_id":{"value":-32.5,"boost":1.0} ; numberFunctionFoldedHexComparison process where serial_event_id == number("0x32", 16); -{"term":{"serial_event_id":{"value":50, +{"term":{"serial_event_id":{"value":50,"boost":1.0} ; wildcardFunctionSingleArgument process where wildcard(process_path, "*\\red_ttp\\wininit.*") ; -"wildcard":{"process_path":{"wildcard":"*\\\\red_ttp\\\\wininit.*" +"wildcard":{"process_path":{"wildcard":"*\\\\red_ttp\\\\wininit.*","boost":1.0}} ; wildcardFunctionTwoArguments process where wildcard(process_path, "*\\red_ttp\\wininit.*", "*\\abc\\*") ; -"wildcard":{"process_path":{"wildcard":"*\\\\red_ttp\\\\wininit.*" -"wildcard":{"process_path":{"wildcard":"*\\\\abc\\\\*" +"wildcard":{"process_path":{"wildcard":"*\\\\red_ttp\\\\wininit.*","boost":1.0}} +"wildcard":{"process_path":{"wildcard":"*\\\\abc\\\\*","boost":1.0}} ; wildcardFunctionThreeArguments process where wildcard(process_path, "*\\red_ttp\\wininit.*", "*\\abc\\*", "*def*") ; -"wildcard":{"process_path":{"wildcard":"*\\\\red_ttp\\\\wininit.*" -"wildcard":{"process_path":{"wildcard":"*\\\\abc\\\\*" -"wildcard":{"process_path":{"wildcard":"*def*" +"wildcard":{"process_path":{"wildcard":"*\\\\red_ttp\\\\wininit.*","boost":1.0}} +"wildcard":{"process_path":{"wildcard":"*\\\\abc\\\\*","boost":1.0}} +"wildcard":{"process_path":{"wildcard":"*def*","boost":1.0}} ; diff --git a/x-pack/plugin/eql/src/test/resources/test_folding.toml b/x-pack/plugin/eql/src/test/resources/test_folding.toml index 119315e5151..ecfb49d5f80 100644 --- a/x-pack/plugin/eql/src/test/resources/test_folding.toml +++ b/x-pack/plugin/eql/src/test/resources/test_folding.toml @@ -210,7 +210,6 @@ description = "Fold string comparisons" [string_case_sensitive_match_comparison] description = "Fold string comparisons" -case_sensitive = true [[string_case_sensitive_match_comparison.fold.tests]] expression = '"Foo" == "foo"' diff --git a/x-pack/plugin/eql/src/test/resources/test_string_functions.toml b/x-pack/plugin/eql/src/test/resources/test_string_functions.toml index c7b059895c3..72cd4ebfd87 100644 --- a/x-pack/plugin/eql/src/test/resources/test_string_functions.toml +++ b/x-pack/plugin/eql/src/test/resources/test_string_functions.toml @@ -104,16 +104,14 @@ description = "Test the `endsWith` function with case matching" [endswith_insensitive] description = "Test the `endsWith` function with case insensitivity" -case_insensitive = true [[endswith_insensitive.fold.tests]] expression = 'endsWith("FooBarBaz", "baz")' - expected = true + expected = false [endswith_sensitive] description = "Test the `endsWith` function with case sensitivity" -case_sensitive = true [[endswith_sensitive.fold.tests]] expression = 'endsWith("FooBarBaz", "baz")' @@ -331,7 +329,6 @@ description = "Test the `startsWith` function" [startswith_case_sensitive] description = "Test the `startsWith` function with case-sensitive matching" -case_sensitive = true [[startswith_case_sensitive.tests]] expression = '''startsWith("FooBar", "Foo")''' @@ -344,7 +341,6 @@ case_sensitive = true [startswith_case_insensitive] description = "Test the `startsWith` function with case-insensitive matching" -case_insensitive = true [[startswith_case_insensitive.fold.tests]] expression = '''startsWith("FooBar", "Foo")''' @@ -352,11 +348,11 @@ case_insensitive = true [[startswith_case_insensitive.fold.tests]] expression = '''startsWith("FooBar", "FOO")''' - expected = true + expected = false [[startswith_case_insensitive.fold.tests]] expression = '''startsWith("FooBar", "foo")''' - expected = true + expected = false [[startswith_case_insensitive.fold.tests]] expression = '''startsWith("FooBar", "Bar")''' @@ -498,11 +494,10 @@ description = "Test that `wildcard` folds with correct case matches." [wildcard_case_insensitive] description = "Test that `wildcard` function folds case insensitive as expected." -case_insensitive = true [[wildcard_case_insensitive.fold.tests]] expression = 'wildcard("FOO", "f*o*o*")' - expected = true + expected = false [[wildcard_case_insensitive.fold.tests]] expression = 'wildcard("bar", "f*o*o*")' @@ -511,7 +506,6 @@ case_insensitive = true [wildcard_case_sensitive] description = "Test that `wildcard` folds case-sensitive matches." -case_sensitive = true [[wildcard_case_sensitive.fold.tests]] expression = 'wildcard("Foo", "F*o*o*")' diff --git a/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/expression/function/FunctionRegistry.java b/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/expression/function/FunctionRegistry.java index 08f7547c7c3..14fd3a64ff8 100644 --- a/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/expression/function/FunctionRegistry.java +++ b/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/expression/function/FunctionRegistry.java @@ -420,13 +420,16 @@ public class FunctionRegistry { public static FunctionDefinition def(Class function, FourParametersFunctionBuilder ctorRef, String... names) { FunctionBuilder builder = (source, children, distinct, cfg) -> { - if (children.size() != 4) { + boolean hasMinimumThree = OptionalArgument.class.isAssignableFrom(function); + if (hasMinimumThree && (children.size() > 4 || children.size() < 3)) { + throw new QlIllegalArgumentException("expects three or four arguments"); + } else if (!hasMinimumThree && children.size() != 4) { throw new QlIllegalArgumentException("expects exactly four arguments"); } if (distinct) { throw new QlIllegalArgumentException("does not support DISTINCT yet it was specified"); } - return ctorRef.build(source, children.get(0), children.get(1), children.get(2), children.get(3)); + return ctorRef.build(source, children.get(0), children.get(1), children.get(2), children.size() == 4 ? children.get(3) : null); }; return def(function, builder, false, names); } diff --git a/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/expression/predicate/regex/Like.java b/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/expression/predicate/regex/Like.java index dfde94b5df7..2e758975311 100644 --- a/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/expression/predicate/regex/Like.java +++ b/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/expression/predicate/regex/Like.java @@ -9,19 +9,43 @@ import org.elasticsearch.xpack.ql.expression.Expression; import org.elasticsearch.xpack.ql.tree.NodeInfo; import org.elasticsearch.xpack.ql.tree.Source; +import java.util.Objects; + public class Like extends RegexMatch { + private final boolean caseInsensitive; + public Like(Source source, Expression left, LikePattern pattern) { + this(source, left, pattern, false); + } + + public Like(Source source, Expression left, LikePattern pattern, boolean caseInsensitive) { super(source, left, pattern); + this.caseInsensitive = caseInsensitive; } @Override protected NodeInfo info() { - return NodeInfo.create(this, Like::new, field(), pattern()); + return NodeInfo.create(this, Like::new, field(), pattern(), caseInsensitive()); } @Override protected Like replaceChild(Expression newLeft) { - return new Like(source(), newLeft, pattern()); + return new Like(source(), newLeft, pattern(), caseInsensitive()); } + + public boolean caseInsensitive() { + return caseInsensitive; + } + + @Override + public boolean equals(Object obj) { + return super.equals(obj) && Objects.equals(((Like) obj).caseInsensitive(), caseInsensitive()); + } + + @Override + public int hashCode() { + return Objects.hash(super.hashCode(), caseInsensitive()); + } + } diff --git a/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/planner/ExpressionTranslators.java b/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/planner/ExpressionTranslators.java index 0562bc4fabb..43181afb305 100644 --- a/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/planner/ExpressionTranslators.java +++ b/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/planner/ExpressionTranslators.java @@ -32,7 +32,6 @@ import org.elasticsearch.xpack.ql.expression.predicate.operator.comparison.LessT import org.elasticsearch.xpack.ql.expression.predicate.operator.comparison.NotEquals; import org.elasticsearch.xpack.ql.expression.predicate.operator.comparison.NullEquals; import org.elasticsearch.xpack.ql.expression.predicate.regex.Like; -import org.elasticsearch.xpack.ql.expression.predicate.regex.LikePattern; import org.elasticsearch.xpack.ql.expression.predicate.regex.RLike; import org.elasticsearch.xpack.ql.expression.predicate.regex.RegexMatch; import org.elasticsearch.xpack.ql.querydsl.query.BoolQuery; @@ -127,8 +126,8 @@ public final class ExpressionTranslators { if (e.field() instanceof FieldAttribute) { targetFieldName = handler.nameOf(((FieldAttribute) e.field()).exactAttribute()); if (e instanceof Like) { - LikePattern p = ((Like) e).pattern(); - q = new WildcardQuery(e.source(), targetFieldName, p.asLuceneWildcard()); + Like l = (Like) e; + q = new WildcardQuery(e.source(), targetFieldName, l.pattern().asLuceneWildcard(), l.caseInsensitive()); } if (e instanceof RLike) { diff --git a/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/querydsl/query/TermQuery.java b/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/querydsl/query/TermQuery.java index 229a459e73e..11dc869055f 100644 --- a/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/querydsl/query/TermQuery.java +++ b/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/querydsl/query/TermQuery.java @@ -6,6 +6,7 @@ package org.elasticsearch.xpack.ql.querydsl.query; import org.elasticsearch.index.query.QueryBuilder; +import org.elasticsearch.index.query.TermQueryBuilder; import org.elasticsearch.xpack.ql.tree.Source; import java.util.Objects; @@ -16,11 +17,17 @@ public class TermQuery extends LeafQuery { private final String term; private final Object value; + private final boolean caseInsensitive; public TermQuery(Source source, String term, Object value) { + this(source, term, value, false); + } + + public TermQuery(Source source, String term, Object value, boolean caseInsensitive) { super(source); this.term = term; this.value = value; + this.caseInsensitive = caseInsensitive; } public String term() { @@ -31,14 +38,20 @@ public class TermQuery extends LeafQuery { return value; } + public Boolean caseInsensitive() { + return caseInsensitive; + } + @Override public QueryBuilder asBuilder() { - return termQuery(term, value); + TermQueryBuilder qb = termQuery(term, value); + // ES does not allow case_insensitive to be set to "false", it should be either "true" or not specified + return caseInsensitive == false ? qb : qb.caseInsensitive(caseInsensitive); } @Override public int hashCode() { - return Objects.hash(term, value); + return Objects.hash(term, value, caseInsensitive); } @Override @@ -53,7 +66,8 @@ public class TermQuery extends LeafQuery { TermQuery other = (TermQuery) obj; return Objects.equals(term, other.term) - && Objects.equals(value, other.value); + && Objects.equals(value, other.value) + && Objects.equals(caseInsensitive, other.caseInsensitive); } @Override diff --git a/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/querydsl/query/WildcardQuery.java b/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/querydsl/query/WildcardQuery.java index 47d5624dbab..3b0cbefa8af 100644 --- a/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/querydsl/query/WildcardQuery.java +++ b/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/querydsl/query/WildcardQuery.java @@ -6,6 +6,7 @@ package org.elasticsearch.xpack.ql.querydsl.query; import org.elasticsearch.index.query.QueryBuilder; +import org.elasticsearch.index.query.WildcardQueryBuilder; import org.elasticsearch.xpack.ql.tree.Source; import java.util.Objects; @@ -15,11 +16,17 @@ import static org.elasticsearch.index.query.QueryBuilders.wildcardQuery; public class WildcardQuery extends LeafQuery { private final String field, query; + private final boolean caseInsensitive; public WildcardQuery(Source source, String field, String query) { + this(source, field, query, false); + } + + public WildcardQuery(Source source, String field, String query, boolean caseInsensitive) { super(source); this.field = field; this.query = query; + this.caseInsensitive = caseInsensitive; } public String field() { @@ -30,14 +37,20 @@ public class WildcardQuery extends LeafQuery { return query; } + public Boolean caseInsensitive() { + return caseInsensitive; + } + @Override public QueryBuilder asBuilder() { - return wildcardQuery(field, query); + WildcardQueryBuilder wb = wildcardQuery(field, query); + // ES does not allow case_insensitive to be set to "false", it should be either "true" or not specified + return caseInsensitive == false ? wb : wb.caseInsensitive(caseInsensitive); } @Override public int hashCode() { - return Objects.hash(field, query); + return Objects.hash(field, query, caseInsensitive); } @Override @@ -52,7 +65,8 @@ public class WildcardQuery extends LeafQuery { WildcardQuery other = (WildcardQuery) obj; return Objects.equals(field, other.field) - && Objects.equals(query, other.query); + && Objects.equals(query, other.query) + && Objects.equals(caseInsensitive, other.caseInsensitive); } @Override