diff --git a/x-pack/plugin/eql/qa/common/src/main/resources/test_queries_supported.toml b/x-pack/plugin/eql/qa/common/src/main/resources/test_queries_supported.toml index f7087265059..f815c083dca 100644 --- a/x-pack/plugin/eql/qa/common/src/main/resources/test_queries_supported.toml +++ b/x-pack/plugin/eql/qa/common/src/main/resources/test_queries_supported.toml @@ -17,3 +17,40 @@ file where between(file_path, "dev", ".json", true) == "\\TestLogs\\something" [[queries]] query = 'process where string(serial_event_id) = "1"' expected_event_ids = [1] + +[[queries]] +# Basic test for modulo function +query = ''' +process where modulo(11, 10) == serial_event_id''' +expected_event_ids = [1] +description = "test built-in modulo math functions" + +[[queries]] +# This query give a different result with ES EQL implementation because it doesn't convert to float data types for division +expected_event_ids = [82, 83] +query = "file where serial_event_id / 2 == 41" + +# Additional EQL queries with arithmetic operations that were not part of the original EQL implementation +[[queries]] +expected_event_ids = [82] +query = "file where 83 - serial_event_id == 1" + +[[queries]] +expected_event_ids = [82] +query = "file where 1 + serial_event_id == 83" + +[[queries]] +expected_event_ids = [82] +query = "file where -serial_event_id + 100 == 18" + +[[queries]] +expected_event_ids = [82] +query = "file where 2 * serial_event_id == 164" + +[[queries]] +expected_event_ids = [66] +query = "file where 66.0 / serial_event_id == 1" + +[[queries]] +expected_event_ids = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 46] +query = "process where serial_event_id + ((1 + 3) * 2 / (3 - 1)) * 2 == 54 or 70 + serial_event_id < 100" diff --git a/x-pack/plugin/eql/qa/common/src/main/resources/test_queries_unsupported.toml b/x-pack/plugin/eql/qa/common/src/main/resources/test_queries_unsupported.toml index aa3d1a32cc1..56a0adfdd19 100644 --- a/x-pack/plugin/eql/qa/common/src/main/resources/test_queries_unsupported.toml +++ b/x-pack/plugin/eql/qa/common/src/main/resources/test_queries_unsupported.toml @@ -826,24 +826,12 @@ expected_event_ids = [] description = "check built-in string functions" [[queries]] -query = ''' -process where add(serial_event_id, 0) == 1 and add(0, 1) == serial_event_id''' -expected_event_ids = [1] -description = "test built-in math functions" - -[[queries]] -query = ''' -process where subtract(serial_event_id, -5) == 6''' -expected_event_ids = [1] -description = "test built-in math functions" - -[[queries]] -query = ''' -process where multiply(6, serial_event_id) == 30 and divide(30, 4.0) == 7.5''' -expected_event_ids = [5] -description = "test built-in math functions" +# This query give a different result with ES EQL implementation because it doesn't convert to float data types for division +expected_event_ids = [82] +query = "file where serial_event_id / 2 == 41" [[queries]] +# Error: Line 1:42: Comparisons against variables are not (currently) supported; offender [serial_event_id] in [==] query = ''' process where modulo(11, add(serial_event_id, 1)) == serial_event_id''' expected_event_ids = [1, 2, 3, 5, 11] @@ -1002,26 +990,6 @@ query = ''' registry where arrayContains(bytes_written_string_list, "missing", "en-US") ''' -[[queries]] -expected_event_ids = [82] -query = "file where serial_event_id - 1 == 81" - -[[queries]] -expected_event_ids = [82] -query = "file where serial_event_id + 1 == 83" - -[[queries]] -expected_event_ids = [82] -query = "file where serial_event_id * 2 == 164" - -[[queries]] -expected_event_ids = [82] -query = "file where serial_event_id / 2 == 41" - -[[queries]] -expected_event_ids = [82] -query = "file where serial_event_id % 40 == 2" - # The following two "between" queries behave slightly different with elasticsearch # due to comparison on keyword field would be case-sensitive and would need to be # file where between(file_path, "dev", ".json", false) == "\\TestLogs\\something" diff --git a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/expression/function/EqlFunctionRegistry.java b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/expression/function/EqlFunctionRegistry.java index 96bf337ec09..eb98acd207b 100644 --- a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/expression/function/EqlFunctionRegistry.java +++ b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/expression/function/EqlFunctionRegistry.java @@ -18,6 +18,11 @@ import org.elasticsearch.xpack.eql.expression.function.scalar.string.ToString; import org.elasticsearch.xpack.eql.expression.function.scalar.string.Wildcard; import org.elasticsearch.xpack.ql.expression.function.FunctionDefinition; import org.elasticsearch.xpack.ql.expression.function.FunctionRegistry; +import org.elasticsearch.xpack.ql.expression.predicate.operator.arithmetic.Add; +import org.elasticsearch.xpack.ql.expression.predicate.operator.arithmetic.Div; +import org.elasticsearch.xpack.ql.expression.predicate.operator.arithmetic.Mod; +import org.elasticsearch.xpack.ql.expression.predicate.operator.arithmetic.Mul; +import org.elasticsearch.xpack.ql.expression.predicate.operator.arithmetic.Sub; import java.util.Locale; @@ -42,6 +47,14 @@ public class EqlFunctionRegistry extends FunctionRegistry { def(StringContains.class, StringContains::new, "stringcontains"), def(Substring.class, Substring::new, "substring"), def(Wildcard.class, Wildcard::new, "wildcard"), + }, + // Arithmetic + new FunctionDefinition[] { + def(Add.class, Add::new, "add"), + def(Div.class, Div::new, "divide"), + def(Mod.class, Mod::new, "modulo"), + def(Mul.class, Mul::new, "multiply"), + def(Sub.class, Sub::new, "subtract"), } }; } diff --git a/x-pack/plugin/eql/src/main/resources/org/elasticsearch/xpack/eql/plugin/eql_whitelist.txt b/x-pack/plugin/eql/src/main/resources/org/elasticsearch/xpack/eql/plugin/eql_whitelist.txt index bb0c136faaf..28016a63ac9 100644 --- a/x-pack/plugin/eql/src/main/resources/org/elasticsearch/xpack/eql/plugin/eql_whitelist.txt +++ b/x-pack/plugin/eql/src/main/resources/org/elasticsearch/xpack/eql/plugin/eql_whitelist.txt @@ -47,7 +47,12 @@ class org.elasticsearch.xpack.ql.expression.function.scalar.whitelist.InternalQl # # Math # + Number add(Number, Number) + Number div(Number, Number) + Number mod(Number, Number) + Number mul(Number, Number) Number neg(Number) + Number sub(Number, Number) } class org.elasticsearch.xpack.eql.expression.function.scalar.whitelist.InternalEqlScriptUtils { diff --git a/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/analysis/VerifierTests.java b/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/analysis/VerifierTests.java index 9e97614b8a8..24f73dadae0 100644 --- a/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/analysis/VerifierTests.java +++ b/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/analysis/VerifierTests.java @@ -119,14 +119,6 @@ public class VerifierTests extends ESTestCase { // Test the known EQL functions that are not supported public void testFunctionVerificationUnknown() { - assertEquals("1:15: Unknown function [add]", - error("process where add(serial_event_id, 0) == 1")); - assertEquals("1:15: Unknown function [subtract], did you mean [substring]?", - error("process where subtract(serial_event_id, -5) == 6")); - assertEquals("1:15: Unknown function [multiply]", - error("process where multiply(6, serial_event_id) == 30")); - assertEquals("1:15: Unknown function [divide]", - error("process where divide(30, 4.0) == 7.5")); assertEquals("1:34: Unknown function [number]", error("process where serial_event_id == number('5')")); assertEquals("1:15: Unknown function [concat]", diff --git a/x-pack/plugin/eql/src/test/resources/queryfolder_tests.txt b/x-pack/plugin/eql/src/test/resources/queryfolder_tests.txt index 22bd439890a..6f3f97db8b4 100644 --- a/x-pack/plugin/eql/src/test/resources/queryfolder_tests.txt +++ b/x-pack/plugin/eql/src/test/resources/queryfolder_tests.txt @@ -185,3 +185,164 @@ process where wildcard(process_path, "*\\red_ttp\\wininit.*", "*\\abc\\*", "*def "wildcard":{"process_path":{"wildcard":"*\\\\abc\\\\*" "wildcard":{"process_path":{"wildcard":"*def*" ; + + +addOperator +process where serial_event_id + 2 == 41 +; +"script":{"source":"InternalQlScriptUtils.nullSafeFilter(InternalQlScriptUtils.eq( +InternalQlScriptUtils.add(InternalQlScriptUtils.docValue(doc,params.v0),params.v1),params.v2))", +"params":{"v0":"serial_event_id","v1":2,"v2":41} +; + +addOperatorReversed +process where 2 + serial_event_id == 41 +; +"script":{"source":"InternalQlScriptUtils.nullSafeFilter(InternalQlScriptUtils.eq( +InternalQlScriptUtils.add(params.v0,InternalQlScriptUtils.docValue(doc,params.v1)),params.v2))", +"params":{"v0":2,"v1":"serial_event_id","v2":41} +; + +addFunction +process where add(serial_event_id, 2) == 41 +; +"script":{"source":"InternalQlScriptUtils.nullSafeFilter(InternalQlScriptUtils.eq( +InternalQlScriptUtils.add(InternalQlScriptUtils.docValue(doc,params.v0),params.v1),params.v2))", +"params":{"v0":"serial_event_id","v1":2,"v2":41} +; + +addFunctionReversed +process where add(2, serial_event_id) == 41 +; +"script":{"source":"InternalQlScriptUtils.nullSafeFilter(InternalQlScriptUtils.eq( +InternalQlScriptUtils.add(params.v0,InternalQlScriptUtils.docValue(doc,params.v1)),params.v2))", +"params":{"v0":2,"v1":"serial_event_id","v2":41} +; + +divideOperator +process where serial_event_id / 2 == 41 +; +"script":{"source":"InternalQlScriptUtils.nullSafeFilter(InternalQlScriptUtils.eq( +InternalQlScriptUtils.div(InternalQlScriptUtils.docValue(doc,params.v0),params.v1),params.v2))", +"params":{"v0":"serial_event_id","v1":2,"v2":41} +; + +divideOperatorReversed +process where 82 / serial_event_id == 41 +; +"script":{"source":"InternalQlScriptUtils.nullSafeFilter(InternalQlScriptUtils.eq( +InternalQlScriptUtils.div(params.v0,InternalQlScriptUtils.docValue(doc,params.v1)),params.v2))", +"params":{"v0":82,"v1":"serial_event_id","v2":41} +; + +divideFunction +process where divide(serial_event_id, 2) == 41 +; +"script":{"source":"InternalQlScriptUtils.nullSafeFilter(InternalQlScriptUtils.eq( +InternalQlScriptUtils.div(InternalQlScriptUtils.docValue(doc,params.v0),params.v1),params.v2))", +"params":{"v0":"serial_event_id","v1":2,"v2":41} +; + +divideFunctionReversed +process where divide(82, serial_event_id) == 41 +; +"script":{"source":"InternalQlScriptUtils.nullSafeFilter(InternalQlScriptUtils.eq( +InternalQlScriptUtils.div(params.v0,InternalQlScriptUtils.docValue(doc,params.v1)),params.v2))", +"params":{"v0":82,"v1":"serial_event_id","v2":41} +; + +moduloOperator +process where serial_event_id % 2 == 41 +; +"script":{"source":"InternalQlScriptUtils.nullSafeFilter(InternalQlScriptUtils.eq( +InternalQlScriptUtils.mod(InternalQlScriptUtils.docValue(doc,params.v0),params.v1),params.v2))", +"params":{"v0":"serial_event_id","v1":2,"v2":41} +; + +moduloOperatorReversed +process where 42 % serial_event_id == 41 +; +"script":{"source":"InternalQlScriptUtils.nullSafeFilter(InternalQlScriptUtils.eq( +InternalQlScriptUtils.mod(params.v0,InternalQlScriptUtils.docValue(doc,params.v1)),params.v2))", +"params":{"v0":42,"v1":"serial_event_id","v2":41} +; + +moduloFunction +process where modulo(serial_event_id, 2) == 41 +; +"script":{"source":"InternalQlScriptUtils.nullSafeFilter(InternalQlScriptUtils.eq( +InternalQlScriptUtils.mod(InternalQlScriptUtils.docValue(doc,params.v0),params.v1),params.v2))", +"params":{"v0":"serial_event_id","v1":2,"v2":41} +; + +moduloFunctionReversed +process where modulo(42, serial_event_id) == 41 +; +"script":{"source":"InternalQlScriptUtils.nullSafeFilter(InternalQlScriptUtils.eq( +InternalQlScriptUtils.mod(params.v0,InternalQlScriptUtils.docValue(doc,params.v1)),params.v2))", +"params":{"v0":42,"v1":"serial_event_id","v2":41} +; + +multiplyOperator +process where serial_event_id * 2 == 41 +; +"script":{"source":"InternalQlScriptUtils.nullSafeFilter(InternalQlScriptUtils.eq( +InternalQlScriptUtils.mul(InternalQlScriptUtils.docValue(doc,params.v0),params.v1),params.v2))", +"params":{"v0":"serial_event_id","v1":2,"v2":41} +; + +multiplyOperatorReversed +process where 2 * serial_event_id == 41 +; +"script":{"source":"InternalQlScriptUtils.nullSafeFilter(InternalQlScriptUtils.eq( +InternalQlScriptUtils.mul(params.v0,InternalQlScriptUtils.docValue(doc,params.v1)),params.v2))", +"params":{"v0":2,"v1":"serial_event_id","v2":41} +; + +multiplyFunction +process where multiply(serial_event_id, 2) == 41 +; +"script":{"source":"InternalQlScriptUtils.nullSafeFilter(InternalQlScriptUtils.eq( +InternalQlScriptUtils.mul(InternalQlScriptUtils.docValue(doc,params.v0),params.v1),params.v2))", +"params":{"v0":"serial_event_id","v1":2,"v2":41} +; + +multiplyFunctionReversed +process where multiply(2, serial_event_id) == 41 +; +"script":{"source":"InternalQlScriptUtils.nullSafeFilter(InternalQlScriptUtils.eq( +InternalQlScriptUtils.mul(params.v0,InternalQlScriptUtils.docValue(doc,params.v1)),params.v2))", +"params":{"v0":2,"v1":"serial_event_id","v2":41} +; + +subtractOperator +process where serial_event_id - 2 == 41 +; +"script":{"source":"InternalQlScriptUtils.nullSafeFilter(InternalQlScriptUtils.eq( +InternalQlScriptUtils.sub(InternalQlScriptUtils.docValue(doc,params.v0),params.v1),params.v2))", +"params":{"v0":"serial_event_id","v1":2,"v2":41} +; + +subtractOperatorReversed +process where 43 - serial_event_id == 41 +; +"script":{"source":"InternalQlScriptUtils.nullSafeFilter(InternalQlScriptUtils.eq( +InternalQlScriptUtils.sub(params.v0,InternalQlScriptUtils.docValue(doc,params.v1)),params.v2))", +"params":{"v0":43,"v1":"serial_event_id","v2":41} +; + +subtractFunction +process where subtract(serial_event_id, 2) == 41 +; +"script":{"source":"InternalQlScriptUtils.nullSafeFilter(InternalQlScriptUtils.eq( +InternalQlScriptUtils.sub(InternalQlScriptUtils.docValue(doc,params.v0),params.v1),params.v2))", +"params":{"v0":"serial_event_id","v1":2,"v2":41} +; + +subtractFunctionReversed +process where subtract(43, serial_event_id) == 41 +; +"script":{"source":"InternalQlScriptUtils.nullSafeFilter(InternalQlScriptUtils.eq( +InternalQlScriptUtils.sub(params.v0,InternalQlScriptUtils.docValue(doc,params.v1)),params.v2))", +"params":{"v0":43,"v1":"serial_event_id","v2":41} +; diff --git a/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/expression/function/scalar/whitelist/InternalQlScriptUtils.java b/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/expression/function/scalar/whitelist/InternalQlScriptUtils.java index f7956bc71b4..ef6d77ada69 100644 --- a/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/expression/function/scalar/whitelist/InternalQlScriptUtils.java +++ b/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/expression/function/scalar/whitelist/InternalQlScriptUtils.java @@ -10,6 +10,7 @@ import org.elasticsearch.index.fielddata.ScriptDocValues; import org.elasticsearch.xpack.ql.expression.predicate.logical.BinaryLogicProcessor.BinaryLogicOperation; import org.elasticsearch.xpack.ql.expression.predicate.logical.NotProcessor; import org.elasticsearch.xpack.ql.expression.predicate.nulls.CheckNullProcessor.CheckNullOperation; +import org.elasticsearch.xpack.ql.expression.predicate.operator.arithmetic.DefaultBinaryArithmeticOperation; import org.elasticsearch.xpack.ql.expression.predicate.operator.arithmetic.UnaryArithmeticProcessor.UnaryArithmeticOperation; import org.elasticsearch.xpack.ql.expression.predicate.operator.comparison.BinaryComparisonProcessor.BinaryComparisonOperation; import org.elasticsearch.xpack.ql.expression.predicate.operator.comparison.InProcessor; @@ -119,7 +120,27 @@ public class InternalQlScriptUtils { // // Math // + public static Number add(Number left, Number right) { + return (Number) DefaultBinaryArithmeticOperation.ADD.apply(left, right); + } + + public static Number div(Number left, Number right) { + return (Number) DefaultBinaryArithmeticOperation.DIV.apply(left, right); + } + + public static Number mod(Number left, Number right) { + return (Number) DefaultBinaryArithmeticOperation.MOD.apply(left, right); + } + + public static Number mul(Number left, Number right) { + return (Number) DefaultBinaryArithmeticOperation.MUL.apply(left, right); + } + public static Number neg(Number value) { return UnaryArithmeticOperation.NEGATE.apply(value); } -} \ No newline at end of file + + public static Number sub(Number left, Number right) { + return (Number) DefaultBinaryArithmeticOperation.SUB.apply(left, right); + } +}