EQL: implement math functions: add, divide, module, multiply, subtract (#55137) (#55737)

* EQL: implement math functions: add, divide, module, multiply, subtract

x-pack/plugin/eql/qa/common/src/main/resources/test_queries_supported.toml

@@ -17,3 +17,40 @@ file where between(file_path, "dev", ".json", true) == "\\TestLogs\\something"
 [[queries]]
 query = 'process where string(serial_event_id) = "1"'
 expected_event_ids  = [1]
+
+[[queries]]
+# Basic test for modulo function
+query = '''
+process where modulo(11, 10) == serial_event_id'''
+expected_event_ids  = [1]
+description = "test built-in modulo math functions"
+
+[[queries]]
+# This query give a different result with ES EQL implementation because it doesn't convert to float data types for division
+expected_event_ids = [82, 83]
+query = "file where serial_event_id / 2 == 41"
+
+# Additional EQL queries with arithmetic operations that were not part of the original EQL implementation
+[[queries]]
+expected_event_ids = [82]
+query = "file where 83 - serial_event_id == 1"
+
+[[queries]]
+expected_event_ids = [82]
+query = "file where 1 + serial_event_id == 83"
+
+[[queries]]
+expected_event_ids = [82]
+query = "file where -serial_event_id + 100 == 18"
+
+[[queries]]
+expected_event_ids = [82]
+query = "file where 2 * serial_event_id == 164"
+
+[[queries]]
+expected_event_ids = [66]
+query = "file where 66.0 / serial_event_id == 1"
+
+[[queries]]
+expected_event_ids = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 46]
+query = "process where serial_event_id + ((1 + 3) * 2 / (3 - 1)) * 2 == 54 or 70 + serial_event_id < 100"

x-pack/plugin/eql/qa/common/src/main/resources/test_queries_unsupported.toml

@@ -826,24 +826,12 @@ expected_event_ids = []
 description = "check built-in string functions"
 
 [[queries]]
-query = '''
-process where add(serial_event_id, 0) == 1 and add(0, 1) == serial_event_id'''
-expected_event_ids  = [1]
-description = "test built-in math functions"
-
-[[queries]]
-query = '''
-process where subtract(serial_event_id, -5) == 6'''
-expected_event_ids  = [1]
-description = "test built-in math functions"
-
-[[queries]]
-query = '''
-process where multiply(6, serial_event_id) == 30 and divide(30, 4.0) == 7.5'''
-expected_event_ids  = [5]
-description = "test built-in math functions"
+# This query give a different result with ES EQL implementation because it doesn't convert to float data types for division
+expected_event_ids = [82]
+query = "file where serial_event_id / 2 == 41"
 
 [[queries]]
+# Error: Line 1:42: Comparisons against variables are not (currently) supported; offender [serial_event_id] in [==]
 query = '''
 process where modulo(11, add(serial_event_id, 1)) == serial_event_id'''
 expected_event_ids  = [1, 2, 3, 5, 11]
@@ -1002,26 +990,6 @@ query = '''
 registry where arrayContains(bytes_written_string_list, "missing", "en-US")
 '''
 
-[[queries]]
-expected_event_ids = [82]
-query = "file where serial_event_id - 1 == 81"
-
-[[queries]]
-expected_event_ids = [82]
-query = "file where serial_event_id + 1 == 83"
-
-[[queries]]
-expected_event_ids = [82]
-query = "file where serial_event_id * 2 == 164"
-
-[[queries]]
-expected_event_ids = [82]
-query = "file where serial_event_id / 2 == 41"
-
-[[queries]]
-expected_event_ids = [82]
-query = "file where serial_event_id % 40 == 2"
-
 # The following two "between" queries behave slightly different with elasticsearch
 # due to comparison on keyword field would be case-sensitive and would need to be
 # file where between(file_path, "dev", ".json", false) == "\\TestLogs\\something"

x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/expression/function/EqlFunctionRegistry.java

@@ -18,6 +18,11 @@ import org.elasticsearch.xpack.eql.expression.function.scalar.string.ToString;
 import org.elasticsearch.xpack.eql.expression.function.scalar.string.Wildcard;
 import org.elasticsearch.xpack.ql.expression.function.FunctionDefinition;
 import org.elasticsearch.xpack.ql.expression.function.FunctionRegistry;
+import org.elasticsearch.xpack.ql.expression.predicate.operator.arithmetic.Add;
+import org.elasticsearch.xpack.ql.expression.predicate.operator.arithmetic.Div;
+import org.elasticsearch.xpack.ql.expression.predicate.operator.arithmetic.Mod;
+import org.elasticsearch.xpack.ql.expression.predicate.operator.arithmetic.Mul;
+import org.elasticsearch.xpack.ql.expression.predicate.operator.arithmetic.Sub;
 
 import java.util.Locale;
 
@@ -42,6 +47,14 @@ public class EqlFunctionRegistry extends FunctionRegistry {
                 def(StringContains.class, StringContains::new, "stringcontains"),
                 def(Substring.class, Substring::new, "substring"),
                 def(Wildcard.class, Wildcard::new, "wildcard"),
+            },
+        // Arithmetic
+            new FunctionDefinition[] {
+                    def(Add.class, Add::new, "add"),
+                    def(Div.class, Div::new, "divide"),
+                    def(Mod.class, Mod::new, "modulo"),
+                    def(Mul.class, Mul::new, "multiply"),
+                    def(Sub.class, Sub::new, "subtract"),
             }
         };
     }

x-pack/plugin/eql/src/main/resources/org/elasticsearch/xpack/eql/plugin/eql_whitelist.txt

@@ -47,7 +47,12 @@ class org.elasticsearch.xpack.ql.expression.function.scalar.whitelist.InternalQl
 #
 # Math
 #
+  Number add(Number, Number)
+  Number div(Number, Number)
+  Number mod(Number, Number)
+  Number mul(Number, Number)
   Number neg(Number)
+  Number sub(Number, Number)
 }
 
 class org.elasticsearch.xpack.eql.expression.function.scalar.whitelist.InternalEqlScriptUtils {

x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/analysis/VerifierTests.java

@@ -119,14 +119,6 @@ public class VerifierTests extends ESTestCase {
 
     // Test the known EQL functions that are not supported
     public void testFunctionVerificationUnknown() {
-        assertEquals("1:15: Unknown function [add]",
-                error("process where add(serial_event_id, 0) == 1"));
-        assertEquals("1:15: Unknown function [subtract], did you mean [substring]?",
-                error("process where subtract(serial_event_id, -5) == 6"));
-        assertEquals("1:15: Unknown function [multiply]",
-                error("process where multiply(6, serial_event_id) == 30"));
-        assertEquals("1:15: Unknown function [divide]",
-                error("process where divide(30, 4.0) == 7.5"));
         assertEquals("1:34: Unknown function [number]",
                 error("process where serial_event_id == number('5')"));
         assertEquals("1:15: Unknown function [concat]",

x-pack/plugin/eql/src/test/resources/queryfolder_tests.txt

@@ -185,3 +185,164 @@ process where wildcard(process_path, "*\\red_ttp\\wininit.*", "*\\abc\\*", "*def
 "wildcard":{"process_path":{"wildcard":"*\\\\abc\\\\*"
 "wildcard":{"process_path":{"wildcard":"*def*"
 ;
+
+
+addOperator
+process where serial_event_id + 2 == 41
+;
+"script":{"source":"InternalQlScriptUtils.nullSafeFilter(InternalQlScriptUtils.eq(
+InternalQlScriptUtils.add(InternalQlScriptUtils.docValue(doc,params.v0),params.v1),params.v2))",
+"params":{"v0":"serial_event_id","v1":2,"v2":41}
+;
+
+addOperatorReversed
+process where 2 + serial_event_id == 41
+;
+"script":{"source":"InternalQlScriptUtils.nullSafeFilter(InternalQlScriptUtils.eq(
+InternalQlScriptUtils.add(params.v0,InternalQlScriptUtils.docValue(doc,params.v1)),params.v2))",
+"params":{"v0":2,"v1":"serial_event_id","v2":41}
+;
+
+addFunction
+process where add(serial_event_id, 2) == 41
+;
+"script":{"source":"InternalQlScriptUtils.nullSafeFilter(InternalQlScriptUtils.eq(
+InternalQlScriptUtils.add(InternalQlScriptUtils.docValue(doc,params.v0),params.v1),params.v2))",
+"params":{"v0":"serial_event_id","v1":2,"v2":41}
+;
+
+addFunctionReversed
+process where add(2, serial_event_id) == 41
+;
+"script":{"source":"InternalQlScriptUtils.nullSafeFilter(InternalQlScriptUtils.eq(
+InternalQlScriptUtils.add(params.v0,InternalQlScriptUtils.docValue(doc,params.v1)),params.v2))",
+"params":{"v0":2,"v1":"serial_event_id","v2":41}
+;
+
+divideOperator
+process where serial_event_id / 2 == 41
+;
+"script":{"source":"InternalQlScriptUtils.nullSafeFilter(InternalQlScriptUtils.eq(
+InternalQlScriptUtils.div(InternalQlScriptUtils.docValue(doc,params.v0),params.v1),params.v2))",
+"params":{"v0":"serial_event_id","v1":2,"v2":41}
+;
+
+divideOperatorReversed
+process where 82 / serial_event_id == 41
+;
+"script":{"source":"InternalQlScriptUtils.nullSafeFilter(InternalQlScriptUtils.eq(
+InternalQlScriptUtils.div(params.v0,InternalQlScriptUtils.docValue(doc,params.v1)),params.v2))",
+"params":{"v0":82,"v1":"serial_event_id","v2":41}
+;
+
+divideFunction
+process where divide(serial_event_id, 2) == 41
+;
+"script":{"source":"InternalQlScriptUtils.nullSafeFilter(InternalQlScriptUtils.eq(
+InternalQlScriptUtils.div(InternalQlScriptUtils.docValue(doc,params.v0),params.v1),params.v2))",
+"params":{"v0":"serial_event_id","v1":2,"v2":41}
+;
+
+divideFunctionReversed
+process where divide(82, serial_event_id) == 41
+;
+"script":{"source":"InternalQlScriptUtils.nullSafeFilter(InternalQlScriptUtils.eq(
+InternalQlScriptUtils.div(params.v0,InternalQlScriptUtils.docValue(doc,params.v1)),params.v2))",
+"params":{"v0":82,"v1":"serial_event_id","v2":41}
+;
+
+moduloOperator
+process where serial_event_id % 2 == 41
+;
+"script":{"source":"InternalQlScriptUtils.nullSafeFilter(InternalQlScriptUtils.eq(
+InternalQlScriptUtils.mod(InternalQlScriptUtils.docValue(doc,params.v0),params.v1),params.v2))",
+"params":{"v0":"serial_event_id","v1":2,"v2":41}
+;
+
+moduloOperatorReversed
+process where 42 % serial_event_id == 41
+;
+"script":{"source":"InternalQlScriptUtils.nullSafeFilter(InternalQlScriptUtils.eq(
+InternalQlScriptUtils.mod(params.v0,InternalQlScriptUtils.docValue(doc,params.v1)),params.v2))",
+"params":{"v0":42,"v1":"serial_event_id","v2":41}
+;
+
+moduloFunction
+process where modulo(serial_event_id, 2) == 41
+;
+"script":{"source":"InternalQlScriptUtils.nullSafeFilter(InternalQlScriptUtils.eq(
+InternalQlScriptUtils.mod(InternalQlScriptUtils.docValue(doc,params.v0),params.v1),params.v2))",
+"params":{"v0":"serial_event_id","v1":2,"v2":41}
+;
+
+moduloFunctionReversed
+process where modulo(42, serial_event_id) == 41
+;
+"script":{"source":"InternalQlScriptUtils.nullSafeFilter(InternalQlScriptUtils.eq(
+InternalQlScriptUtils.mod(params.v0,InternalQlScriptUtils.docValue(doc,params.v1)),params.v2))",
+"params":{"v0":42,"v1":"serial_event_id","v2":41}
+;
+
+multiplyOperator
+process where serial_event_id * 2 == 41
+;
+"script":{"source":"InternalQlScriptUtils.nullSafeFilter(InternalQlScriptUtils.eq(
+InternalQlScriptUtils.mul(InternalQlScriptUtils.docValue(doc,params.v0),params.v1),params.v2))",
+"params":{"v0":"serial_event_id","v1":2,"v2":41}
+;
+
+multiplyOperatorReversed
+process where 2 * serial_event_id == 41
+;
+"script":{"source":"InternalQlScriptUtils.nullSafeFilter(InternalQlScriptUtils.eq(
+InternalQlScriptUtils.mul(params.v0,InternalQlScriptUtils.docValue(doc,params.v1)),params.v2))",
+"params":{"v0":2,"v1":"serial_event_id","v2":41}
+;
+
+multiplyFunction
+process where multiply(serial_event_id, 2) == 41
+;
+"script":{"source":"InternalQlScriptUtils.nullSafeFilter(InternalQlScriptUtils.eq(
+InternalQlScriptUtils.mul(InternalQlScriptUtils.docValue(doc,params.v0),params.v1),params.v2))",
+"params":{"v0":"serial_event_id","v1":2,"v2":41}
+;
+
+multiplyFunctionReversed
+process where multiply(2, serial_event_id) == 41
+;
+"script":{"source":"InternalQlScriptUtils.nullSafeFilter(InternalQlScriptUtils.eq(
+InternalQlScriptUtils.mul(params.v0,InternalQlScriptUtils.docValue(doc,params.v1)),params.v2))",
+"params":{"v0":2,"v1":"serial_event_id","v2":41}
+;
+
+subtractOperator
+process where serial_event_id - 2 == 41
+;
+"script":{"source":"InternalQlScriptUtils.nullSafeFilter(InternalQlScriptUtils.eq(
+InternalQlScriptUtils.sub(InternalQlScriptUtils.docValue(doc,params.v0),params.v1),params.v2))",
+"params":{"v0":"serial_event_id","v1":2,"v2":41}
+;
+
+subtractOperatorReversed
+process where 43 - serial_event_id == 41
+;
+"script":{"source":"InternalQlScriptUtils.nullSafeFilter(InternalQlScriptUtils.eq(
+InternalQlScriptUtils.sub(params.v0,InternalQlScriptUtils.docValue(doc,params.v1)),params.v2))",
+"params":{"v0":43,"v1":"serial_event_id","v2":41}
+;
+
+subtractFunction
+process where subtract(serial_event_id, 2) == 41
+;
+"script":{"source":"InternalQlScriptUtils.nullSafeFilter(InternalQlScriptUtils.eq(
+InternalQlScriptUtils.sub(InternalQlScriptUtils.docValue(doc,params.v0),params.v1),params.v2))",
+"params":{"v0":"serial_event_id","v1":2,"v2":41}
+;
+
+subtractFunctionReversed
+process where subtract(43, serial_event_id) == 41
+;
+"script":{"source":"InternalQlScriptUtils.nullSafeFilter(InternalQlScriptUtils.eq(
+InternalQlScriptUtils.sub(params.v0,InternalQlScriptUtils.docValue(doc,params.v1)),params.v2))",
+"params":{"v0":43,"v1":"serial_event_id","v2":41}
+;

x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/expression/function/scalar/whitelist/InternalQlScriptUtils.java

@@ -10,6 +10,7 @@ import org.elasticsearch.index.fielddata.ScriptDocValues;
 import org.elasticsearch.xpack.ql.expression.predicate.logical.BinaryLogicProcessor.BinaryLogicOperation;
 import org.elasticsearch.xpack.ql.expression.predicate.logical.NotProcessor;
 import org.elasticsearch.xpack.ql.expression.predicate.nulls.CheckNullProcessor.CheckNullOperation;
+import org.elasticsearch.xpack.ql.expression.predicate.operator.arithmetic.DefaultBinaryArithmeticOperation;
 import org.elasticsearch.xpack.ql.expression.predicate.operator.arithmetic.UnaryArithmeticProcessor.UnaryArithmeticOperation;
 import org.elasticsearch.xpack.ql.expression.predicate.operator.comparison.BinaryComparisonProcessor.BinaryComparisonOperation;
 import org.elasticsearch.xpack.ql.expression.predicate.operator.comparison.InProcessor;
@@ -119,7 +120,27 @@ public class InternalQlScriptUtils {
     //
     // Math
     //
+    public static Number add(Number left, Number right) {
+        return (Number) DefaultBinaryArithmeticOperation.ADD.apply(left, right);
+    }
+
+    public static Number div(Number left, Number right) {
+        return (Number) DefaultBinaryArithmeticOperation.DIV.apply(left, right);
+    }
+
+    public static Number mod(Number left, Number right) {
+        return (Number) DefaultBinaryArithmeticOperation.MOD.apply(left, right);
+    }
+
+    public static Number mul(Number left, Number right) {
+        return (Number) DefaultBinaryArithmeticOperation.MUL.apply(left, right);
+    }
+
     public static Number neg(Number value) {
         return UnaryArithmeticOperation.NEGATE.apply(value);
     }
-}
+
+    public static Number sub(Number left, Number right) {
+        return (Number) DefaultBinaryArithmeticOperation.SUB.apply(left, right);
+    }
+}