EQL: Add concat function (#55193)

* EQL: Add concat function
* EQL: for loop spacing for concat
* EQL: return unresolved arguments to concat early
* EQL: Add concat integration tests
* EQL: Fix concat query fail test
* EQL: Add class for concat function testing
* EQL: Add concat integration tests
* EQL: Update concat() null behavior

x-pack/plugin/eql/qa/common/src/main/resources/test_queries_supported.toml

@@ -14,6 +14,39 @@ query = '''
 file where between(file_path, "dev", ".json", true) == "\\TestLogs\\something"
 '''
 
+[[queries]]
+description = "test string concatenation. update test to avoid case-sensitivity issues"
+query = '''
+process where concat(serial_event_id, '::', process_name, '::', opcode) == '5::wininit.exe::3'
+'''
+expected_event_ids  = [5]
+
+
+[[queries]]
+query = 'process where concat(serial_event_id) = "1"'
+expected_event_ids  = [1]
+
+[[queries]]
+query = 'process where serial_event_id < 5 and concat(process_name, parent_process_name) != null'
+expected_event_ids  = [2, 3]
+
+
+[[queries]]
+query = 'process where serial_event_id < 5 and concat(process_name, parent_process_name) == null'
+expected_event_ids  = [1, 4]
+
+
+[[queries]]
+query = 'process where serial_event_id < 5 and concat(process_name, null, null) == null'
+expected_event_ids  = [1, 2, 3, 4]
+
+
+
+[[queries]]
+query = 'process where serial_event_id < 5 and concat(parent_process_name, null) == null'
+expected_event_ids  = [1, 2, 3, 4]
+
+
 [[queries]]
 query = 'process where string(serial_event_id) = "1"'
 expected_event_ids  = [1]

x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/expression/function/EqlFunctionRegistry.java

@@ -8,6 +8,7 @@ package org.elasticsearch.xpack.eql.expression.function;
 
 import org.elasticsearch.xpack.eql.expression.function.scalar.string.CIDRMatch;
 import org.elasticsearch.xpack.eql.expression.function.scalar.string.Between;
+import org.elasticsearch.xpack.eql.expression.function.scalar.string.Concat;
 import org.elasticsearch.xpack.eql.expression.function.scalar.string.EndsWith;
 import org.elasticsearch.xpack.eql.expression.function.scalar.string.IndexOf;
 import org.elasticsearch.xpack.eql.expression.function.scalar.string.Length;
@@ -40,6 +41,7 @@ public class EqlFunctionRegistry extends FunctionRegistry {
             new FunctionDefinition[] {
                 def(Between.class, Between::new, 2, "between"),
                 def(CIDRMatch.class, CIDRMatch::new, "cidrmatch"),
+                def(Concat.class, Concat::new, "concat"),
                 def(EndsWith.class, EndsWith::new, "endswith"),
                 def(IndexOf.class, IndexOf::new, "indexof"),
                 def(Length.class, Length::new, "length"),

x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/expression/function/scalar/string/Concat.java

@@ -0,0 +1,106 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+package org.elasticsearch.xpack.eql.expression.function.scalar.string;
+
+import org.elasticsearch.xpack.ql.expression.Expression;
+import org.elasticsearch.xpack.ql.expression.Expressions;
+import org.elasticsearch.xpack.ql.expression.function.scalar.ScalarFunction;
+import org.elasticsearch.xpack.ql.expression.gen.pipeline.Pipe;
+import org.elasticsearch.xpack.ql.expression.gen.script.ParamsBuilder;
+import org.elasticsearch.xpack.ql.expression.gen.script.ScriptTemplate;
+import org.elasticsearch.xpack.ql.tree.NodeInfo;
+import org.elasticsearch.xpack.ql.tree.Source;
+import org.elasticsearch.xpack.ql.type.DataType;
+import org.elasticsearch.xpack.ql.type.DataTypes;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.StringJoiner;
+
+import static org.elasticsearch.xpack.eql.expression.function.scalar.string.ConcatFunctionProcessor.doProcess;
+import static org.elasticsearch.xpack.ql.expression.TypeResolutions.isExact;
+import static org.elasticsearch.xpack.ql.expression.gen.script.ParamsBuilder.paramsBuilder;
+
+/**
+ * EQL specific concat function to build a string of all input arguments concatenated.
+ */
+public class Concat extends ScalarFunction {
+
+    private final List<Expression> values;
+
+    public Concat(Source source, List<Expression> values) {
+        super(source, values);
+        this.values = values;
+    }
+
+    @Override
+    protected TypeResolution resolveType() {
+        if (!childrenResolved()) {
+            return new TypeResolution("Unresolved children");
+        }
+
+        TypeResolution resolution = TypeResolution.TYPE_RESOLVED;
+        for (Expression value : values) {
+            resolution = isExact(value, sourceText(), Expressions.ParamOrdinal.DEFAULT);
+
+            if (resolution.unresolved()) {
+                return resolution;
+            }
+        }
+
+        return resolution;
+    }
+
+    @Override
+    protected Pipe makePipe() {
+        return new ConcatFunctionPipe(source(), this, Expressions.pipe(values));
+    }
+
+    @Override
+    public boolean foldable() {
+        return Expressions.foldable(values);
+    }
+
+    @Override
+    public Object fold() {
+        return doProcess(Expressions.fold(values));
+    }
+
+    @Override
+    protected NodeInfo<? extends Expression> info() {
+        return NodeInfo.create(this, Concat::new, values);
+    }
+
+    @Override
+    public ScriptTemplate asScript() {
+        List<ScriptTemplate> templates = new ArrayList<>();
+        for (Expression ex : children()) {
+            templates.add(asScript(ex));
+        }
+
+        StringJoiner template = new StringJoiner(",", "{eql}.concat([", "])");
+        ParamsBuilder params = paramsBuilder();
+
+        for (ScriptTemplate scriptTemplate : templates) {
+            template.add(scriptTemplate.template());
+            params.script(scriptTemplate.params());
+        }
+
+        return new ScriptTemplate(formatTemplate(template.toString()), params.build(), dataType());
+    }
+
+    @Override
+    public DataType dataType() {
+        return DataTypes.KEYWORD;
+    }
+
+    @Override
+    public Expression replaceChildren(List<Expression> newChildren) {
+        return new Concat(source(), newChildren);
+    }
+
+}

x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/expression/function/scalar/string/ConcatFunctionPipe.java

@@ -0,0 +1,105 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.eql.expression.function.scalar.string;
+
+import org.elasticsearch.xpack.ql.execution.search.QlSourceBuilder;
+import org.elasticsearch.xpack.ql.expression.Expression;
+import org.elasticsearch.xpack.ql.expression.gen.pipeline.Pipe;
+import org.elasticsearch.xpack.ql.expression.gen.processor.Processor;
+import org.elasticsearch.xpack.ql.tree.NodeInfo;
+import org.elasticsearch.xpack.ql.tree.Source;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Objects;
+
+public class ConcatFunctionPipe extends Pipe {
+
+    private final List<Pipe> values;
+
+    public ConcatFunctionPipe(Source source, Expression expression, List<Pipe> values) {
+        super(source, expression, values);
+        this.values = values;
+    }
+
+    @Override
+    public final Pipe replaceChildren(List<Pipe> newChildren) {
+        return new ConcatFunctionPipe(source(), expression(), newChildren);
+    }
+
+    @Override
+    public final Pipe resolveAttributes(AttributeResolver resolver) {
+        List<Pipe> newValues = new ArrayList<>(values.size());
+        for (Pipe v : values) {
+            newValues.add(v.resolveAttributes(resolver));
+        }
+
+        if (newValues == values) {
+            return this;
+        }
+
+        return replaceChildren(newValues);
+    }
+
+    @Override
+    public boolean supportedByAggsOnlyQuery() {
+        for (Pipe p : values) {
+            if (p.supportedByAggsOnlyQuery() == false) {
+                return false;
+            }
+        }
+        return true;
+    }
+
+    @Override
+    public boolean resolved() {
+        for (Pipe p : values) {
+            if (p.resolved() == false) {
+                return false;
+            }
+        }
+        return true;
+    }
+
+    @Override
+    public final void collectFields(QlSourceBuilder sourceBuilder) {
+        for (Pipe v : values) {
+            v.collectFields(sourceBuilder);
+        }
+    }
+
+    @Override
+    protected NodeInfo<ConcatFunctionPipe> info() {
+        return NodeInfo.create(this, ConcatFunctionPipe::new, expression(), values);
+    }
+
+    @Override
+    public ConcatFunctionProcessor asProcessor() {
+        List<Processor> processors = new ArrayList<>(values.size());
+        for (Pipe p: values) {
+            processors.add(p.asProcessor());
+        }
+        return new ConcatFunctionProcessor(processors);
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(values);
+    }
+
+    @Override
+    public boolean equals(Object obj) {
+        if (this == obj) {
+            return true;
+        }
+
+        if (obj == null || getClass() != obj.getClass()) {
+            return false;
+        }
+
+        return Objects.equals(values, ((ConcatFunctionPipe) obj).values);
+    }
+}

x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/expression/function/scalar/string/ConcatFunctionProcessor.java

@@ -0,0 +1,83 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.eql.expression.function.scalar.string;
+
+import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.xpack.ql.expression.gen.processor.Processor;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Objects;
+
+public class ConcatFunctionProcessor implements Processor {
+
+    public static final String NAME = "scon";
+
+    private final List<Processor> values;
+
+    public ConcatFunctionProcessor(List<Processor> values) {
+        this.values = values;
+    }
+
+    @Override
+    public final void writeTo(StreamOutput out) throws IOException {
+        for (Processor v: values) {
+            out.writeNamedWriteable(v);
+        }
+    }
+
+    @Override
+    public Object process(Object input) {
+        List<Object> processed = new ArrayList<>(values.size());
+        for (Processor v: values) {
+            processed.add(v.process(input));
+        }
+        return doProcess(processed);
+    }
+
+    public static Object doProcess(List<Object> inputs) {
+        if (inputs == null) {
+            return null;
+        }
+
+        StringBuilder str = new StringBuilder();
+
+        for (Object input: inputs) {
+            if (input == null) {
+                return null;
+            }
+
+            str.append(input.toString());
+        }
+
+        return str.toString();
+    }
+
+    @Override
+    public boolean equals(Object obj) {
+        if (this == obj) {
+            return true;
+        }
+
+        if (obj == null || getClass() != obj.getClass()) {
+            return false;
+        }
+
+        return Objects.equals(values, ((ConcatFunctionProcessor) obj).values);
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(values);
+    }
+
+
+    @Override
+    public String getWriteableName() {
+        return NAME;
+    }
+}

x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/expression/function/scalar/whitelist/InternalEqlScriptUtils.java

@@ -7,6 +7,7 @@
 package org.elasticsearch.xpack.eql.expression.function.scalar.whitelist;
 
 import org.elasticsearch.xpack.eql.expression.function.scalar.string.BetweenFunctionProcessor;
+import org.elasticsearch.xpack.eql.expression.function.scalar.string.ConcatFunctionProcessor;
 import org.elasticsearch.xpack.eql.expression.function.scalar.string.EndsWithFunctionProcessor;
 import org.elasticsearch.xpack.eql.expression.function.scalar.string.IndexOfFunctionProcessor;
 import org.elasticsearch.xpack.eql.expression.function.scalar.string.LengthFunctionProcessor;
@@ -16,6 +17,8 @@ import org.elasticsearch.xpack.eql.expression.function.scalar.string.SubstringFu
 import org.elasticsearch.xpack.eql.expression.function.scalar.string.ToStringFunctionProcessor;
 import org.elasticsearch.xpack.ql.expression.function.scalar.whitelist.InternalQlScriptUtils;
 
+import java.util.List;
+
 /*
  * Whitelisted class for EQL scripts.
  * Acts as a registry of the various static methods used <b>internally</b> by the scalar functions
@@ -29,6 +32,10 @@ public class InternalEqlScriptUtils extends InternalQlScriptUtils {
         return (String) BetweenFunctionProcessor.doProcess(s, left, right, greedy, caseSensitive);
     }
 
+    public static String concat(List<Object> values) {
+        return (String) ConcatFunctionProcessor.doProcess(values);
+    }
+
     public static Boolean endsWith(String s, String pattern) {
         return (Boolean) EndsWithFunctionProcessor.doProcess(s, pattern);
     }

x-pack/plugin/eql/src/main/resources/org/elasticsearch/xpack/eql/plugin/eql_whitelist.txt

@@ -61,6 +61,7 @@ class org.elasticsearch.xpack.eql.expression.function.scalar.whitelist.InternalE
 # ASCII Functions
 #
   String  between(String, String, String, Boolean, Boolean)
+  String  concat(java.util.List)
   Boolean endsWith(String, String)
   Integer indexOf(String, String, Number)
   Integer length(String)

x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/analysis/VerifierTests.java

@@ -119,8 +119,6 @@ public class VerifierTests extends ESTestCase {
     public void testFunctionVerificationUnknown() {
         assertEquals("1:34: Unknown function [number]",
                 error("process where serial_event_id == number('5')"));
-        assertEquals("1:15: Unknown function [concat]",
-                error("process where concat(serial_event_id, ':', process_name, opcode) == '5:winINIT.exe3'"));
     }
 
     // Test unsupported array indexes

x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/expression/function/scalar/string/ConcatFunctionProcessorTests.java

@@ -0,0 +1,40 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+package org.elasticsearch.xpack.eql.expression.function.scalar.string;
+
+import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.xpack.ql.expression.Expression;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import static org.elasticsearch.xpack.ql.expression.function.scalar.FunctionTestUtils.l;
+import static org.elasticsearch.xpack.ql.tree.Source.EMPTY;
+
+
+public class ConcatFunctionProcessorTests extends ESTestCase {
+
+    private static Object process(Object ... arguments) {
+        List<Expression> literals = new ArrayList<>(arguments.length);
+        for (Object arg : arguments) {
+            literals.add(l(arg));
+        }
+        return new Concat(EMPTY, literals).makePipe().asProcessor().process(null);
+    }
+
+    public void testConcat() {
+        assertEquals(process(), "");
+        assertNull(process((Object) null));
+        assertEquals(process("foo"), "foo");
+        assertEquals(process(true), "true");
+        assertEquals(process(3.14), "3.14");
+        assertEquals(process("foo", "::", "bar", "::", "baz"), "foo::bar::baz");
+        assertNull(process("foo", "::", null, "::", "baz"));
+        assertNull(process("foo", "::", null, "::", null));
+        assertEquals(process("foo", "::", 1.0, "::", "baz"), "foo::1.0::baz");
+    }
+}

x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/planner/QueryFolderFailTests.java

@@ -93,6 +93,14 @@ public class QueryFolderFailTests extends AbstractQueryFolderTestCase {
             "line 1:15: argument of [cidrMatch(source_address, 12345)] must be [string], found value [12345] type [integer]", msg);
     }
 
+    public void testConcatWithInexact() {
+        VerificationException e = expectThrows(VerificationException.class,
+            () -> plan("process where concat(plain_text)"));
+        String msg = e.getMessage();
+        assertEquals("Found 1 problem\nline 1:15: [concat(plain_text)] cannot operate on field of data type "
+            + "[text]: No keyword/multi-field defined exact matches for [plain_text]; define one or use MATCH/QUERY instead", msg);
+    }
+
     public void testEndsWithFunctionWithInexact() {
         VerificationException e = expectThrows(VerificationException.class,
             () -> plan("process where endsWith(plain_text, \"foo\") == true"));
@@ -101,6 +109,20 @@ public class QueryFolderFailTests extends AbstractQueryFolderTestCase {
             + "[text]: No keyword/multi-field defined exact matches for [plain_text]; define one or use MATCH/QUERY instead", msg);
     }
 
+    public void testIndexOfFunctionWithInexact() {
+        VerificationException e = expectThrows(VerificationException.class,
+            () -> plan("process where indexOf(plain_text, \"foo\") == 1"));
+        String msg = e.getMessage();
+        assertEquals("Found 1 problem\nline 1:15: [indexOf(plain_text, \"foo\")] cannot operate on first argument field of data type "
+            + "[text]: No keyword/multi-field defined exact matches for [plain_text]; define one or use MATCH/QUERY instead", msg);
+
+        e = expectThrows(VerificationException.class,
+            () -> plan("process where indexOf(\"bla\", plain_text) == 1"));
+        msg = e.getMessage();
+        assertEquals("Found 1 problem\nline 1:15: [indexOf(\"bla\", plain_text)] cannot operate on second argument field of data type "
+            + "[text]: No keyword/multi-field defined exact matches for [plain_text]; define one or use MATCH/QUERY instead", msg);
+    }
+
     public void testLengthFunctionWithInexact() {
         VerificationException e = expectThrows(VerificationException.class,
             () -> plan("process where length(plain_text) > 0"));
@@ -151,28 +173,6 @@ public class QueryFolderFailTests extends AbstractQueryFolderTestCase {
                 "offender [parent_process_name] in [process_name in (parent_process_name, \"SYSTEM\")]", msg);
     }
 
-    public void testStartsWithFunctionWithInexact() {
-        VerificationException e = expectThrows(VerificationException.class,
-                () -> plan("process where startsWith(plain_text, \"foo\") == true"));
-        String msg = e.getMessage();
-        assertEquals("Found 1 problem\nline 1:15: [startsWith(plain_text, \"foo\")] cannot operate on first argument field of data type "
-                + "[text]: No keyword/multi-field defined exact matches for [plain_text]; define one or use MATCH/QUERY instead", msg);
-    }
-
-    public void testIndexOfFunctionWithInexact() {
-        VerificationException e = expectThrows(VerificationException.class,
-                () -> plan("process where indexOf(plain_text, \"foo\") == 1"));
-        String msg = e.getMessage();
-        assertEquals("Found 1 problem\nline 1:15: [indexOf(plain_text, \"foo\")] cannot operate on first argument field of data type "
-                + "[text]: No keyword/multi-field defined exact matches for [plain_text]; define one or use MATCH/QUERY instead", msg);
-
-        e = expectThrows(VerificationException.class,
-                () -> plan("process where indexOf(\"bla\", plain_text) == 1"));
-        msg = e.getMessage();
-        assertEquals("Found 1 problem\nline 1:15: [indexOf(\"bla\", plain_text)] cannot operate on second argument field of data type "
-                + "[text]: No keyword/multi-field defined exact matches for [plain_text]; define one or use MATCH/QUERY instead", msg);
-    }
-
     public void testSequenceWithBeforeBy() {
         String msg = errorParsing("sequence with maxspan=1s by key [a where true] [b where true]");
         assertEquals("1:2: Please specify sequence [by] before [with] not after", msg);
@@ -183,6 +183,15 @@ public class QueryFolderFailTests extends AbstractQueryFolderTestCase {
         assertEquals("1:24: No time unit specified, did you mean [s] as in [30s]?", msg);
     }
 
+    public void testStartsWithFunctionWithInexact() {
+        VerificationException e = expectThrows(VerificationException.class,
+                () -> plan("process where startsWith(plain_text, \"foo\") == true"));
+        String msg = e.getMessage();
+        assertEquals("Found 1 problem\nline 1:15: [startsWith(plain_text, \"foo\")] cannot operate on first argument field of data type "
+                + "[text]: No keyword/multi-field defined exact matches for [plain_text]; define one or use MATCH/QUERY instead", msg);
+    }
+
+
     public void testStringContainsWrongParams() {
         assertEquals("1:16: error building [stringcontains]: expects exactly two arguments",
                 errorParsing("process where stringContains()"));

x-pack/plugin/eql/src/test/resources/queryfolder_tests.txt

@@ -145,6 +145,14 @@ InternalEqlScriptUtils.between(InternalQlScriptUtils.docValue(doc,params.v0),par
 "params":{"v0":"process_name","v1":"s","v2":"e","v3":false,"v4":false,"v5":"yst"}
 ;
 
+concatFunction
+process where concat(process_name, "::foo::", null, 1) == "net.exe::foo::1"
+;
+"script":{"source":"InternalQlScriptUtils.nullSafeFilter(InternalQlScriptUtils.eq(
+InternalEqlScriptUtils.concat([InternalQlScriptUtils.docValue(doc,params.v0),params.v1,params.v2,params.v3]),params.v4))",
+"params":{"v0":"process_name","v1":"::foo::","v2":null,"v3":1,"v4":"net.exe::foo::1"}
+;
+
 cidrMatchFunctionOne
 process where cidrMatch(source_address, "10.0.0.0/8")
 ;

x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/expression/Expressions.java

@@ -107,6 +107,15 @@ public final class Expressions {
         return true;
     }
 
+    public static List<Object> fold(List<? extends Expression> exps) {
+        List<Object> folded = new ArrayList<>(exps.size());
+        for (Expression exp : exps) {
+            folded.add(exp.fold());
+        }
+
+        return folded;
+    }
+
     public static AttributeSet references(List<? extends Expression> exps) {
         if (exps.isEmpty()) {
             return AttributeSet.EMPTY;