EQL: Length function implementation (#54209) (#54490)

(cherry picked from commit 18493467e55e014be2c9e0ebdf734e9d7fc4beaa)

x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/expression/function/EqlFunctionRegistry.java

@@ -6,6 +6,7 @@
 
 package org.elasticsearch.xpack.eql.expression.function;
 
+import org.elasticsearch.xpack.eql.expression.function.scalar.string.Length;
 import org.elasticsearch.xpack.eql.expression.function.scalar.string.Substring;
 import org.elasticsearch.xpack.ql.expression.function.FunctionDefinition;
 import org.elasticsearch.xpack.ql.expression.function.FunctionRegistry;
@@ -23,8 +24,9 @@ public class EqlFunctionRegistry extends FunctionRegistry {
         // Scalar functions
         // String
             new FunctionDefinition[] {
-                def(Substring.class, Substring::new, "substring"),
-            },
+                def(Length.class, Length::new, "length"),
+                def(Substring.class, Substring::new, "substring")
+            }
         };
     }
 

x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/expression/function/scalar/string/Length.java

@@ -0,0 +1,106 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+package org.elasticsearch.xpack.eql.expression.function.scalar.string;
+
+import org.elasticsearch.xpack.ql.expression.Expression;
+import org.elasticsearch.xpack.ql.expression.Expressions;
+import org.elasticsearch.xpack.ql.expression.Expressions.ParamOrdinal;
+import org.elasticsearch.xpack.ql.expression.FieldAttribute;
+import org.elasticsearch.xpack.ql.expression.function.scalar.ScalarFunction;
+import org.elasticsearch.xpack.ql.expression.gen.pipeline.Pipe;
+import org.elasticsearch.xpack.ql.expression.gen.script.ScriptTemplate;
+import org.elasticsearch.xpack.ql.expression.gen.script.Scripts;
+import org.elasticsearch.xpack.ql.tree.NodeInfo;
+import org.elasticsearch.xpack.ql.tree.Source;
+import org.elasticsearch.xpack.ql.type.DataType;
+import org.elasticsearch.xpack.ql.type.DataTypes;
+
+import java.util.Arrays;
+import java.util.List;
+import java.util.Locale;
+
+import static java.lang.String.format;
+import static org.elasticsearch.xpack.eql.expression.function.scalar.string.LengthFunctionProcessor.doProcess;
+import static org.elasticsearch.xpack.ql.expression.TypeResolutions.isStringAndExact;
+import static org.elasticsearch.xpack.ql.expression.gen.script.ParamsBuilder.paramsBuilder;
+
+/**
+ * EQL specific length function acting on every type of field, not only strings.
+ * For strings it will return the length of that specific string, for any other type it will return 0.
+ */
+public class Length extends ScalarFunction {
+
+    private final Expression source;
+    
+    public Length(Source source, Expression src) {
+        super(source, Arrays.asList(src));
+        this.source = src;
+    }
+
+    @Override
+    protected TypeResolution resolveType() {
+        if (!childrenResolved()) {
+            return new TypeResolution("Unresolved children");
+        }
+        
+        return isStringAndExact(source, sourceText(), ParamOrdinal.DEFAULT);
+    }
+
+    @Override
+    protected Pipe makePipe() {
+        return new LengthFunctionPipe(source(), this, Expressions.pipe(source));
+    }
+
+    @Override
+    public boolean foldable() {
+        return source.foldable();
+    }
+
+    @Override
+    public Object fold() {
+        return doProcess(source.fold());
+    }
+
+    @Override
+    protected NodeInfo<? extends Expression> info() {
+        return NodeInfo.create(this, Length::new, source);
+    }
+
+    @Override
+    public ScriptTemplate asScript() {
+        ScriptTemplate sourceScript = asScript(source);
+
+        return new ScriptTemplate(format(Locale.ROOT, formatTemplate("{eql}.%s(%s)"),
+                "length",
+                sourceScript.template()),
+                paramsBuilder()
+                    .script(sourceScript.params())
+                    .build(), dataType());
+    }
+
+    @Override
+    public ScriptTemplate scriptWithField(FieldAttribute field) {
+        return new ScriptTemplate(processScript(Scripts.DOC_VALUE),
+                paramsBuilder().variable(field.exactAttribute().name()).build(),
+                dataType());
+    }
+
+    @Override
+    public DataType dataType() {
+        return DataTypes.INTEGER;
+    }
+
+    @Override
+    public Expression replaceChildren(List<Expression> newChildren) {
+        if (newChildren.size() != 1) {
+            throw new IllegalArgumentException("expected [1] children but received [" + newChildren.size() + "]");
+        }
+
+        return new Length(source(), newChildren.get(0));
+    }
+
+}

x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/expression/function/scalar/string/LengthFunctionPipe.java

@@ -0,0 +1,94 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.eql.expression.function.scalar.string;
+
+import org.elasticsearch.xpack.ql.execution.search.QlSourceBuilder;
+import org.elasticsearch.xpack.ql.expression.Expression;
+import org.elasticsearch.xpack.ql.expression.gen.pipeline.Pipe;
+import org.elasticsearch.xpack.ql.tree.NodeInfo;
+import org.elasticsearch.xpack.ql.tree.Source;
+
+import java.util.Arrays;
+import java.util.List;
+import java.util.Objects;
+
+public class LengthFunctionPipe extends Pipe {
+
+    private final Pipe source;
+
+    public LengthFunctionPipe(Source source, Expression expression, Pipe src) {
+        super(source, expression, Arrays.asList(src));
+        this.source = src;
+    }
+
+    @Override
+    public final Pipe replaceChildren(List<Pipe> newChildren) {
+        if (newChildren.size() != 1) {
+            throw new IllegalArgumentException("expected [1] children but received [" + newChildren.size() + "]");
+        }
+        return replaceChildren(newChildren.get(0));
+    }
+
+    @Override
+    public final Pipe resolveAttributes(AttributeResolver resolver) {
+        Pipe newSource = source.resolveAttributes(resolver);
+        if (newSource == source) {
+            return this;
+        }
+        return replaceChildren(newSource);
+    }
+
+    @Override
+    public boolean supportedByAggsOnlyQuery() {
+        return source.supportedByAggsOnlyQuery();
+    }
+
+    @Override
+    public boolean resolved() {
+        return source.resolved();
+    }
+
+    protected Pipe replaceChildren(Pipe newSource) {
+        return new LengthFunctionPipe(source(), expression(), newSource);
+    }
+
+    @Override
+    public final void collectFields(QlSourceBuilder sourceBuilder) {
+        source.collectFields(sourceBuilder);
+    }
+
+    @Override
+    protected NodeInfo<LengthFunctionPipe> info() {
+        return NodeInfo.create(this, LengthFunctionPipe::new, expression(), source);
+    }
+
+    @Override
+    public LengthFunctionProcessor asProcessor() {
+        return new LengthFunctionProcessor(source.asProcessor());
+    }
+    
+    public Pipe src() {
+        return source;
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(source);
+    }
+
+    @Override
+    public boolean equals(Object obj) {
+        if (this == obj) {
+            return true;
+        }
+
+        if (obj == null || getClass() != obj.getClass()) {
+            return false;
+        }
+
+        return Objects.equals(source, ((LengthFunctionPipe) obj).source);
+    }
+}

x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/expression/function/scalar/string/LengthFunctionProcessor.java

@@ -0,0 +1,78 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.eql.expression.function.scalar.string;
+
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.xpack.eql.EqlIllegalArgumentException;
+import org.elasticsearch.xpack.ql.expression.gen.processor.Processor;
+
+import java.io.IOException;
+import java.util.Objects;
+
+public class LengthFunctionProcessor implements Processor {
+
+    public static final String NAME = "slen";
+
+    private final Processor source;
+
+    public LengthFunctionProcessor(Processor source) {
+        this.source = source;
+    }
+
+    public LengthFunctionProcessor(StreamInput in) throws IOException {
+        source = in.readNamedWriteable(Processor.class);
+    }
+
+    @Override
+    public final void writeTo(StreamOutput out) throws IOException {
+        out.writeNamedWriteable(source);
+    }
+
+    @Override
+    public Object process(Object input) {
+        return doProcess(source.process(input));
+    }
+
+    public static Object doProcess(Object source) {
+        if (source == null) {
+            return null;
+        }
+        if (source instanceof String == false && source instanceof Character == false) {
+            throw new EqlIllegalArgumentException("A string/char is required; received [{}]", source);
+        }
+
+        return source.toString().length();
+    }
+    
+    protected Processor source() {
+        return source;
+    }
+    
+    @Override
+    public boolean equals(Object obj) {
+        if (this == obj) {
+            return true;
+        }
+        
+        if (obj == null || getClass() != obj.getClass()) {
+            return false;
+        }
+        
+        return Objects.equals(source(), ((LengthFunctionProcessor) obj).source());
+    }
+    
+    @Override
+    public int hashCode() {
+        return Objects.hash(source());
+    }
+    
+
+    @Override
+    public String getWriteableName() {
+        return NAME;
+    }
+}

x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/expression/function/scalar/whitelist/InternalEqlScriptUtils.java

@@ -6,6 +6,7 @@
 
 package org.elasticsearch.xpack.eql.expression.function.scalar.whitelist;
 
+import org.elasticsearch.xpack.eql.expression.function.scalar.string.LengthFunctionProcessor;
 import org.elasticsearch.xpack.eql.expression.function.scalar.string.SubstringFunctionProcessor;
 import org.elasticsearch.xpack.ql.expression.function.scalar.whitelist.InternalQlScriptUtils;
 
@@ -18,6 +19,10 @@ public class InternalEqlScriptUtils extends InternalQlScriptUtils {
 
     InternalEqlScriptUtils() {}
 
+    public static Integer length(String s) {
+        return (Integer) LengthFunctionProcessor.doProcess(s);
+    }
+
     public static String substring(String s, Number start, Number end) {
         return (String) SubstringFunctionProcessor.doProcess(s, start, end);
     }

x-pack/plugin/eql/src/main/resources/org/elasticsearch/xpack/eql/plugin/eql_whitelist.txt

@@ -55,5 +55,6 @@ class org.elasticsearch.xpack.eql.expression.function.scalar.whitelist.InternalE
 #
 # ASCII Functions
 # 
+  Integer length(String)
   String  substring(String, Number, Number)
 }

x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/expression/function/scalar/string/LengthProcessorTests.java

@@ -0,0 +1,41 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+package org.elasticsearch.xpack.eql.expression.function.scalar.string;
+
+import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.xpack.ql.QlIllegalArgumentException;
+import org.elasticsearch.xpack.ql.expression.Literal;
+import org.elasticsearch.xpack.ql.expression.LiteralTests;
+
+import static org.elasticsearch.xpack.ql.expression.function.scalar.FunctionTestUtils.l;
+import static org.elasticsearch.xpack.ql.tree.Source.EMPTY;
+import static org.elasticsearch.xpack.ql.type.DataTypes.KEYWORD;
+import static org.hamcrest.Matchers.startsWith;
+
+public class LengthProcessorTests extends ESTestCase {
+
+    public void testLengthFunctionWithValidInput() {
+        assertEquals(9, new Length(EMPTY, l("foobarbar")).makePipe().asProcessor().process(null));
+        assertEquals(0, new Length(EMPTY, l("")).makePipe().asProcessor().process(null));
+        assertEquals(1, new Length(EMPTY, l('f')).makePipe().asProcessor().process(null));
+    }
+    
+    public void testLengthFunctionInputsValidation() {
+        QlIllegalArgumentException siae = expectThrows(QlIllegalArgumentException.class,
+                () -> new Length(EMPTY, l(5)).makePipe().asProcessor().process(null));
+        assertEquals("A string/char is required; received [5]", siae.getMessage());
+        siae = expectThrows(QlIllegalArgumentException.class, () -> new Length(EMPTY, l(true)).makePipe().asProcessor().process(null));
+        assertEquals("A string/char is required; received [true]", siae.getMessage());
+    }
+
+    public void testLengthFunctionWithRandomInvalidDataType() {
+        Literal literal = randomValueOtherThanMany(v -> v.dataType() == KEYWORD, () -> LiteralTests.randomLiteral());
+        QlIllegalArgumentException siae = expectThrows(QlIllegalArgumentException.class,
+                () -> new Length(EMPTY, literal).makePipe().asProcessor().process(null));
+        assertThat(siae.getMessage(), startsWith("A string/char is required; received"));
+    }
+}

x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/planner/AbstractQueryFolderTestCase.java

@@ -26,7 +26,7 @@ public abstract class AbstractQueryFolderTestCase extends ESTestCase {
     protected Optimizer optimizer = new Optimizer();
     protected Planner planner = new Planner();
 
-    protected IndexResolution index = IndexResolution.valid(new EsIndex("test", loadMapping("mapping-default.json")));
+    protected IndexResolution index = IndexResolution.valid(new EsIndex("test", loadMapping("mapping-default.json", true)));
 
     protected PhysicalPlan plan(IndexResolution resolution, String eql) {
         return planner.plan(optimizer.optimize(analyzer.analyze(preAnalyzer.preAnalyze(parser.createStatement(eql), resolution))));

x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/planner/QueryFolderFailTests.java

@@ -24,4 +24,12 @@ public class QueryFolderFailTests extends AbstractQueryFolderTestCase {
         assertEquals("Found 1 problem\nline 1:35: Comparisons against variables are not (currently) supported; " +
             "offender [parent_process_name] in [process_name in (parent_process_name, \"SYSTEM\")]", msg);
     }
+
+    public void testLengthFunctionWithInexact() {
+        VerificationException e = expectThrows(VerificationException.class,
+                () -> plan("process where length(plain_text) > 0"));
+        String msg = e.getMessage();
+        assertEquals("Found 1 problem\nline 1:15: [length(plain_text)] cannot operate on field of data type [text]: No keyword/multi-field "
+                + "defined exact matches for [plain_text]; define one or use MATCH/QUERY instead", msg);
+    }
 }

x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/planner/QueryFolderOkTests.java

@@ -95,7 +95,7 @@ public class QueryFolderOkTests extends AbstractQueryFolderTestCase {
         PhysicalPlan p = plan(query);
         assertEquals(EsQueryExec.class, p.getClass());
         EsQueryExec eqe = (EsQueryExec) p;
-        assertEquals(25, eqe.output().size());
+        assertEquals(27, eqe.output().size());
         assertEquals(KEYWORD, eqe.output().get(0).dataType());
 
         final String query = eqe.queryContainer().toString().replaceAll("\\s+", "");

x-pack/plugin/eql/src/test/resources/mapping-default.json

@@ -81,6 +81,12 @@
         },
         "exit_code" : {
             "type" : "long"
+        },
+        "plain_text" : {
+            "type" : "text"
+        },
+        "constant_keyword" : {
+            "type" : "constant_keyword"
         }
     }
 }

x-pack/plugin/eql/src/test/resources/queryfolder_tests.txt

@@ -61,6 +61,27 @@ process where process_path == "*\\red_ttp\\wininit.*" and opcode in (0,1,2,3)
 {"terms":{"opcode":[0,1,2,3]
 
 
+lengthFunctionWithExactSubField
+process where length(file_name) > 0
+"script":{"source":"InternalQlScriptUtils.nullSafeFilter(InternalQlScriptUtils.gt(
+InternalEqlScriptUtils.length(InternalQlScriptUtils.docValue(doc,params.v0)),params.v1))",
+"params":{"v0":"file_name.keyword","v1":0}
+
+
+lengthFunctionWithExactField
+process where 12 == length(user_name)
+"script":{"source":"InternalQlScriptUtils.nullSafeFilter(InternalQlScriptUtils.eq(
+InternalEqlScriptUtils.length(InternalQlScriptUtils.docValue(doc,params.v0)),params.v1))",
+"params":{"v0":"user_name","v1":12}
+
+
+lengthFunctionWithConstantKeyword
+process where 5 > length(constant_keyword)
+"script":{"source":"InternalQlScriptUtils.nullSafeFilter(InternalQlScriptUtils.lt(
+InternalEqlScriptUtils.length(InternalQlScriptUtils.docValue(doc,params.v0)),params.v1))",
+"params":{"v0":"constant_keyword","v1":5}
+
+
 substringFunction
 process where substring(file_name, -4) == '.exe'
 "script":{"source":"InternalQlScriptUtils.nullSafeFilter(InternalQlScriptUtils.eq(