From 364ea0a3c03dde73a7e81baeeadc8131ed264a65 Mon Sep 17 00:00:00 2001 From: Andrei Stefan Date: Tue, 31 Mar 2020 16:49:18 +0300 Subject: [PATCH] EQL: Length function implementation (#54209) (#54490) (cherry picked from commit 18493467e55e014be2c9e0ebdf734e9d7fc4beaa) --- .../function/EqlFunctionRegistry.java | 6 +- .../function/scalar/string/Length.java | 106 ++++++++++++++++++ .../scalar/string/LengthFunctionPipe.java | 94 ++++++++++++++++ .../string/LengthFunctionProcessor.java | 78 +++++++++++++ .../whitelist/InternalEqlScriptUtils.java | 5 + .../xpack/eql/plugin/eql_whitelist.txt | 1 + .../scalar/string/LengthProcessorTests.java | 41 +++++++ .../planner/AbstractQueryFolderTestCase.java | 2 +- .../eql/planner/QueryFolderFailTests.java | 8 ++ .../xpack/eql/planner/QueryFolderOkTests.java | 2 +- .../src/test/resources/mapping-default.json | 6 + .../src/test/resources/queryfolder_tests.txt | 21 ++++ 12 files changed, 366 insertions(+), 4 deletions(-) create mode 100644 x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/expression/function/scalar/string/Length.java create mode 100644 x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/expression/function/scalar/string/LengthFunctionPipe.java create mode 100644 x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/expression/function/scalar/string/LengthFunctionProcessor.java create mode 100644 x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/expression/function/scalar/string/LengthProcessorTests.java diff --git a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/expression/function/EqlFunctionRegistry.java b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/expression/function/EqlFunctionRegistry.java index 119e12fa9f3..4d7b319cc66 100644 --- a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/expression/function/EqlFunctionRegistry.java +++ b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/expression/function/EqlFunctionRegistry.java @@ -6,6 +6,7 @@ package org.elasticsearch.xpack.eql.expression.function; +import org.elasticsearch.xpack.eql.expression.function.scalar.string.Length; import org.elasticsearch.xpack.eql.expression.function.scalar.string.Substring; import org.elasticsearch.xpack.ql.expression.function.FunctionDefinition; import org.elasticsearch.xpack.ql.expression.function.FunctionRegistry; @@ -23,8 +24,9 @@ public class EqlFunctionRegistry extends FunctionRegistry { // Scalar functions // String new FunctionDefinition[] { - def(Substring.class, Substring::new, "substring"), - }, + def(Length.class, Length::new, "length"), + def(Substring.class, Substring::new, "substring") + } }; } diff --git a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/expression/function/scalar/string/Length.java b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/expression/function/scalar/string/Length.java new file mode 100644 index 00000000000..3607ba27a3f --- /dev/null +++ b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/expression/function/scalar/string/Length.java @@ -0,0 +1,106 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.eql.expression.function.scalar.string; + +import org.elasticsearch.xpack.ql.expression.Expression; +import org.elasticsearch.xpack.ql.expression.Expressions; +import org.elasticsearch.xpack.ql.expression.Expressions.ParamOrdinal; +import org.elasticsearch.xpack.ql.expression.FieldAttribute; +import org.elasticsearch.xpack.ql.expression.function.scalar.ScalarFunction; +import org.elasticsearch.xpack.ql.expression.gen.pipeline.Pipe; +import org.elasticsearch.xpack.ql.expression.gen.script.ScriptTemplate; +import org.elasticsearch.xpack.ql.expression.gen.script.Scripts; +import org.elasticsearch.xpack.ql.tree.NodeInfo; +import org.elasticsearch.xpack.ql.tree.Source; +import org.elasticsearch.xpack.ql.type.DataType; +import org.elasticsearch.xpack.ql.type.DataTypes; + +import java.util.Arrays; +import java.util.List; +import java.util.Locale; + +import static java.lang.String.format; +import static org.elasticsearch.xpack.eql.expression.function.scalar.string.LengthFunctionProcessor.doProcess; +import static org.elasticsearch.xpack.ql.expression.TypeResolutions.isStringAndExact; +import static org.elasticsearch.xpack.ql.expression.gen.script.ParamsBuilder.paramsBuilder; + +/** + * EQL specific length function acting on every type of field, not only strings. + * For strings it will return the length of that specific string, for any other type it will return 0. + */ +public class Length extends ScalarFunction { + + private final Expression source; + + public Length(Source source, Expression src) { + super(source, Arrays.asList(src)); + this.source = src; + } + + @Override + protected TypeResolution resolveType() { + if (!childrenResolved()) { + return new TypeResolution("Unresolved children"); + } + + return isStringAndExact(source, sourceText(), ParamOrdinal.DEFAULT); + } + + @Override + protected Pipe makePipe() { + return new LengthFunctionPipe(source(), this, Expressions.pipe(source)); + } + + @Override + public boolean foldable() { + return source.foldable(); + } + + @Override + public Object fold() { + return doProcess(source.fold()); + } + + @Override + protected NodeInfo info() { + return NodeInfo.create(this, Length::new, source); + } + + @Override + public ScriptTemplate asScript() { + ScriptTemplate sourceScript = asScript(source); + + return new ScriptTemplate(format(Locale.ROOT, formatTemplate("{eql}.%s(%s)"), + "length", + sourceScript.template()), + paramsBuilder() + .script(sourceScript.params()) + .build(), dataType()); + } + + @Override + public ScriptTemplate scriptWithField(FieldAttribute field) { + return new ScriptTemplate(processScript(Scripts.DOC_VALUE), + paramsBuilder().variable(field.exactAttribute().name()).build(), + dataType()); + } + + @Override + public DataType dataType() { + return DataTypes.INTEGER; + } + + @Override + public Expression replaceChildren(List newChildren) { + if (newChildren.size() != 1) { + throw new IllegalArgumentException("expected [1] children but received [" + newChildren.size() + "]"); + } + + return new Length(source(), newChildren.get(0)); + } + +} diff --git a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/expression/function/scalar/string/LengthFunctionPipe.java b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/expression/function/scalar/string/LengthFunctionPipe.java new file mode 100644 index 00000000000..798b9e1ba1a --- /dev/null +++ b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/expression/function/scalar/string/LengthFunctionPipe.java @@ -0,0 +1,94 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.eql.expression.function.scalar.string; + +import org.elasticsearch.xpack.ql.execution.search.QlSourceBuilder; +import org.elasticsearch.xpack.ql.expression.Expression; +import org.elasticsearch.xpack.ql.expression.gen.pipeline.Pipe; +import org.elasticsearch.xpack.ql.tree.NodeInfo; +import org.elasticsearch.xpack.ql.tree.Source; + +import java.util.Arrays; +import java.util.List; +import java.util.Objects; + +public class LengthFunctionPipe extends Pipe { + + private final Pipe source; + + public LengthFunctionPipe(Source source, Expression expression, Pipe src) { + super(source, expression, Arrays.asList(src)); + this.source = src; + } + + @Override + public final Pipe replaceChildren(List newChildren) { + if (newChildren.size() != 1) { + throw new IllegalArgumentException("expected [1] children but received [" + newChildren.size() + "]"); + } + return replaceChildren(newChildren.get(0)); + } + + @Override + public final Pipe resolveAttributes(AttributeResolver resolver) { + Pipe newSource = source.resolveAttributes(resolver); + if (newSource == source) { + return this; + } + return replaceChildren(newSource); + } + + @Override + public boolean supportedByAggsOnlyQuery() { + return source.supportedByAggsOnlyQuery(); + } + + @Override + public boolean resolved() { + return source.resolved(); + } + + protected Pipe replaceChildren(Pipe newSource) { + return new LengthFunctionPipe(source(), expression(), newSource); + } + + @Override + public final void collectFields(QlSourceBuilder sourceBuilder) { + source.collectFields(sourceBuilder); + } + + @Override + protected NodeInfo info() { + return NodeInfo.create(this, LengthFunctionPipe::new, expression(), source); + } + + @Override + public LengthFunctionProcessor asProcessor() { + return new LengthFunctionProcessor(source.asProcessor()); + } + + public Pipe src() { + return source; + } + + @Override + public int hashCode() { + return Objects.hash(source); + } + + @Override + public boolean equals(Object obj) { + if (this == obj) { + return true; + } + + if (obj == null || getClass() != obj.getClass()) { + return false; + } + + return Objects.equals(source, ((LengthFunctionPipe) obj).source); + } +} diff --git a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/expression/function/scalar/string/LengthFunctionProcessor.java b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/expression/function/scalar/string/LengthFunctionProcessor.java new file mode 100644 index 00000000000..6f58a9df28d --- /dev/null +++ b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/expression/function/scalar/string/LengthFunctionProcessor.java @@ -0,0 +1,78 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.eql.expression.function.scalar.string; + +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.xpack.eql.EqlIllegalArgumentException; +import org.elasticsearch.xpack.ql.expression.gen.processor.Processor; + +import java.io.IOException; +import java.util.Objects; + +public class LengthFunctionProcessor implements Processor { + + public static final String NAME = "slen"; + + private final Processor source; + + public LengthFunctionProcessor(Processor source) { + this.source = source; + } + + public LengthFunctionProcessor(StreamInput in) throws IOException { + source = in.readNamedWriteable(Processor.class); + } + + @Override + public final void writeTo(StreamOutput out) throws IOException { + out.writeNamedWriteable(source); + } + + @Override + public Object process(Object input) { + return doProcess(source.process(input)); + } + + public static Object doProcess(Object source) { + if (source == null) { + return null; + } + if (source instanceof String == false && source instanceof Character == false) { + throw new EqlIllegalArgumentException("A string/char is required; received [{}]", source); + } + + return source.toString().length(); + } + + protected Processor source() { + return source; + } + + @Override + public boolean equals(Object obj) { + if (this == obj) { + return true; + } + + if (obj == null || getClass() != obj.getClass()) { + return false; + } + + return Objects.equals(source(), ((LengthFunctionProcessor) obj).source()); + } + + @Override + public int hashCode() { + return Objects.hash(source()); + } + + + @Override + public String getWriteableName() { + return NAME; + } +} diff --git a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/expression/function/scalar/whitelist/InternalEqlScriptUtils.java b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/expression/function/scalar/whitelist/InternalEqlScriptUtils.java index 959334a73d8..a6b950074f9 100644 --- a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/expression/function/scalar/whitelist/InternalEqlScriptUtils.java +++ b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/expression/function/scalar/whitelist/InternalEqlScriptUtils.java @@ -6,6 +6,7 @@ package org.elasticsearch.xpack.eql.expression.function.scalar.whitelist; +import org.elasticsearch.xpack.eql.expression.function.scalar.string.LengthFunctionProcessor; import org.elasticsearch.xpack.eql.expression.function.scalar.string.SubstringFunctionProcessor; import org.elasticsearch.xpack.ql.expression.function.scalar.whitelist.InternalQlScriptUtils; @@ -18,6 +19,10 @@ public class InternalEqlScriptUtils extends InternalQlScriptUtils { InternalEqlScriptUtils() {} + public static Integer length(String s) { + return (Integer) LengthFunctionProcessor.doProcess(s); + } + public static String substring(String s, Number start, Number end) { return (String) SubstringFunctionProcessor.doProcess(s, start, end); } diff --git a/x-pack/plugin/eql/src/main/resources/org/elasticsearch/xpack/eql/plugin/eql_whitelist.txt b/x-pack/plugin/eql/src/main/resources/org/elasticsearch/xpack/eql/plugin/eql_whitelist.txt index 6839583d14c..fc61e1fe2b5 100644 --- a/x-pack/plugin/eql/src/main/resources/org/elasticsearch/xpack/eql/plugin/eql_whitelist.txt +++ b/x-pack/plugin/eql/src/main/resources/org/elasticsearch/xpack/eql/plugin/eql_whitelist.txt @@ -55,5 +55,6 @@ class org.elasticsearch.xpack.eql.expression.function.scalar.whitelist.InternalE # # ASCII Functions # + Integer length(String) String substring(String, Number, Number) } diff --git a/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/expression/function/scalar/string/LengthProcessorTests.java b/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/expression/function/scalar/string/LengthProcessorTests.java new file mode 100644 index 00000000000..af03a230169 --- /dev/null +++ b/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/expression/function/scalar/string/LengthProcessorTests.java @@ -0,0 +1,41 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.eql.expression.function.scalar.string; + +import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.xpack.ql.QlIllegalArgumentException; +import org.elasticsearch.xpack.ql.expression.Literal; +import org.elasticsearch.xpack.ql.expression.LiteralTests; + +import static org.elasticsearch.xpack.ql.expression.function.scalar.FunctionTestUtils.l; +import static org.elasticsearch.xpack.ql.tree.Source.EMPTY; +import static org.elasticsearch.xpack.ql.type.DataTypes.KEYWORD; +import static org.hamcrest.Matchers.startsWith; + +public class LengthProcessorTests extends ESTestCase { + + public void testLengthFunctionWithValidInput() { + assertEquals(9, new Length(EMPTY, l("foobarbar")).makePipe().asProcessor().process(null)); + assertEquals(0, new Length(EMPTY, l("")).makePipe().asProcessor().process(null)); + assertEquals(1, new Length(EMPTY, l('f')).makePipe().asProcessor().process(null)); + } + + public void testLengthFunctionInputsValidation() { + QlIllegalArgumentException siae = expectThrows(QlIllegalArgumentException.class, + () -> new Length(EMPTY, l(5)).makePipe().asProcessor().process(null)); + assertEquals("A string/char is required; received [5]", siae.getMessage()); + siae = expectThrows(QlIllegalArgumentException.class, () -> new Length(EMPTY, l(true)).makePipe().asProcessor().process(null)); + assertEquals("A string/char is required; received [true]", siae.getMessage()); + } + + public void testLengthFunctionWithRandomInvalidDataType() { + Literal literal = randomValueOtherThanMany(v -> v.dataType() == KEYWORD, () -> LiteralTests.randomLiteral()); + QlIllegalArgumentException siae = expectThrows(QlIllegalArgumentException.class, + () -> new Length(EMPTY, literal).makePipe().asProcessor().process(null)); + assertThat(siae.getMessage(), startsWith("A string/char is required; received")); + } +} diff --git a/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/planner/AbstractQueryFolderTestCase.java b/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/planner/AbstractQueryFolderTestCase.java index 7c54e31db79..b8819bcf4c4 100644 --- a/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/planner/AbstractQueryFolderTestCase.java +++ b/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/planner/AbstractQueryFolderTestCase.java @@ -26,7 +26,7 @@ public abstract class AbstractQueryFolderTestCase extends ESTestCase { protected Optimizer optimizer = new Optimizer(); protected Planner planner = new Planner(); - protected IndexResolution index = IndexResolution.valid(new EsIndex("test", loadMapping("mapping-default.json"))); + protected IndexResolution index = IndexResolution.valid(new EsIndex("test", loadMapping("mapping-default.json", true))); protected PhysicalPlan plan(IndexResolution resolution, String eql) { return planner.plan(optimizer.optimize(analyzer.analyze(preAnalyzer.preAnalyze(parser.createStatement(eql), resolution)))); diff --git a/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/planner/QueryFolderFailTests.java b/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/planner/QueryFolderFailTests.java index c8542e780ff..9029a31715e 100644 --- a/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/planner/QueryFolderFailTests.java +++ b/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/planner/QueryFolderFailTests.java @@ -24,4 +24,12 @@ public class QueryFolderFailTests extends AbstractQueryFolderTestCase { assertEquals("Found 1 problem\nline 1:35: Comparisons against variables are not (currently) supported; " + "offender [parent_process_name] in [process_name in (parent_process_name, \"SYSTEM\")]", msg); } + + public void testLengthFunctionWithInexact() { + VerificationException e = expectThrows(VerificationException.class, + () -> plan("process where length(plain_text) > 0")); + String msg = e.getMessage(); + assertEquals("Found 1 problem\nline 1:15: [length(plain_text)] cannot operate on field of data type [text]: No keyword/multi-field " + + "defined exact matches for [plain_text]; define one or use MATCH/QUERY instead", msg); + } } diff --git a/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/planner/QueryFolderOkTests.java b/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/planner/QueryFolderOkTests.java index 25a4f673b1f..08d2d5e6529 100644 --- a/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/planner/QueryFolderOkTests.java +++ b/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/planner/QueryFolderOkTests.java @@ -95,7 +95,7 @@ public class QueryFolderOkTests extends AbstractQueryFolderTestCase { PhysicalPlan p = plan(query); assertEquals(EsQueryExec.class, p.getClass()); EsQueryExec eqe = (EsQueryExec) p; - assertEquals(25, eqe.output().size()); + assertEquals(27, eqe.output().size()); assertEquals(KEYWORD, eqe.output().get(0).dataType()); final String query = eqe.queryContainer().toString().replaceAll("\\s+", ""); diff --git a/x-pack/plugin/eql/src/test/resources/mapping-default.json b/x-pack/plugin/eql/src/test/resources/mapping-default.json index 6c9ff933705..a065003ddd5 100644 --- a/x-pack/plugin/eql/src/test/resources/mapping-default.json +++ b/x-pack/plugin/eql/src/test/resources/mapping-default.json @@ -81,6 +81,12 @@ }, "exit_code" : { "type" : "long" + }, + "plain_text" : { + "type" : "text" + }, + "constant_keyword" : { + "type" : "constant_keyword" } } } diff --git a/x-pack/plugin/eql/src/test/resources/queryfolder_tests.txt b/x-pack/plugin/eql/src/test/resources/queryfolder_tests.txt index 66bbb5f29b1..a509b41415f 100644 --- a/x-pack/plugin/eql/src/test/resources/queryfolder_tests.txt +++ b/x-pack/plugin/eql/src/test/resources/queryfolder_tests.txt @@ -61,6 +61,27 @@ process where process_path == "*\\red_ttp\\wininit.*" and opcode in (0,1,2,3) {"terms":{"opcode":[0,1,2,3] +lengthFunctionWithExactSubField +process where length(file_name) > 0 +"script":{"source":"InternalQlScriptUtils.nullSafeFilter(InternalQlScriptUtils.gt( +InternalEqlScriptUtils.length(InternalQlScriptUtils.docValue(doc,params.v0)),params.v1))", +"params":{"v0":"file_name.keyword","v1":0} + + +lengthFunctionWithExactField +process where 12 == length(user_name) +"script":{"source":"InternalQlScriptUtils.nullSafeFilter(InternalQlScriptUtils.eq( +InternalEqlScriptUtils.length(InternalQlScriptUtils.docValue(doc,params.v0)),params.v1))", +"params":{"v0":"user_name","v1":12} + + +lengthFunctionWithConstantKeyword +process where 5 > length(constant_keyword) +"script":{"source":"InternalQlScriptUtils.nullSafeFilter(InternalQlScriptUtils.lt( +InternalEqlScriptUtils.length(InternalQlScriptUtils.docValue(doc,params.v0)),params.v1))", +"params":{"v0":"constant_keyword","v1":5} + + substringFunction process where substring(file_name, -4) == '.exe' "script":{"source":"InternalQlScriptUtils.nullSafeFilter(InternalQlScriptUtils.eq(