From 868798e4db96c54b7f249b62341d84873dddd32d Mon Sep 17 00:00:00 2001 From: Aleksandr Maus Date: Tue, 7 Apr 2020 16:52:30 -0400 Subject: [PATCH] EQL: implement between function (#54277) (#54913) --- .../test/eql/CommonEqlActionTestCase.java | 1 + .../resources/test_queries_supported.toml | 15 ++ .../resources/test_queries_unsupported.toml | 41 +---- .../function/EqlFunctionRegistry.java | 2 + .../function/scalar/string/Between.java | 155 ++++++++++++++++++ .../scalar/string/BetweenFunctionPipe.java | 129 +++++++++++++++ .../string/BetweenFunctionProcessor.java | 120 ++++++++++++++ .../function/scalar/string/StringUtils.java | 46 ++++++ .../whitelist/InternalEqlScriptUtils.java | 5 + .../xpack/eql/plugin/eql_whitelist.txt | 1 + .../xpack/eql/analysis/VerifierTests.java | 4 - .../string/BetweenFunctionProcessorTests.java | 37 +++++ .../scalar/string/StringUtilsTests.java | 66 ++++++++ .../eql/planner/QueryFolderFailTests.java | 45 +++++ .../src/test/resources/queryfolder_tests.txt | 10 +- .../xpack/ql/expression/Expressions.java | 4 +- .../expression/function/FunctionRegistry.java | 40 +++++ .../elasticsearch/xpack/ql/util/Check.java | 12 ++ 18 files changed, 691 insertions(+), 42 deletions(-) create mode 100644 x-pack/plugin/eql/qa/common/src/main/resources/test_queries_supported.toml create mode 100644 x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/expression/function/scalar/string/Between.java create mode 100644 x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/expression/function/scalar/string/BetweenFunctionPipe.java create mode 100644 x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/expression/function/scalar/string/BetweenFunctionProcessor.java create mode 100644 x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/expression/function/scalar/string/BetweenFunctionProcessorTests.java diff --git a/x-pack/plugin/eql/qa/common/src/main/java/org/elasticsearch/test/eql/CommonEqlActionTestCase.java b/x-pack/plugin/eql/qa/common/src/main/java/org/elasticsearch/test/eql/CommonEqlActionTestCase.java index 8e42d605234..4622575c4de 100644 --- a/x-pack/plugin/eql/qa/common/src/main/java/org/elasticsearch/test/eql/CommonEqlActionTestCase.java +++ b/x-pack/plugin/eql/qa/common/src/main/java/org/elasticsearch/test/eql/CommonEqlActionTestCase.java @@ -118,6 +118,7 @@ public abstract class CommonEqlActionTestCase extends ESRestTestCase { // Load EQL validation specs List specs = EqlSpecLoader.load("/test_queries.toml", true); + specs.addAll(EqlSpecLoader.load("/test_queries_supported.toml", true)); List unsupportedSpecs = EqlSpecLoader.load("/test_queries_unsupported.toml", false); // Validate only currently supported specs diff --git a/x-pack/plugin/eql/qa/common/src/main/resources/test_queries_supported.toml b/x-pack/plugin/eql/qa/common/src/main/resources/test_queries_supported.toml new file mode 100644 index 00000000000..056c9ee1599 --- /dev/null +++ b/x-pack/plugin/eql/qa/common/src/main/resources/test_queries_supported.toml @@ -0,0 +1,15 @@ +# This file is populated with additional EQL queries that were not present in the original EQL python implementation +# test_queries.toml file in order to keep the original unchanges and easier to sync with the EQL reference implementation tests. + + +[[queries]] +expected_event_ids = [95] +query = ''' +file where between(file_path, "dev", ".json", false) == "\\TestLogs\\something" +''' + +[[queries]] +expected_event_ids = [95] +query = ''' +file where between(file_path, "dev", ".json", true) == "\\TestLogs\\something" +''' diff --git a/x-pack/plugin/eql/qa/common/src/main/resources/test_queries_unsupported.toml b/x-pack/plugin/eql/qa/common/src/main/resources/test_queries_unsupported.toml index f45e477e320..f252030c4cc 100644 --- a/x-pack/plugin/eql/qa/common/src/main/resources/test_queries_unsupported.toml +++ b/x-pack/plugin/eql/qa/common/src/main/resources/test_queries_unsupported.toml @@ -1040,48 +1040,17 @@ query = "file where serial_event_id / 2 == 41" expected_event_ids = [82] query = "file where serial_event_id % 40 == 2" -[[queries]] -expected_event_ids = [1, 2] -query = ''' -process where between(process_name, "s", "e") == "yst" -''' - -[[queries]] -expected_event_ids = [1, 2] -query = ''' -process where between(process_name, "s", "e", false) == "yst" -''' - -[[queries]] -expected_event_ids = [] -query = ''' -process where between(process_name, "s", "e", false, true) == "yst" -''' - -[[queries]] -expected_event_ids = [1, 2, 42] -query = ''' -process where between(process_name, "s", "e", false, true) == "t" -''' - -[[queries]] -expected_event_ids = [1, 2] -query = ''' -process where between(process_name, "S", "e", false, true) == "yst" -''' - -[[queries]] -expected_event_ids = [1] -query = ''' -process where between(process_name, "s", "e", true) == "ystem Idle Proc" -''' - +# The following two "between" queries behave slightly different with elasticsearch +# due to comparison on keyword field would be case-sensitive and would need to be +# file where between(file_path, "dev", ".json", false) == "\\TestLogs\\something" +# blacklisted, check for modified query in test_queries_supported.toml [[queries]] expected_event_ids = [95] query = ''' file where between(file_path, "dev", ".json", false) == "\\testlogs\\something" ''' +# blacklisted, check for modified query in test_queries_supported.toml [[queries]] expected_event_ids = [95] query = ''' diff --git a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/expression/function/EqlFunctionRegistry.java b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/expression/function/EqlFunctionRegistry.java index 7657c4e5b6e..4333f592023 100644 --- a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/expression/function/EqlFunctionRegistry.java +++ b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/expression/function/EqlFunctionRegistry.java @@ -6,6 +6,7 @@ package org.elasticsearch.xpack.eql.expression.function; +import org.elasticsearch.xpack.eql.expression.function.scalar.string.Between; import org.elasticsearch.xpack.eql.expression.function.scalar.string.EndsWith; import org.elasticsearch.xpack.eql.expression.function.scalar.string.Length; import org.elasticsearch.xpack.eql.expression.function.scalar.string.StartsWith; @@ -27,6 +28,7 @@ public class EqlFunctionRegistry extends FunctionRegistry { // Scalar functions // String new FunctionDefinition[] { + def(Between.class, Between::new, 2, "between"), def(EndsWith.class, EndsWith::new, "endswith"), def(Length.class, Length::new, "length"), def(StartsWith.class, StartsWith::new, "startswith"), diff --git a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/expression/function/scalar/string/Between.java b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/expression/function/scalar/string/Between.java new file mode 100644 index 00000000000..a3ee623ded0 --- /dev/null +++ b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/expression/function/scalar/string/Between.java @@ -0,0 +1,155 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.eql.expression.function.scalar.string; + +import org.elasticsearch.xpack.ql.expression.Expression; +import org.elasticsearch.xpack.ql.expression.Expressions; +import org.elasticsearch.xpack.ql.expression.FieldAttribute; +import org.elasticsearch.xpack.ql.expression.Literal; +import org.elasticsearch.xpack.ql.expression.function.OptionalArgument; +import org.elasticsearch.xpack.ql.expression.function.scalar.ScalarFunction; +import org.elasticsearch.xpack.ql.expression.gen.pipeline.Pipe; +import org.elasticsearch.xpack.ql.expression.gen.script.ScriptTemplate; +import org.elasticsearch.xpack.ql.expression.gen.script.Scripts; +import org.elasticsearch.xpack.ql.tree.NodeInfo; +import org.elasticsearch.xpack.ql.tree.Source; +import org.elasticsearch.xpack.ql.type.DataType; +import org.elasticsearch.xpack.ql.type.DataTypes; + +import java.util.Arrays; +import java.util.List; +import java.util.Locale; + +import static java.lang.String.format; +import static org.elasticsearch.xpack.eql.expression.function.scalar.string.BetweenFunctionProcessor.doProcess; +import static org.elasticsearch.xpack.ql.expression.TypeResolutions.isBoolean; +import static org.elasticsearch.xpack.ql.expression.TypeResolutions.isStringAndExact; +import static org.elasticsearch.xpack.ql.expression.gen.script.ParamsBuilder.paramsBuilder; + +/** + * EQL specific between function. + * between(source, left, right[, greedy=false, case_sensitive=false]) + * Extracts a substring from source that’s between left and right substrings + */ +public class Between extends ScalarFunction implements OptionalArgument { + + private final Expression source, left, right, greedy, caseSensitive; + + public Between(Source source, Expression src, Expression left, Expression right, Expression greedy, Expression caseSensitive) { + super(source, Arrays.asList(src, left, right, toDefault(greedy), toDefault(caseSensitive))); + this.source = src; + this.left = left; + this.right = right; + this.greedy = arguments().get(3); + this.caseSensitive = arguments().get(4); + } + + private static Expression toDefault(Expression exp) { + return exp != null ? exp : Literal.FALSE; + } + + @Override + protected TypeResolution resolveType() { + if (!childrenResolved()) { + return new TypeResolution("Unresolved children"); + } + + TypeResolution resolution = isStringAndExact(source, sourceText(), Expressions.ParamOrdinal.FIRST); + if (resolution.unresolved()) { + return resolution; + } + + resolution = isStringAndExact(left, sourceText(), Expressions.ParamOrdinal.SECOND); + if (resolution.unresolved()) { + return resolution; + } + + resolution = isStringAndExact(right, sourceText(), Expressions.ParamOrdinal.THIRD); + if (resolution.unresolved()) { + return resolution; + } + + resolution = isBoolean(greedy, sourceText(), Expressions.ParamOrdinal.FOURTH); + if (resolution.unresolved()) { + return resolution; + } + + return isBoolean(caseSensitive, sourceText(), Expressions.ParamOrdinal.FIFTH); + } + + @Override + protected Pipe makePipe() { + return new BetweenFunctionPipe(source(), this, Expressions.pipe(source), + Expressions.pipe(left), Expressions.pipe(right), + Expressions.pipe(greedy), Expressions.pipe(caseSensitive)); + } + + @Override + public boolean foldable() { + return source.foldable() && left.foldable() && right.foldable() && greedy.foldable() && caseSensitive.foldable(); + } + + @Override + public Object fold() { + return doProcess(source.fold(), left.fold(), right.fold(), greedy.fold(), caseSensitive.fold()); + } + + @Override + protected NodeInfo info() { + return NodeInfo.create(this, Between::new, source, left, right, greedy, caseSensitive); + } + + @Override + public ScriptTemplate asScript() { + ScriptTemplate sourceScript = asScript(source); + ScriptTemplate leftScript = asScript(left); + ScriptTemplate rightScript = asScript(right); + ScriptTemplate greedyScript = asScript(greedy); + ScriptTemplate caseSensitiveScript = asScript(caseSensitive); + + return asScriptFrom(sourceScript, leftScript, rightScript, greedyScript, caseSensitiveScript); + } + + protected ScriptTemplate asScriptFrom(ScriptTemplate sourceScript, ScriptTemplate leftScript, + ScriptTemplate rightScript, ScriptTemplate greedyScript, ScriptTemplate caseSensitiveScript) { + return new ScriptTemplate(format(Locale.ROOT, formatTemplate("{eql}.%s(%s,%s,%s,%s,%s)"), + "between", + sourceScript.template(), + leftScript.template(), + rightScript.template(), + greedyScript.template(), + caseSensitiveScript.template()), + paramsBuilder() + .script(sourceScript.params()) + .script(leftScript.params()) + .script(rightScript.params()) + .script(greedyScript.params()) + .script(caseSensitiveScript.params()) + .build(), dataType()); + } + + @Override + public ScriptTemplate scriptWithField(FieldAttribute field) { + return new ScriptTemplate(processScript(Scripts.DOC_VALUE), + paramsBuilder().variable(field.exactAttribute().name()).build(), + dataType()); + } + + @Override + public DataType dataType() { + return DataTypes.KEYWORD; + } + + @Override + public Expression replaceChildren(List newChildren) { + if (newChildren.size() != 5) { + throw new IllegalArgumentException("expected [5] children but received [" + newChildren.size() + "]"); + } + + return new Between(source(), newChildren.get(0), newChildren.get(1), newChildren.get(2), newChildren.get(3), newChildren.get(4)); + } +} diff --git a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/expression/function/scalar/string/BetweenFunctionPipe.java b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/expression/function/scalar/string/BetweenFunctionPipe.java new file mode 100644 index 00000000000..9526735a867 --- /dev/null +++ b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/expression/function/scalar/string/BetweenFunctionPipe.java @@ -0,0 +1,129 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.eql.expression.function.scalar.string; + +import org.elasticsearch.xpack.ql.execution.search.QlSourceBuilder; +import org.elasticsearch.xpack.ql.expression.Expression; +import org.elasticsearch.xpack.ql.expression.gen.pipeline.Pipe; +import org.elasticsearch.xpack.ql.tree.NodeInfo; +import org.elasticsearch.xpack.ql.tree.Source; + +import java.util.Arrays; +import java.util.List; +import java.util.Objects; + +public class BetweenFunctionPipe extends Pipe { + + private final Pipe source, left, right, greedy, caseSensitive; + + public BetweenFunctionPipe(Source source, Expression expression, Pipe src, Pipe left, Pipe right, Pipe greedy, Pipe caseSensitive) { + super(source, expression, Arrays.asList(src, left, right, greedy, caseSensitive)); + this.source = src; + this.left = left; + this.right = right; + this.greedy = greedy; + this.caseSensitive = caseSensitive; + } + + @Override + public final Pipe replaceChildren(List newChildren) { + if (newChildren.size() != 5) { + throw new IllegalArgumentException("expected [5] children but received [" + newChildren.size() + "]"); + } + return replaceChildren(newChildren.get(0), newChildren.get(1), newChildren.get(2), newChildren.get(3), newChildren.get(4)); + } + + @Override + public final Pipe resolveAttributes(AttributeResolver resolver) { + Pipe newSource = source.resolveAttributes(resolver); + Pipe newLeft = left.resolveAttributes(resolver); + Pipe newRight = right.resolveAttributes(resolver); + Pipe newGreedy = greedy.resolveAttributes(resolver); + Pipe newCaseSensitive = caseSensitive.resolveAttributes(resolver); + if (newSource == source && newLeft == left && newRight == right && newGreedy == greedy && newCaseSensitive == caseSensitive) { + return this; + } + return replaceChildren(newSource, newLeft, newRight, newGreedy, newCaseSensitive); + } + + @Override + public boolean supportedByAggsOnlyQuery() { + return source.supportedByAggsOnlyQuery() && left.supportedByAggsOnlyQuery() && right.supportedByAggsOnlyQuery() + && greedy.supportedByAggsOnlyQuery() && caseSensitive.supportedByAggsOnlyQuery(); + } + + @Override + public boolean resolved() { + return source.resolved() && left.resolved() && right.resolved() && greedy.resolved() && caseSensitive.resolved(); + } + + protected Pipe replaceChildren(Pipe newSource, Pipe newLeft, Pipe newRight, Pipe newGreedy, Pipe newCaseSensitive) { + return new BetweenFunctionPipe(source(), expression(), newSource, newLeft, newRight, newGreedy, newCaseSensitive); + } + + @Override + public final void collectFields(QlSourceBuilder sourceBuilder) { + source.collectFields(sourceBuilder); + left.collectFields(sourceBuilder); + right.collectFields(sourceBuilder); + greedy.collectFields(sourceBuilder); + caseSensitive.collectFields(sourceBuilder); + } + + @Override + protected NodeInfo info() { + return NodeInfo.create(this, BetweenFunctionPipe::new, expression(), source, left, right, greedy, caseSensitive); + } + + @Override + public BetweenFunctionProcessor asProcessor() { + return new BetweenFunctionProcessor(source.asProcessor(), left.asProcessor(), right.asProcessor(), + greedy.asProcessor(), caseSensitive.asProcessor()); + } + + public Pipe src() { + return source; + } + + public Pipe left() { + return left; + } + + public Pipe right() { + return right; + } + + public Pipe greedy() { + return greedy; + } + + public Pipe caseSensitive() { + return caseSensitive; + } + + @Override + public int hashCode() { + return Objects.hash(source(), left(), right(), greedy(), caseSensitive()); + } + + @Override + public boolean equals(Object obj) { + if (this == obj) { + return true; + } + + if (obj == null || getClass() != obj.getClass()) { + return false; + } + + BetweenFunctionPipe other = (BetweenFunctionPipe) obj; + return Objects.equals(source(), other.source()) + && Objects.equals(left(), other.left()) + && Objects.equals(right(), other.right()) + && Objects.equals(greedy(), other.greedy()) + && Objects.equals(caseSensitive(), other.caseSensitive()); + } +} diff --git a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/expression/function/scalar/string/BetweenFunctionProcessor.java b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/expression/function/scalar/string/BetweenFunctionProcessor.java new file mode 100644 index 00000000000..5139cb3421c --- /dev/null +++ b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/expression/function/scalar/string/BetweenFunctionProcessor.java @@ -0,0 +1,120 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.eql.expression.function.scalar.string; + +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.xpack.ql.expression.gen.processor.Processor; +import org.elasticsearch.xpack.ql.util.Check; + +import java.io.IOException; +import java.util.Objects; + +public class BetweenFunctionProcessor implements Processor { + + public static final String NAME = "sbtw"; + + private final Processor source, left, right, greedy, caseSensitive; + + public BetweenFunctionProcessor(Processor source, Processor left, Processor right, Processor greedy, Processor caseSensitive) { + this.source = source; + this.left = left; + this.right = right; + this.greedy = greedy; + this.caseSensitive = caseSensitive; + } + + public BetweenFunctionProcessor(StreamInput in) throws IOException { + source = in.readNamedWriteable(Processor.class); + left = in.readNamedWriteable(Processor.class); + right = in.readNamedWriteable(Processor.class); + greedy = in.readNamedWriteable(Processor.class); + caseSensitive = in.readNamedWriteable(Processor.class); + } + + @Override + public final void writeTo(StreamOutput out) throws IOException { + out.writeNamedWriteable(source); + out.writeNamedWriteable(left); + out.writeNamedWriteable(right); + out.writeNamedWriteable(greedy); + out.writeNamedWriteable(caseSensitive); + } + + @Override + public String getWriteableName() { + return NAME; + } + + @Override + public Object process(Object input) { + return doProcess(source.process(input), left.process(input), right.process(input), + greedy.process(input), caseSensitive.process(input)); + } + + public static Object doProcess(Object source, Object left, Object right, Object greedy, Object caseSensitive) { + if (source == null) { + return null; + } + + Check.isString(source); + Check.isString(left); + Check.isString(right); + + Check.isBoolean(greedy); + Check.isBoolean(caseSensitive); + + String str = source.toString(); + String strRight = right.toString(); + String strLeft = left.toString(); + boolean bGreedy = ((Boolean) greedy).booleanValue(); + boolean bCaseSensitive = ((Boolean) caseSensitive).booleanValue(); + return StringUtils.between(str, strLeft, strRight, bGreedy, bCaseSensitive); + } + + protected Processor source() { + return source; + } + + public Processor left() { + return left; + } + + public Processor right() { + return right; + } + + public Processor greedy() { + return greedy; + } + + public Processor caseSensitive() { + return caseSensitive; + } + + @Override + public int hashCode() { + return Objects.hash(source(), left(), right(), greedy(), caseSensitive()); + } + + @Override + public boolean equals(Object obj) { + if (this == obj) { + return true; + } + + if (obj == null || getClass() != obj.getClass()) { + return false; + } + + BetweenFunctionProcessor other = (BetweenFunctionProcessor) obj; + return Objects.equals(source(), other.source()) + && Objects.equals(left(), other.left()) + && Objects.equals(right(), other.right()) + && Objects.equals(greedy(), other.greedy()) + && Objects.equals(caseSensitive(), other.caseSensitive()); + } +} diff --git a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/expression/function/scalar/string/StringUtils.java b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/expression/function/scalar/string/StringUtils.java index 11657872c7f..c3dcb14a699 100644 --- a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/expression/function/scalar/string/StringUtils.java +++ b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/expression/function/scalar/string/StringUtils.java @@ -8,12 +8,58 @@ package org.elasticsearch.xpack.eql.expression.function.scalar.string; import org.elasticsearch.common.Strings; +import java.util.Locale; + import static org.elasticsearch.common.Strings.hasLength; +import static org.elasticsearch.xpack.ql.util.StringUtils.EMPTY; final class StringUtils { private StringUtils() {} + /** + * Extracts a substring from string between left and right strings. + * Port of "between" function from the original EQL python implementation. + * + * @param string string to search. + * @param left left bounding substring to search for. + * @param right right bounding substring to search for. + * @param greedy match the longest substring if true. + * @param caseSensitive match case when searching for {@code left} and {@code right} strings. + * @return the substring in between {@code left} and {@code right} strings. + */ + static String between(String string, String left, String right, boolean greedy, boolean caseSensitive) { + if (hasLength(string) == false || hasLength(left) == false || hasLength(right) == false) { + return string; + } + + String matchString = string; + if (caseSensitive == false) { + matchString = matchString.toLowerCase(Locale.ROOT); + left = left.toLowerCase(Locale.ROOT); + right = right.toLowerCase(Locale.ROOT); + } + + int idx = matchString.indexOf(left); + if (idx == -1) { + return EMPTY; + } + + int start = idx + left.length(); + + if (greedy) { + idx = matchString.lastIndexOf(right); + } else { + idx = matchString.indexOf(right, start); + } + + if (idx == -1) { + return EMPTY; + } + + return string.substring(start, idx); + } + /** * Returns a substring using the Python slice semantics, meaning * start and end can be negative diff --git a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/expression/function/scalar/whitelist/InternalEqlScriptUtils.java b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/expression/function/scalar/whitelist/InternalEqlScriptUtils.java index 67ff257f9b4..14070d20936 100644 --- a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/expression/function/scalar/whitelist/InternalEqlScriptUtils.java +++ b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/expression/function/scalar/whitelist/InternalEqlScriptUtils.java @@ -6,6 +6,7 @@ package org.elasticsearch.xpack.eql.expression.function.scalar.whitelist; +import org.elasticsearch.xpack.eql.expression.function.scalar.string.BetweenFunctionProcessor; import org.elasticsearch.xpack.eql.expression.function.scalar.string.EndsWithFunctionProcessor; import org.elasticsearch.xpack.eql.expression.function.scalar.string.LengthFunctionProcessor; import org.elasticsearch.xpack.eql.expression.function.scalar.string.StartsWithFunctionProcessor; @@ -21,6 +22,10 @@ public class InternalEqlScriptUtils extends InternalQlScriptUtils { InternalEqlScriptUtils() {} + public static String between(String s, String left, String right, Boolean greedy, Boolean caseSensitive) { + return (String) BetweenFunctionProcessor.doProcess(s, left, right, greedy, caseSensitive); + } + public static Boolean endsWith(String s, String pattern) { return (Boolean) EndsWithFunctionProcessor.doProcess(s, pattern); } diff --git a/x-pack/plugin/eql/src/main/resources/org/elasticsearch/xpack/eql/plugin/eql_whitelist.txt b/x-pack/plugin/eql/src/main/resources/org/elasticsearch/xpack/eql/plugin/eql_whitelist.txt index 8c6e8c7d1f0..783ff9dabfb 100644 --- a/x-pack/plugin/eql/src/main/resources/org/elasticsearch/xpack/eql/plugin/eql_whitelist.txt +++ b/x-pack/plugin/eql/src/main/resources/org/elasticsearch/xpack/eql/plugin/eql_whitelist.txt @@ -55,6 +55,7 @@ class org.elasticsearch.xpack.eql.expression.function.scalar.whitelist.InternalE # # ASCII Functions # + String between(String, String, String, Boolean, Boolean) Boolean endsWith(String, String) Integer length(String) Boolean startsWith(String, String) diff --git a/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/analysis/VerifierTests.java b/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/analysis/VerifierTests.java index 99d83b7adea..0c022597b2b 100644 --- a/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/analysis/VerifierTests.java +++ b/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/analysis/VerifierTests.java @@ -135,12 +135,8 @@ public class VerifierTests extends ESTestCase { error("process where serial_event_id == number('5')")); assertEquals("1:15: Unknown function [concat]", error("process where concat(serial_event_id, ':', process_name, opcode) == '5:winINIT.exe3'")); - assertEquals("1:15: Unknown function [between]", - error("process where between(process_name, \"s\", \"e\") == \"yst\"")); assertEquals("1:15: Unknown function [cidrMatch]", error("network where cidrMatch(source_address, \"192.168.0.0/16\", \"10.6.48.157/8\")")); - assertEquals("1:22: Unknown function [between]", - error("process where length(between(process_name, 'g', 'e')) > 0")); } // Test unsupported array indexes diff --git a/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/expression/function/scalar/string/BetweenFunctionProcessorTests.java b/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/expression/function/scalar/string/BetweenFunctionProcessorTests.java new file mode 100644 index 00000000000..060418692d1 --- /dev/null +++ b/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/expression/function/scalar/string/BetweenFunctionProcessorTests.java @@ -0,0 +1,37 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.eql.expression.function.scalar.string; + +import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.xpack.ql.QlIllegalArgumentException; +import org.elasticsearch.xpack.ql.util.StringUtils; + +import static org.hamcrest.Matchers.equalTo; + +public class BetweenFunctionProcessorTests extends ESTestCase { + public void testNullOrEmptyParameters() throws Exception { + String left = randomBoolean() ? null : randomAlphaOfLength(10); + String right = randomBoolean() ? null : randomAlphaOfLength(10); + Boolean greedy = randomBoolean() ? null : randomBoolean(); + Boolean caseSensitive = randomBoolean() ? null : randomBoolean(); + + String source = randomBoolean() ? null : StringUtils.EMPTY; + + // The source parameter can be null. Expect exception if any of other parameters is null. + if ((source != null) && (left == null || right == null || greedy == null || caseSensitive == null)) { + QlIllegalArgumentException e = expectThrows(QlIllegalArgumentException.class, + () -> BetweenFunctionProcessor.doProcess(source, left, right, greedy, caseSensitive)); + if (left == null || right == null) { + assertThat(e.getMessage(), equalTo("A string/char is required; received [null]")); + } else { + assertThat(e.getMessage(), equalTo("A boolean is required; received [null]")); + } + } else { + assertThat(BetweenFunctionProcessor.doProcess(source, left, right, greedy, caseSensitive), equalTo(source)); + } + } +} diff --git a/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/expression/function/scalar/string/StringUtilsTests.java b/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/expression/function/scalar/string/StringUtilsTests.java index 0abf9c5eb75..d097d02e52e 100644 --- a/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/expression/function/scalar/string/StringUtilsTests.java +++ b/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/expression/function/scalar/string/StringUtilsTests.java @@ -9,6 +9,8 @@ package org.elasticsearch.xpack.eql.expression.function.scalar.string; import org.elasticsearch.test.ESTestCase; import static org.elasticsearch.xpack.eql.expression.function.scalar.string.StringUtils.substringSlice; +import static org.elasticsearch.xpack.ql.util.StringUtils.EMPTY; +import static org.hamcrest.Matchers.equalTo; public class StringUtilsTests extends ESTestCase { @@ -72,4 +74,68 @@ public class StringUtilsTests extends ESTestCase { public void testNullValue() { assertNull(substringSlice(null, 0, 0)); } + + public void testBetweenNullOrEmptyString() throws Exception { + String left = randomAlphaOfLength(10); + String right = randomAlphaOfLength(10); + boolean greedy = randomBoolean(); + boolean caseSensitive = randomBoolean(); + + String string = randomBoolean() ? null : EMPTY; + assertThat(StringUtils.between(string, left, right, greedy, caseSensitive), equalTo(string)); + } + + public void testBetweenEmptyNullLeftRight() throws Exception { + String string = randomAlphaOfLength(10); + String left = randomBoolean() ? null : ""; + String right = randomBoolean() ? null : ""; + boolean greedy = randomBoolean(); + boolean caseSensitive = randomBoolean(); + assertThat(StringUtils.between(string, left, right, greedy, caseSensitive), equalTo(string)); + } + + // Test from EQL doc https://eql.readthedocs.io/en/latest/query-guide/functions.html + public void testBetweenBasicEQLExamples() { + assertThat(StringUtils.between("welcome to event query language", " ", " ", false, false), + equalTo("to")); + assertThat(StringUtils.between("welcome to event query language", " ", " ", true, false), + equalTo("to event query")); + assertThat(StringUtils.between("System Idle Process", "s", "e", true, false), + equalTo("ystem Idle Proc")); + + assertThat(StringUtils.between("C:\\workspace\\dev\\TestLogs\\something.json", "dev", ".json", false, false), + equalTo("\\TestLogs\\something")); + + assertThat(StringUtils.between("C:\\workspace\\dev\\TestLogs\\something.json", "dev", ".json", true, false), + equalTo("\\TestLogs\\something")); + + assertThat(StringUtils.between("System Idle Process", "s", "e", false, false), + equalTo("yst")); + + + assertThat(StringUtils.between("C:\\workspace\\dev\\TestLogs\\something.json", "dev", ".json", false, true), + equalTo("\\TestLogs\\something")); + + assertThat(StringUtils.between("C:\\workspace\\dev\\TestLogs\\something.json", "Test", ".json", false, true), + equalTo("Logs\\something")); + + assertThat(StringUtils.between("C:\\workspace\\dev\\TestLogs\\something.json", "test", ".json", false, true), + equalTo("")); + + assertThat(StringUtils.between("C:\\workspace\\dev\\TestLogs\\something.json", "dev", ".json", true, true), + equalTo("\\TestLogs\\something")); + + assertThat(StringUtils.between("C:\\workspace\\dev\\TestLogs\\something.json", "Test", ".json", true, true), + equalTo("Logs\\something")); + + assertThat(StringUtils.between("C:\\workspace\\dev\\TestLogs\\something.json", "test", ".json", true, true), + equalTo("")); + + assertThat(StringUtils.between("System Idle Process", "S", "e", false, true), + equalTo("yst")); + + assertThat(StringUtils.between("System Idle Process", "Y", "e", false, true), + equalTo("")); + + } } diff --git a/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/planner/QueryFolderFailTests.java b/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/planner/QueryFolderFailTests.java index 18abaa5aede..9ba8acbf36f 100644 --- a/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/planner/QueryFolderFailTests.java +++ b/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/planner/QueryFolderFailTests.java @@ -80,4 +80,49 @@ public class QueryFolderFailTests extends AbstractQueryFolderTestCase { assertEquals("Found 1 problem\n" + "line 1:15: first argument of [wildcard(pid, '*.exe')] must be [string], found value [pid] type [long]", msg); } + + public void testBetweenMissingOrNullParams() { + final String[] queries = { + "process where between() == \"yst\"", + "process where between(process_name) == \"yst\"", + "process where between(process_name, \"s\") == \"yst\"", + "process where between(null) == \"yst\"", + "process where between(process_name, null) == \"yst\"", + "process where between(process_name, \"s\", \"e\", false, false, true) == \"yst\"", + }; + + for (String query : queries) { + ParsingException e = expectThrows(ParsingException.class, + () -> plan(query)); + assertEquals("line 1:16: error building [between]: expects between three and five arguments", e.getMessage()); + } + } + + private String error(String query) { + VerificationException e = expectThrows(VerificationException.class, + () -> plan(query)); + + assertTrue(e.getMessage().startsWith("Found ")); + final String header = "Found 1 problem\nline "; + return e.getMessage().substring(header.length()); + } + + public void testBetweenWrongTypeParams() { + assertEquals("1:15: second argument of [between(process_name, 1, 2)] must be [string], found value [1] type [integer]", + error("process where between(process_name, 1, 2)")); + + assertEquals("1:15: third argument of [between(process_name, \"s\", 2)] must be [string], found value [2] type [integer]", + error("process where between(process_name, \"s\", 2)")); + + assertEquals("1:15: fourth argument of [between(process_name, \"s\", \"e\", 1)] must be [boolean], found value [1] type [integer]", + error("process where between(process_name, \"s\", \"e\", 1)")); + + assertEquals("1:15: fourth argument of [between(process_name, \"s\", \"e\", \"true\")] must be [boolean], " + + "found value [\"true\"] type [keyword]", + error("process where between(process_name, \"s\", \"e\", \"true\")")); + + assertEquals("1:15: fifth argument of [between(process_name, \"s\", \"e\", false, 2)] must be [boolean], " + + "found value [2] type [integer]", + error("process where between(process_name, \"s\", \"e\", false, 2)")); + } } diff --git a/x-pack/plugin/eql/src/test/resources/queryfolder_tests.txt b/x-pack/plugin/eql/src/test/resources/queryfolder_tests.txt index ffd1ac3e787..ea9b35b1efa 100644 --- a/x-pack/plugin/eql/src/test/resources/queryfolder_tests.txt +++ b/x-pack/plugin/eql/src/test/resources/queryfolder_tests.txt @@ -113,6 +113,14 @@ InternalEqlScriptUtils.substring(InternalQlScriptUtils.docValue(doc,params.v0),p "params":{"v0":"file_name.keyword","v1":-4,"v2":null,"v3":".exe"} ; +betweenFunction +process where between(process_name, "s", "e") == "yst" +; +"script":{"source":"InternalQlScriptUtils.nullSafeFilter(InternalQlScriptUtils.eq( +InternalEqlScriptUtils.between(InternalQlScriptUtils.docValue(doc,params.v0),params.v1,params.v2,params.v3,params.v4),params.v5))", +"params":{"v0":"process_name","v1":"s","v2":"e","v3":false,"v4":false,"v5":"yst"} +; + wildcardFunctionSingleArgument process where wildcard(process_path, "*\\red_ttp\\wininit.*") ; @@ -132,4 +140,4 @@ process where wildcard(process_path, "*\\red_ttp\\wininit.*", "*\\abc\\*", "*def "wildcard":{"process_path":{"wildcard":"*\\\\red_ttp\\\\wininit.*" "wildcard":{"process_path":{"wildcard":"*\\\\abc\\\\*" "wildcard":{"process_path":{"wildcard":"*def*" -; \ No newline at end of file +; diff --git a/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/expression/Expressions.java b/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/expression/Expressions.java index fa21d33969c..364dbc8f3ac 100644 --- a/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/expression/Expressions.java +++ b/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/expression/Expressions.java @@ -31,7 +31,8 @@ public final class Expressions { FIRST, SECOND, THIRD, - FOURTH; + FOURTH, + FIFTH; public static ParamOrdinal fromIndex(int index) { switch (index) { @@ -39,6 +40,7 @@ public final class Expressions { case 1: return ParamOrdinal.SECOND; case 2: return ParamOrdinal.THIRD; case 3: return ParamOrdinal.FOURTH; + case 4: return ParamOrdinal.FIFTH; default: return ParamOrdinal.DEFAULT; } } diff --git a/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/expression/function/FunctionRegistry.java b/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/expression/function/FunctionRegistry.java index 0db5318f75b..2ee980ed2a8 100644 --- a/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/expression/function/FunctionRegistry.java +++ b/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/expression/function/FunctionRegistry.java @@ -33,6 +33,16 @@ import static java.util.stream.Collectors.toList; public class FunctionRegistry { + // Translation table for error messaging in the following function + private static final String[] NUM_NAMES = { + "zero", + "one", + "two", + "three", + "four", + "five", + }; + // list of functions grouped by type of functions (aggregate, statistics, math etc) and ordered alphabetically inside each group // a single function will have one entry for itself with its name associated to its instance and, also, one entry for each alias // it has with the alias name associated to the FunctionDefinition instance @@ -403,6 +413,36 @@ public class FunctionRegistry { T build(Source source, Expression src, Expression exp1, Expression exp2, Expression exp3); } + @SuppressWarnings("overloads") // These are ambiguous if you aren't using ctor references but we always do + public static FunctionDefinition def(Class function, + FiveParametersFunctionBuilder ctorRef, + int numOptionalParams, String... names) { + FunctionBuilder builder = (source, children, distinct, cfg) -> { + final int NUM_TOTAL_PARAMS = 5; + boolean hasOptionalParams = OptionalArgument.class.isAssignableFrom(function); + if (hasOptionalParams && (children.size() > NUM_TOTAL_PARAMS || children.size() < NUM_TOTAL_PARAMS - numOptionalParams)) { + throw new QlIllegalArgumentException("expects between " + NUM_NAMES[NUM_TOTAL_PARAMS - numOptionalParams] + + " and " + NUM_NAMES[NUM_TOTAL_PARAMS] + " arguments"); + } else if (hasOptionalParams == false && children.size() != NUM_TOTAL_PARAMS) { + throw new QlIllegalArgumentException("expects exactly " + NUM_NAMES[NUM_TOTAL_PARAMS] + " arguments"); + } + if (distinct) { + throw new QlIllegalArgumentException("does not support DISTINCT yet it was specified"); + } + return ctorRef.build(source, + children.size() > 0 ? children.get(0) : null, + children.size() > 1 ? children.get(1) : null, + children.size() > 2 ? children.get(2) : null, + children.size() > 3 ? children.get(3) : null, + children.size() > 4 ? children.get(4) : null); + }; + return def(function, builder, false, names); + } + + protected interface FiveParametersFunctionBuilder { + T build(Source source, Expression src, Expression exp1, Expression exp2, Expression exp3, Expression exp4); + } + /** * Special method to create function definition for Cast as its * signature is not compatible with {@link UnresolvedFunction} diff --git a/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/util/Check.java b/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/util/Check.java index ae50379a47a..604ab8002df 100644 --- a/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/util/Check.java +++ b/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/util/Check.java @@ -35,4 +35,16 @@ public abstract class Check { throw new QlIllegalArgumentException(message, values); } } + + public static void isString(Object obj) { + if (!(obj instanceof String || obj instanceof Character)) { + throw new QlIllegalArgumentException("A string/char is required; received [{}]", obj); + } + } + + public static void isBoolean(Object obj) { + if (!(obj instanceof Boolean)) { + throw new QlIllegalArgumentException("A boolean is required; received [{}]", obj); + } + } }