From 022f829d8497bc6fdb1a3face8101496c9acc09c Mon Sep 17 00:00:00 2001 From: Ross Wolf <31489089+rw-access@users.noreply.github.com> Date: Fri, 3 Apr 2020 10:14:32 -0600 Subject: [PATCH] EQL: Add wildcard function (#54020) * EQL: Add wildcard function * EQL: Cleanup Wildcard.getArguments * EQL: Cleanup Wildcard and rearrange methods * EQL: Wildcard newline lint * EQL: Make StringUtils function final * EQL: Make Wildcard.asLikes return ScalarFunction * QL: Restore BinaryLogic.java * EQL: Add Wildcard PR feedback * EQL: Add Wildcard verification tests * EQL: Switch wildcard to isFoldable test * EQL: Change wildcard test to numeric field * EQL: Remove Wildcard.get_arguments --- .../src/main/resources/test_queries.toml | 9 +- .../function/EqlFunctionRegistry.java | 6 +- .../function/scalar/string/Wildcard.java | 128 ++++++++++++++++++ .../xpack/eql/optimizer/Optimizer.java | 26 ++-- .../xpack/eql/util/StringUtils.java | 29 ++++ .../xpack/eql/parser/ExpressionTests.java | 2 +- .../eql/planner/QueryFolderFailTests.java | 32 +++++ .../src/test/resources/queryfolder_tests.txt | 18 +++ .../xpack/ql/expression/Expressions.java | 15 +- .../expression/function/FunctionRegistry.java | 22 +++ 10 files changed, 267 insertions(+), 20 deletions(-) create mode 100644 x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/expression/function/scalar/string/Wildcard.java create mode 100644 x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/util/StringUtils.java diff --git a/x-pack/plugin/eql/qa/common/src/main/resources/test_queries.toml b/x-pack/plugin/eql/qa/common/src/main/resources/test_queries.toml index e2ee95c12e2..f69f8c8953e 100644 --- a/x-pack/plugin/eql/qa/common/src/main/resources/test_queries.toml +++ b/x-pack/plugin/eql/qa/common/src/main/resources/test_queries.toml @@ -209,7 +209,14 @@ registry where length(bytes_written_string_list) == 2 and bytes_written_string_l [[queries]] query = ''' -registry where key_path == "*\\MACHINE\\SAM\\SAM\\*\\Account\\Us*ers\\00*03E9\\F"''' +registry where key_path == "*\\MACHINE\\SAM\\SAM\\*\\Account\\Us*ers\\00*03E9\\F" +''' +expected_event_ids = [79] + +[[queries]] +query = ''' +registry where wildcard(key_path, "*\\MACHINE\\SAM\\SAM\\*\\Account\\Us*ers\\00*03E9\\F") +''' expected_event_ids = [79] [[queries]] diff --git a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/expression/function/EqlFunctionRegistry.java b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/expression/function/EqlFunctionRegistry.java index ec297a9ad2b..7657c4e5b6e 100644 --- a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/expression/function/EqlFunctionRegistry.java +++ b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/expression/function/EqlFunctionRegistry.java @@ -10,6 +10,7 @@ import org.elasticsearch.xpack.eql.expression.function.scalar.string.EndsWith; import org.elasticsearch.xpack.eql.expression.function.scalar.string.Length; import org.elasticsearch.xpack.eql.expression.function.scalar.string.StartsWith; import org.elasticsearch.xpack.eql.expression.function.scalar.string.Substring; +import org.elasticsearch.xpack.eql.expression.function.scalar.string.Wildcard; import org.elasticsearch.xpack.ql.expression.function.FunctionDefinition; import org.elasticsearch.xpack.ql.expression.function.FunctionRegistry; @@ -20,7 +21,7 @@ public class EqlFunctionRegistry extends FunctionRegistry { public EqlFunctionRegistry() { super(functions()); } - + private static FunctionDefinition[][] functions() { return new FunctionDefinition[][] { // Scalar functions @@ -29,7 +30,8 @@ public class EqlFunctionRegistry extends FunctionRegistry { def(EndsWith.class, EndsWith::new, "endswith"), def(Length.class, Length::new, "length"), def(StartsWith.class, StartsWith::new, "startswith"), - def(Substring.class, Substring::new, "substring") + def(Substring.class, Substring::new, "substring"), + def(Wildcard.class, Wildcard::new, "wildcard"), } }; } diff --git a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/expression/function/scalar/string/Wildcard.java b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/expression/function/scalar/string/Wildcard.java new file mode 100644 index 00000000000..f3c6e2448d2 --- /dev/null +++ b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/expression/function/scalar/string/Wildcard.java @@ -0,0 +1,128 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.eql.expression.function.scalar.string; + +import org.elasticsearch.xpack.eql.EqlIllegalArgumentException; +import org.elasticsearch.xpack.eql.util.StringUtils; +import org.elasticsearch.xpack.ql.expression.Expression; +import org.elasticsearch.xpack.ql.expression.Expressions; +import org.elasticsearch.xpack.ql.expression.Expressions.ParamOrdinal; +import org.elasticsearch.xpack.ql.expression.function.scalar.ScalarFunction; +import org.elasticsearch.xpack.ql.expression.gen.pipeline.Pipe; +import org.elasticsearch.xpack.ql.expression.gen.script.ScriptTemplate; +import org.elasticsearch.xpack.ql.expression.predicate.logical.Or; +import org.elasticsearch.xpack.ql.expression.predicate.regex.Like; +import org.elasticsearch.xpack.ql.tree.NodeInfo; +import org.elasticsearch.xpack.ql.tree.Source; +import org.elasticsearch.xpack.ql.type.DataType; +import org.elasticsearch.xpack.ql.type.DataTypes; +import org.elasticsearch.xpack.ql.util.CollectionUtils; + +import java.util.Collections; +import java.util.List; + +import static org.elasticsearch.xpack.ql.expression.TypeResolutions.isFoldable; +import static org.elasticsearch.xpack.ql.expression.TypeResolutions.isString; +import static org.elasticsearch.xpack.ql.expression.TypeResolutions.isStringAndExact; + +/** + * EQL wildcard function. Matches the form: + * wildcard(field, "*wildcard*pattern*", ...) + */ +public class Wildcard extends ScalarFunction { + + private final Expression field; + private final List patterns; + + public Wildcard(Source source, Expression field, List patterns) { + super(source, CollectionUtils.combine(Collections.singletonList(field), patterns)); + this.field = field; + this.patterns = patterns; + } + + @Override + protected NodeInfo info() { + return NodeInfo.create(this, Wildcard::new, field, patterns); + } + + @Override + public Expression replaceChildren(List newChildren) { + if (newChildren.size() < 2) { + throw new IllegalArgumentException("expected at least [2] children but received [" + newChildren.size() + "]"); + } + + return new Wildcard(source(), newChildren.get(0), newChildren.subList(1, newChildren.size())); + } + + @Override + public DataType dataType() { + return DataTypes.BOOLEAN; + } + + @Override + protected TypeResolution resolveType() { + if (childrenResolved() == false) { + return new TypeResolution("Unresolved children"); + } + + TypeResolution lastResolution = isStringAndExact(field, sourceText(), ParamOrdinal.FIRST); + if (lastResolution.unresolved()) { + return lastResolution; + } + + int index = 1; + + for (Expression p: patterns) { + + lastResolution = isFoldable(p, sourceText(), ParamOrdinal.fromIndex(index)); + if (lastResolution.unresolved()) { + break; + } + + lastResolution = isString(p, sourceText(), ParamOrdinal.fromIndex(index)); + if (lastResolution.unresolved()) { + break; + } + + index++; + } + + return lastResolution; + } + + @Override + public boolean foldable() { + return Expressions.foldable(children()) && asLikes().foldable(); + } + + @Override + public Object fold() { + return asLikes().fold(); + } + + @Override + protected Pipe makePipe() { + throw new EqlIllegalArgumentException("Wildcard.makePipe() should not be called directly"); + } + + @Override + public ScriptTemplate asScript() { + throw new EqlIllegalArgumentException("Wildcard.asScript() should not be called directly"); + } + + public ScalarFunction asLikes() { + ScalarFunction result = null; + + for (Expression pattern: patterns) { + String wcString = pattern.fold().toString(); + Like like = new Like(source(), field, StringUtils.toLikePattern(wcString)); + result = result == null ? like : new Or(source(), result, like); + } + + return result; + } +} diff --git a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/optimizer/Optimizer.java b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/optimizer/Optimizer.java index aea8042c07e..5b639fd44da 100644 --- a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/optimizer/Optimizer.java +++ b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/optimizer/Optimizer.java @@ -6,6 +6,8 @@ package org.elasticsearch.xpack.eql.optimizer; +import org.elasticsearch.xpack.eql.expression.function.scalar.string.Wildcard; +import org.elasticsearch.xpack.eql.util.StringUtils; import org.elasticsearch.xpack.ql.expression.Expression; import org.elasticsearch.xpack.ql.expression.predicate.logical.Not; import org.elasticsearch.xpack.ql.expression.predicate.nulls.IsNotNull; @@ -14,7 +16,6 @@ import org.elasticsearch.xpack.ql.expression.predicate.operator.comparison.Binar import org.elasticsearch.xpack.ql.expression.predicate.operator.comparison.Equals; import org.elasticsearch.xpack.ql.expression.predicate.operator.comparison.NotEquals; import org.elasticsearch.xpack.ql.expression.predicate.regex.Like; -import org.elasticsearch.xpack.ql.expression.predicate.regex.LikePattern; import org.elasticsearch.xpack.ql.optimizer.OptimizerRules.BooleanLiteralsOnTheRight; import org.elasticsearch.xpack.ql.optimizer.OptimizerRules.BooleanSimplification; import org.elasticsearch.xpack.ql.optimizer.OptimizerRules.CombineBinaryComparisons; @@ -48,6 +49,7 @@ public class Optimizer extends RuleExecutor { new ReplaceNullChecks(), new PropagateEquals(), new CombineBinaryComparisons(), + new ReplaceWildcardFunction(), // prune/elimination new PruneFilters(), new PruneLiteralsInOrderBy() @@ -60,6 +62,14 @@ public class Optimizer extends RuleExecutor { } + private static class ReplaceWildcardFunction extends OptimizerRule { + + @Override + protected LogicalPlan rule(Filter filter) { + return filter.transformExpressionsUp(e -> e instanceof Wildcard ? ((Wildcard) e).asLikes() : e); + } + } + private static class ReplaceWildcards extends OptimizerRule { private static boolean isWildcard(Expression expr) { @@ -70,18 +80,6 @@ public class Optimizer extends RuleExecutor { return false; } - private static LikePattern toLikePattern(String s) { - // pick a character that is guaranteed not to be in the string, because it isn't allowed to escape itself - char escape = 1; - - // replace wildcards with % and escape special characters - String likeString = s.replace("%", escape + "%") - .replace("_", escape + "_") - .replace("*", "%"); - - return new LikePattern(likeString, escape); - } - @Override protected LogicalPlan rule(Filter filter) { return filter.transformExpressionsUp(e -> { @@ -91,7 +89,7 @@ public class Optimizer extends RuleExecutor { if (isWildcard(cmp.right())) { String wcString = cmp.right().fold().toString(); - Expression like = new Like(e.source(), cmp.left(), toLikePattern(wcString)); + Expression like = new Like(e.source(), cmp.left(), StringUtils.toLikePattern(wcString)); if (e instanceof NotEquals) { like = new Not(e.source(), like); diff --git a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/util/StringUtils.java b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/util/StringUtils.java new file mode 100644 index 00000000000..edd98397d0c --- /dev/null +++ b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/util/StringUtils.java @@ -0,0 +1,29 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.eql.util; + +import org.elasticsearch.xpack.ql.expression.predicate.regex.LikePattern; + +public final class StringUtils { + + private StringUtils() {} + + /** + * Convert an EQL wildcard string to a LikePattern. + */ + public static LikePattern toLikePattern(String s) { + // pick a character that is guaranteed not to be in the string, because it isn't allowed to escape itself + char escape = 1; + + // replace wildcards with % and escape special characters + String likeString = s.replace("%", escape + "%") + .replace("_", escape + "_") + .replace("*", "%"); + + return new LikePattern(likeString, escape); + } +} diff --git a/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/parser/ExpressionTests.java b/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/parser/ExpressionTests.java index 5a6c4b6573e..4f0601e4d16 100644 --- a/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/parser/ExpressionTests.java +++ b/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/parser/ExpressionTests.java @@ -52,7 +52,7 @@ public class ExpressionTests extends ESTestCase { } - public void testStrings() throws Exception { + public void testStrings() { assertEquals("hello\"world", unquoteString("'hello\"world'")); assertEquals("hello'world", unquoteString("\"hello'world\"")); assertEquals("hello\nworld", unquoteString("'hello\\nworld'")); diff --git a/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/planner/QueryFolderFailTests.java b/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/planner/QueryFolderFailTests.java index 77209b7c67c..18abaa5aede 100644 --- a/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/planner/QueryFolderFailTests.java +++ b/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/planner/QueryFolderFailTests.java @@ -7,6 +7,7 @@ package org.elasticsearch.xpack.eql.planner; import org.elasticsearch.xpack.eql.analysis.VerificationException; +import org.elasticsearch.xpack.ql.ParsingException; import org.elasticsearch.xpack.ql.QlIllegalArgumentException; public class QueryFolderFailTests extends AbstractQueryFolderTestCase { @@ -48,4 +49,35 @@ public class QueryFolderFailTests extends AbstractQueryFolderTestCase { assertEquals("Found 1 problem\nline 1:15: [startsWith(plain_text, \"foo\")] cannot operate on first argument field of data type " + "[text]: No keyword/multi-field defined exact matches for [plain_text]; define one or use MATCH/QUERY instead", msg); } + + public void testWildcardNotEnoughArguments() { + ParsingException e = expectThrows(ParsingException.class, + () -> plan("process where wildcard(process_name)")); + String msg = e.getMessage(); + assertEquals("line 1:16: error building [wildcard]: expects at least two arguments", msg); + } + + public void testWildcardAgainstVariable() { + VerificationException e = expectThrows(VerificationException.class, + () -> plan("process where wildcard(process_name, parent_process_name)")); + String msg = e.getMessage(); + assertEquals("Found 1 problem\nline 1:15: second argument of [wildcard(process_name, parent_process_name)] " + + "must be a constant, received [parent_process_name]", msg); + } + + public void testWildcardWithNumericPattern() { + VerificationException e = expectThrows(VerificationException.class, + () -> plan("process where wildcard(process_name, 1)")); + String msg = e.getMessage(); + assertEquals("Found 1 problem\n" + + "line 1:15: second argument of [wildcard(process_name, 1)] must be [string], found value [1] type [integer]", msg); + } + + public void testWildcardWithNumericField() { + VerificationException e = expectThrows(VerificationException.class, + () -> plan("process where wildcard(pid, '*.exe')")); + String msg = e.getMessage(); + assertEquals("Found 1 problem\n" + + "line 1:15: first argument of [wildcard(pid, '*.exe')] must be [string], found value [pid] type [long]", msg); + } } diff --git a/x-pack/plugin/eql/src/test/resources/queryfolder_tests.txt b/x-pack/plugin/eql/src/test/resources/queryfolder_tests.txt index 5b8e15b201d..a1ef814495e 100644 --- a/x-pack/plugin/eql/src/test/resources/queryfolder_tests.txt +++ b/x-pack/plugin/eql/src/test/resources/queryfolder_tests.txt @@ -102,3 +102,21 @@ process where substring(file_name, -4) == '.exe' InternalEqlScriptUtils.substring(InternalQlScriptUtils.docValue(doc,params.v0),params.v1,params.v2),params.v3))", "params":{"v0":"file_name.keyword","v1":-4,"v2":null,"v3":".exe"} + +wildcardFunctionSingleArgument +process where wildcard(process_path, "*\\red_ttp\\wininit.*") +"wildcard":{"process_path":{"wildcard":"*\\\\red_ttp\\\\wininit.*" + + +wildcardFunctionTwoArguments +process where wildcard(process_path, "*\\red_ttp\\wininit.*", "*\\abc\\*") +"wildcard":{"process_path":{"wildcard":"*\\\\red_ttp\\\\wininit.*" +"wildcard":{"process_path":{"wildcard":"*\\\\abc\\\\*" + + +wildcardFunctionThreeArguments +process where wildcard(process_path, "*\\red_ttp\\wininit.*", "*\\abc\\*", "*def*") +"wildcard":{"process_path":{"wildcard":"*\\\\red_ttp\\\\wininit.*" +"wildcard":{"process_path":{"wildcard":"*\\\\abc\\\\*" +"wildcard":{"process_path":{"wildcard":"*def*" + diff --git a/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/expression/Expressions.java b/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/expression/Expressions.java index dfb235ba774..fa21d33969c 100644 --- a/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/expression/Expressions.java +++ b/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/expression/Expressions.java @@ -31,9 +31,20 @@ public final class Expressions { FIRST, SECOND, THIRD, - FOURTH + FOURTH; + + public static ParamOrdinal fromIndex(int index) { + switch (index) { + case 0: return ParamOrdinal.FIRST; + case 1: return ParamOrdinal.SECOND; + case 2: return ParamOrdinal.THIRD; + case 3: return ParamOrdinal.FOURTH; + default: return ParamOrdinal.DEFAULT; + } + } } + private Expressions() {} public static NamedExpression wrapAsNamed(Expression exp) { @@ -205,4 +216,4 @@ public final class Expressions { public static String id(Expression e) { return Integer.toHexString(e.hashCode()); } -} \ No newline at end of file +} diff --git a/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/expression/function/FunctionRegistry.java b/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/expression/function/FunctionRegistry.java index 0d1a36edb05..0db5318f75b 100644 --- a/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/expression/function/FunctionRegistry.java +++ b/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/expression/function/FunctionRegistry.java @@ -421,4 +421,26 @@ public class FunctionRegistry { protected interface CastFunctionBuilder { T build(Source source, Expression expression, DataType dataType); } + + @SuppressWarnings("overloads") // These are ambiguous if you aren't using ctor references but we always do + public static FunctionDefinition def(Class function, + TwoParametersVariadicBuilder ctorRef, String... names) { + FunctionBuilder builder = (source, children, distinct, cfg) -> { + boolean hasMinimumOne = OptionalArgument.class.isAssignableFrom(function); + if (hasMinimumOne && children.size() < 1) { + throw new QlIllegalArgumentException("expects at least one argument"); + } else if (!hasMinimumOne && children.size() < 2) { + throw new QlIllegalArgumentException("expects at least two arguments"); + } + if (distinct) { + throw new QlIllegalArgumentException("does not support DISTINCT yet it was specified"); + } + return ctorRef.build(source, children.get(0), children.subList(1, children.size())); + }; + return def(function, builder, false, names); + } + + protected interface TwoParametersVariadicBuilder { + T build(Source source, Expression src, List remaining); + } }