From d6813cb348e1034fc1af61bc2d22e62d207a004a Mon Sep 17 00:00:00 2001 From: Ross Wolf <31489089+rw-access@users.noreply.github.com> Date: Fri, 6 Mar 2020 13:11:59 -0700 Subject: [PATCH] EQL: Convert wildcards to LIKE in analyzer (#51901) * EQL: Convert wildcard comparisons to Like * EQL: Simplify wildcard handling, update tests * EQL: Lint fixes for Optimizer.java --- .../xpack/eql/optimizer/Optimizer.java | 57 +++++++++ .../xpack/eql/optimizer/OptimizerTests.java | 111 ++++++++++++++++++ 2 files changed, 168 insertions(+) create mode 100644 x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/optimizer/OptimizerTests.java diff --git a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/optimizer/Optimizer.java b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/optimizer/Optimizer.java index f802fa79d12..979c6223800 100644 --- a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/optimizer/Optimizer.java +++ b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/optimizer/Optimizer.java @@ -6,14 +6,23 @@ package org.elasticsearch.xpack.eql.optimizer; +import org.elasticsearch.xpack.ql.expression.Expression; +import org.elasticsearch.xpack.ql.expression.predicate.logical.Not; +import org.elasticsearch.xpack.ql.expression.predicate.operator.comparison.BinaryComparison; +import org.elasticsearch.xpack.ql.expression.predicate.operator.comparison.Equals; +import org.elasticsearch.xpack.ql.expression.predicate.operator.comparison.NotEquals; +import org.elasticsearch.xpack.ql.expression.predicate.regex.Like; +import org.elasticsearch.xpack.ql.expression.predicate.regex.LikePattern; import org.elasticsearch.xpack.ql.optimizer.OptimizerRules.BooleanLiteralsOnTheRight; import org.elasticsearch.xpack.ql.optimizer.OptimizerRules.BooleanSimplification; import org.elasticsearch.xpack.ql.optimizer.OptimizerRules.CombineBinaryComparisons; import org.elasticsearch.xpack.ql.optimizer.OptimizerRules.ConstantFolding; +import org.elasticsearch.xpack.ql.optimizer.OptimizerRules.OptimizerRule; import org.elasticsearch.xpack.ql.optimizer.OptimizerRules.PropagateEquals; import org.elasticsearch.xpack.ql.optimizer.OptimizerRules.PruneFilters; import org.elasticsearch.xpack.ql.optimizer.OptimizerRules.PruneLiteralsInOrderBy; import org.elasticsearch.xpack.ql.optimizer.OptimizerRules.SetAsOptimized; +import org.elasticsearch.xpack.ql.plan.logical.Filter; import org.elasticsearch.xpack.ql.plan.logical.LogicalPlan; import org.elasticsearch.xpack.ql.rule.RuleExecutor; @@ -33,6 +42,7 @@ public class Optimizer extends RuleExecutor { new BooleanSimplification(), new BooleanLiteralsOnTheRight(), // needs to occur before BinaryComparison combinations + new ReplaceWildcards(), new PropagateEquals(), new CombineBinaryComparisons(), // prune/elimination @@ -45,4 +55,51 @@ public class Optimizer extends RuleExecutor { return Arrays.asList(operators, label); } + + + private static class ReplaceWildcards extends OptimizerRule { + + private static boolean isWildcard(Expression expr) { + if (expr.foldable()) { + Object value = expr.fold(); + return value instanceof String && value.toString().contains("*"); + } + return false; + } + + private static LikePattern toLikePattern(String s) { + // pick a character that is guaranteed not to be in the string, because it isn't allowed to escape itself + char escape = 1; + + // replace wildcards with % and escape special characters + String likeString = s.replace("%", escape + "%") + .replace("_", escape + "_") + .replace("*", "%"); + + return new LikePattern(likeString, escape); + } + + @Override + protected LogicalPlan rule(Filter filter) { + return filter.transformExpressionsUp(e -> { + // expr == "wildcard*phrase" || expr != "wildcard*phrase" + if (e instanceof Equals || e instanceof NotEquals) { + BinaryComparison cmp = (BinaryComparison) e; + + if (isWildcard(cmp.right())) { + String wcString = cmp.right().fold().toString(); + Expression like = new Like(e.source(), cmp.left(), toLikePattern(wcString)); + + if (e instanceof NotEquals) { + like = new Not(e.source(), like); + } + + e = like; + } + } + + return e; + }); + } + } } diff --git a/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/optimizer/OptimizerTests.java b/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/optimizer/OptimizerTests.java new file mode 100644 index 00000000000..6b53fec8caa --- /dev/null +++ b/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/optimizer/OptimizerTests.java @@ -0,0 +1,111 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.eql.optimizer; + +import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.xpack.eql.analysis.Analyzer; +import org.elasticsearch.xpack.eql.analysis.PreAnalyzer; +import org.elasticsearch.xpack.eql.analysis.Verifier; +import org.elasticsearch.xpack.eql.expression.function.EqlFunctionRegistry; +import org.elasticsearch.xpack.eql.parser.EqlParser; +import org.elasticsearch.xpack.ql.expression.FieldAttribute; +import org.elasticsearch.xpack.ql.expression.predicate.logical.And; +import org.elasticsearch.xpack.ql.expression.predicate.logical.Not; +import org.elasticsearch.xpack.ql.expression.predicate.regex.Like; +import org.elasticsearch.xpack.ql.index.EsIndex; +import org.elasticsearch.xpack.ql.index.IndexResolution; +import org.elasticsearch.xpack.ql.plan.logical.Filter; +import org.elasticsearch.xpack.ql.plan.logical.LogicalPlan; +import org.elasticsearch.xpack.ql.plan.logical.OrderBy; +import org.elasticsearch.xpack.ql.type.EsField; +import org.elasticsearch.xpack.ql.type.TypesTests; + +import java.util.Map; + +public class OptimizerTests extends ESTestCase { + + + private static final String INDEX_NAME = "test"; + private EqlParser parser = new EqlParser(); + private IndexResolution index = loadIndexResolution("mapping-default.json"); + private static Map loadEqlMapping(String name) { + return TypesTests.loadMapping(name); + } + + private IndexResolution loadIndexResolution(String name) { + return IndexResolution.valid(new EsIndex(INDEX_NAME, loadEqlMapping(name))); + } + + private LogicalPlan accept(IndexResolution resolution, String eql) { + PreAnalyzer preAnalyzer = new PreAnalyzer(); + Analyzer analyzer = new Analyzer(new EqlFunctionRegistry(), new Verifier()); + Optimizer optimizer = new Optimizer(); + return optimizer.optimize(analyzer.analyze(preAnalyzer.preAnalyze(parser.createStatement(eql), resolution))); + } + + private LogicalPlan accept(String eql) { + return accept(index, eql); + } + + + public void testEqualsWildcard() { + for (String q : new String[]{"foo where command_line == '* bar *'", "foo where '* bar *' == command_line"}) { + LogicalPlan plan = accept(q); + assertTrue(plan instanceof OrderBy); + plan = ((OrderBy) plan).child(); + assertTrue(plan instanceof Filter); + + Filter filter = (Filter) plan; + And condition = (And) filter.condition(); + assertTrue(condition.right() instanceof Like); + + Like like = (Like) condition.right(); + assertEquals(((FieldAttribute) like.field()).name(), "command_line"); + assertEquals(like.pattern().asJavaRegex(), "^.* bar .*$"); + assertEquals(like.pattern().asLuceneWildcard(), "* bar *"); + assertEquals(like.pattern().asIndexNameWildcard(), "* bar *"); + } + } + + public void testNotEqualsWildcard() { + for (String q : new String[]{"foo where command_line != '* baz *'", "foo where '* baz *' != command_line"}) { + + LogicalPlan plan = accept(q); + assertTrue(plan instanceof OrderBy); + plan = ((OrderBy) plan).child(); + assertTrue(plan instanceof Filter); + + Filter filter = (Filter) plan; + And condition = (And) filter.condition(); + assertTrue(condition.right() instanceof Not); + + Not not = (Not) condition.right(); + Like like = (Like) not.field(); + assertEquals(((FieldAttribute) like.field()).name(), "command_line"); + assertEquals(like.pattern().asJavaRegex(), "^.* baz .*$"); + assertEquals(like.pattern().asLuceneWildcard(), "* baz *"); + assertEquals(like.pattern().asIndexNameWildcard(), "* baz *"); + } + } + + public void testWildcardEscapes() { + LogicalPlan plan = accept("foo where command_line == '* %bar_ * \\\\ \\n \\r \\t'"); + assertTrue(plan instanceof OrderBy); + plan = ((OrderBy) plan).child(); + assertTrue(plan instanceof Filter); + + Filter filter = (Filter) plan; + And condition = (And) filter.condition(); + assertTrue(condition.right() instanceof Like); + + Like like = (Like) condition.right(); + assertEquals(((FieldAttribute) like.field()).name(), "command_line"); + assertEquals(like.pattern().asJavaRegex(), "^.* %bar_ .* \\\\ \n \r \t$"); + assertEquals(like.pattern().asLuceneWildcard(), "* %bar_ * \\\\ \n \r \t"); + assertEquals(like.pattern().asIndexNameWildcard(), "* %bar_ * \\ \n \r \t"); + } +}