honeymoose
/
OpenSearch
mirror of https://github.com/honeymoose/OpenSearch.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

EQL: Convert wildcards to LIKE in analyzer (#51901) 

* EQL: Convert wildcard comparisons to Like
* EQL: Simplify wildcard handling, update tests
* EQL: Lint fixes for Optimizer.java
This commit is contained in:
Ross Wolf 2020-03-06 13:11:59 -07:00
parent f96ad5c32d
commit d6813cb348
No known key found for this signature in database
GPG Key ID: 6A4E50040D9A723A
2 changed files with 168 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

57
x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/optimizer/Optimizer.java
View File

@ -6,14 +6,23 @@
package org.elasticsearch.xpack.eql.optimizer;
import org.elasticsearch.xpack.ql.expression.Expression;
import org.elasticsearch.xpack.ql.expression.predicate.logical.Not;
import org.elasticsearch.xpack.ql.expression.predicate.operator.comparison.BinaryComparison;
import org.elasticsearch.xpack.ql.expression.predicate.operator.comparison.Equals;
import org.elasticsearch.xpack.ql.expression.predicate.operator.comparison.NotEquals;
import org.elasticsearch.xpack.ql.expression.predicate.regex.Like;
import org.elasticsearch.xpack.ql.expression.predicate.regex.LikePattern;
import org.elasticsearch.xpack.ql.optimizer.OptimizerRules.BooleanLiteralsOnTheRight;
import org.elasticsearch.xpack.ql.optimizer.OptimizerRules.BooleanSimplification;
import org.elasticsearch.xpack.ql.optimizer.OptimizerRules.CombineBinaryComparisons;
import org.elasticsearch.xpack.ql.optimizer.OptimizerRules.ConstantFolding;
import org.elasticsearch.xpack.ql.optimizer.OptimizerRules.OptimizerRule;
import org.elasticsearch.xpack.ql.optimizer.OptimizerRules.PropagateEquals;
import org.elasticsearch.xpack.ql.optimizer.OptimizerRules.PruneFilters;
import org.elasticsearch.xpack.ql.optimizer.OptimizerRules.PruneLiteralsInOrderBy;
import org.elasticsearch.xpack.ql.optimizer.OptimizerRules.SetAsOptimized;
import org.elasticsearch.xpack.ql.plan.logical.Filter;
import org.elasticsearch.xpack.ql.plan.logical.LogicalPlan;
import org.elasticsearch.xpack.ql.rule.RuleExecutor;
@ -33,6 +42,7 @@ public class Optimizer extends RuleExecutor<LogicalPlan> {
new BooleanSimplification(),
new BooleanLiteralsOnTheRight(),
// needs to occur before BinaryComparison combinations
new ReplaceWildcards(),
new PropagateEquals(),
new CombineBinaryComparisons(),
// prune/elimination
@ -45,4 +55,51 @@ public class Optimizer extends RuleExecutor<LogicalPlan> {
return Arrays.asList(operators, label);
}
private static class ReplaceWildcards extends OptimizerRule<Filter> {
private static boolean isWildcard(Expression expr) {
if (expr.foldable()) {
Object value = expr.fold();
return value instanceof String && value.toString().contains("*");
}
return false;
}
private static LikePattern toLikePattern(String s) {
// pick a character that is guaranteed not to be in the string, because it isn't allowed to escape itself
char escape = 1;
// replace wildcards with % and escape special characters
String likeString = s.replace("%", escape + "%")
.replace("_", escape + "_")
.replace("*", "%");
return new LikePattern(likeString, escape);
}
@Override
protected LogicalPlan rule(Filter filter) {
return filter.transformExpressionsUp(e -> {
// expr == "wildcard*phrase" || expr != "wildcard*phrase"
if (e instanceof Equals || e instanceof NotEquals) {
BinaryComparison cmp = (BinaryComparison) e;
if (isWildcard(cmp.right())) {
String wcString = cmp.right().fold().toString();
Expression like = new Like(e.source(), cmp.left(), toLikePattern(wcString));
if (e instanceof NotEquals) {
like = new Not(e.source(), like);
}
e = like;
}
}
return e;
});
}
}
}

111
x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/optimizer/OptimizerTests.java Normal file
View File

@ -0,0 +1,111 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License;
* you may not use this file except in compliance with the Elastic License.
*/
package org.elasticsearch.xpack.eql.optimizer;
import org.elasticsearch.test.ESTestCase;
import org.elasticsearch.xpack.eql.analysis.Analyzer;
import org.elasticsearch.xpack.eql.analysis.PreAnalyzer;
import org.elasticsearch.xpack.eql.analysis.Verifier;
import org.elasticsearch.xpack.eql.expression.function.EqlFunctionRegistry;
import org.elasticsearch.xpack.eql.parser.EqlParser;
import org.elasticsearch.xpack.ql.expression.FieldAttribute;
import org.elasticsearch.xpack.ql.expression.predicate.logical.And;
import org.elasticsearch.xpack.ql.expression.predicate.logical.Not;
import org.elasticsearch.xpack.ql.expression.predicate.regex.Like;
import org.elasticsearch.xpack.ql.index.EsIndex;
import org.elasticsearch.xpack.ql.index.IndexResolution;
import org.elasticsearch.xpack.ql.plan.logical.Filter;
import org.elasticsearch.xpack.ql.plan.logical.LogicalPlan;
import org.elasticsearch.xpack.ql.plan.logical.OrderBy;
import org.elasticsearch.xpack.ql.type.EsField;
import org.elasticsearch.xpack.ql.type.TypesTests;
import java.util.Map;
public class OptimizerTests extends ESTestCase {
private static final String INDEX_NAME = "test";
private EqlParser parser = new EqlParser();
private IndexResolution index = loadIndexResolution("mapping-default.json");
private static Map<String, EsField> loadEqlMapping(String name) {
return TypesTests.loadMapping(name);
}
private IndexResolution loadIndexResolution(String name) {
return IndexResolution.valid(new EsIndex(INDEX_NAME, loadEqlMapping(name)));
}
private LogicalPlan accept(IndexResolution resolution, String eql) {
PreAnalyzer preAnalyzer = new PreAnalyzer();
Analyzer analyzer = new Analyzer(new EqlFunctionRegistry(), new Verifier());
Optimizer optimizer = new Optimizer();
return optimizer.optimize(analyzer.analyze(preAnalyzer.preAnalyze(parser.createStatement(eql), resolution)));
}
private LogicalPlan accept(String eql) {
return accept(index, eql);
}
public void testEqualsWildcard() {
for (String q : new String[]{"foo where command_line == '* bar *'", "foo where '* bar *' == command_line"}) {
LogicalPlan plan = accept(q);
assertTrue(plan instanceof OrderBy);
plan = ((OrderBy) plan).child();
assertTrue(plan instanceof Filter);
Filter filter = (Filter) plan;
And condition = (And) filter.condition();
assertTrue(condition.right() instanceof Like);
Like like = (Like) condition.right();
assertEquals(((FieldAttribute) like.field()).name(), "command_line");
assertEquals(like.pattern().asJavaRegex(), "^.* bar .*$");
assertEquals(like.pattern().asLuceneWildcard(), "* bar *");
assertEquals(like.pattern().asIndexNameWildcard(), "* bar *");
}
}
public void testNotEqualsWildcard() {
for (String q : new String[]{"foo where command_line != '* baz *'", "foo where '* baz *' != command_line"}) {
LogicalPlan plan = accept(q);
assertTrue(plan instanceof OrderBy);
plan = ((OrderBy) plan).child();
assertTrue(plan instanceof Filter);
Filter filter = (Filter) plan;
And condition = (And) filter.condition();
assertTrue(condition.right() instanceof Not);
Not not = (Not) condition.right();
Like like = (Like) not.field();
assertEquals(((FieldAttribute) like.field()).name(), "command_line");
assertEquals(like.pattern().asJavaRegex(), "^.* baz .*$");
assertEquals(like.pattern().asLuceneWildcard(), "* baz *");
assertEquals(like.pattern().asIndexNameWildcard(), "* baz *");
}
}
public void testWildcardEscapes() {
LogicalPlan plan = accept("foo where command_line == '* %bar_ * \\\\ \\n \\r \\t'");
assertTrue(plan instanceof OrderBy);
plan = ((OrderBy) plan).child();
assertTrue(plan instanceof Filter);
Filter filter = (Filter) plan;
And condition = (And) filter.condition();
assertTrue(condition.right() instanceof Like);
Like like = (Like) condition.right();
assertEquals(((FieldAttribute) like.field()).name(), "command_line");
assertEquals(like.pattern().asJavaRegex(), "^.* %bar_ .* \\\\ \n \r \t$");
assertEquals(like.pattern().asLuceneWildcard(), "* %bar_ * \\\\ \n \r \t");
assertEquals(like.pattern().asIndexNameWildcard(), "* %bar_ * \\ \n \r \t");
}
}