honeymoose
/
OpenSearch
mirror of https://github.com/honeymoose/OpenSearch.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

EQL: Remove wildcard functionality from : (#63276) 

Restrict : operator to only case insensitive matching on strings

Close #63262

(cherry picked from commit bc02e77150cdd85594dfac4f03d8aeb85aaddbb3)
This commit is contained in:
Costin Leau 2020-10-05 22:50:35 +03:00 committed by Costin Leau
parent 22aea11016
commit 6856306dcf
5 changed files with 74 additions and 179 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

134
x-pack/plugin/eql/qa/common/src/main/resources/test_queries.toml
View File

@ -342,19 +342,19 @@ query = '''
registry where length(bytes_written_string_list) == 2 and bytes_written_string_list[1] : "EN"
'''
[[queries]]
name = "keyPathWildcard"
query = '''
registry where key_path : "*\\MACHINE\\SAM\\SAM\\*\\Account\\Us*ers\\00*03E9\\F"
'''
expected_event_ids = [79]
[[queries]]
name = "processPathWildcardAndIN"
query = '''
process where process_path : "*\\red_ttp\\wininit.*" and opcode in (0,1,2,3,4)
'''
expected_event_ids = [84, 85]
# [[queries]]
# name = "keyPathWildcard"
# query = '''
# registry where key_path : "*\\MACHINE\\SAM\\SAM\\*\\Account\\Us*ers\\00*03E9\\F"
# '''
# expected_event_ids = [79]
#
# [[queries]]
# name = "processPathWildcardAndIN"
# query = '''
# process where process_path : "*\\red_ttp\\wininit.*" and opcode in (0,1,2,3,4)
# '''
# expected_event_ids = [84, 85]
[[queries]]
name = "descendant1"
@ -385,13 +385,13 @@ process where opcode==1 and process_name : "smss.exe"
'''
expected_event_ids = [78]
[[queries]]
name = "wildcardAndMultipleConditions1"
query = '''
file where file_path:"*\\red_ttp\\winin*.*"
and opcode in (0,1,2) and user_name:"vagrant"
'''
expected_event_ids = [83, 86]
# [[queries]]
# name = "wildcardAndMultipleConditions1"
# query = '''
# file where file_path:"*\\red_ttp\\winin*.*"
# and opcode in (0,1,2) and user_name:"vagrant"
# '''
# expected_event_ids = [83, 86]
[[queries]]
name = "wildcardAndMultipleConditions2"
@ -401,13 +401,13 @@ file where file_path:"*\\red_ttp\\winin*.*"
'''
expected_event_ids = []
[[queries]]
name = "wildcardAndMultipleConditions3"
query = '''
file where file_path:"*\\red_ttp\\winin*.*"
and opcode not in (3, 4, 5, 6 ,7) and user_name:"vagrant"
'''
expected_event_ids = [83, 86]
# [[queries]]
# name = "wildcardAndMultipleConditions3"
# query = '''
# file where file_path:"*\\red_ttp\\winin*.*"
# and opcode not in (3, 4, 5, 6 ,7) and user_name:"vagrant"
# '''
# expected_event_ids = [83, 86]
[[queries]]
@ -878,16 +878,16 @@ sequence
'''
expected_event_ids = [87, 92]
[[queries]]
name = "doubleSameSequenceWithByUntilAndHead1"
query = '''
sequence
[file where opcode==0 and file_name:"*.exe"] by unique_pid
[file where opcode==0 and file_name:"*.exe"] by unique_pid
until [process where opcode==5000] by unique_ppid
| head 1
'''
expected_event_ids = [55, 61]
# [[queries]]
# name = "doubleSameSequenceWithByUntilAndHead1"
# query = '''
# sequence
# [file where opcode==0 and file_name:"*.exe"] by unique_pid
# [file where opcode==0 and file_name:"*.exe"] by unique_pid
# until [process where opcode==5000] by unique_ppid
# | head 1
# '''
# expected_event_ids = [55, 61]
[[queries]]
name = "doubleSameSequenceWithByUntilAndHead2"
@ -1040,16 +1040,16 @@ query = '''
registry where length(bad_field) > 0
'''
[[queries]]
name = "multipleConditions2"
query = '''
process where opcode == 1
and process_name in ("net.exe", "net1.exe")
and not (parent_process_name : "net.exe"
and process_name : "net1.exe")
and command_line : "*group *admin*" and command_line != "* /add*"
'''
expected_event_ids = [97]
# [[queries]]
# name = "multipleConditions2"
# query = '''
# process where opcode == 1
# and process_name in ("net.exe", "net1.exe")
# and not (parent_process_name : "net.exe"
# and process_name : "net1.exe")
# and command_line : "*group *admin*" and command_line != "* /add*"
# '''
# expected_event_ids = [97]
[[queries]]
name = "anyWithUnique"
@ -1246,26 +1246,26 @@ sequence
'''
expected_event_ids = [54, 55, 56, 54, 61, 62, 54, 67, 68, 54, 72, 73]
[[queries]]
name = "wildcard1"
query = '''
process where command_line : "*%*"
'''
expected_event_ids = [4, 6, 28]
[[queries]]
name = "wildcard2"
query = '''
process where command_line : "*%*%*"
'''
expected_event_ids = [4, 6, 28]
[[queries]]
name = "wildcard3"
query = '''
process where command_line : "%*%*"
'''
expected_event_ids = [4, 6, 28]
# [[queries]]
# name = "wildcard1"
# query = '''
# process where command_line : "*%*"
# '''
# expected_event_ids = [4, 6, 28]
#
# [[queries]]
# name = "wildcard2"
# query = '''
# process where command_line : "*%*%*"
# '''
# expected_event_ids = [4, 6, 28]
#
# [[queries]]
# name = "wildcard3"
# query = '''
# process where command_line : "%*%*"
# '''
# expected_event_ids = [4, 6, 28]
[[queries]]
name = "uniqueCount1"

51
x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/optimizer/Optimizer.java
View File

@ -7,15 +7,12 @@
package org.elasticsearch.xpack.eql.optimizer;
import org.elasticsearch.xpack.eql.EqlIllegalArgumentException;
import org.elasticsearch.xpack.eql.expression.predicate.operator.comparison.InsensitiveBinaryComparison;
import org.elasticsearch.xpack.eql.expression.predicate.operator.comparison.InsensitiveNotEquals;
import org.elasticsearch.xpack.eql.plan.logical.Join;
import org.elasticsearch.xpack.eql.plan.logical.KeyedFilter;
import org.elasticsearch.xpack.eql.plan.logical.LimitWithOffset;
import org.elasticsearch.xpack.eql.plan.physical.LocalRelation;
import org.elasticsearch.xpack.eql.session.Payload.Type;
import org.elasticsearch.xpack.eql.util.MathUtils;
import org.elasticsearch.xpack.eql.util.StringUtils;
import org.elasticsearch.xpack.ql.expression.Expression;
import org.elasticsearch.xpack.ql.expression.Literal;
import org.elasticsearch.xpack.ql.expression.NamedExpression;
@ -24,14 +21,12 @@ import org.elasticsearch.xpack.ql.expression.Order.NullsPosition;
import org.elasticsearch.xpack.ql.expression.Order.OrderDirection;
import org.elasticsearch.xpack.ql.expression.predicate.Predicates;
import org.elasticsearch.xpack.ql.expression.predicate.logical.And;
import org.elasticsearch.xpack.ql.expression.predicate.logical.Not;
import org.elasticsearch.xpack.ql.expression.predicate.logical.Or;
import org.elasticsearch.xpack.ql.expression.predicate.nulls.IsNotNull;
import org.elasticsearch.xpack.ql.expression.predicate.nulls.IsNull;
import org.elasticsearch.xpack.ql.expression.predicate.operator.comparison.BinaryComparison;
import org.elasticsearch.xpack.ql.expression.predicate.operator.comparison.Equals;
import org.elasticsearch.xpack.ql.expression.predicate.operator.comparison.NotEquals;
import org.elasticsearch.xpack.ql.expression.predicate.regex.Like;
import org.elasticsearch.xpack.ql.optimizer.OptimizerRules.BooleanEqualsSimplification;
import org.elasticsearch.xpack.ql.optimizer.OptimizerRules.BooleanLiteralsOnTheRight;
import org.elasticsearch.xpack.ql.optimizer.OptimizerRules.BooleanSimplification;
@ -70,7 +65,6 @@ public class Optimizer extends RuleExecutor<LogicalPlan> {
@Override
protected Iterable<RuleExecutor<LogicalPlan>.Batch> batches() {
Batch substitutions = new Batch("Substitution", Limiter.ONCE,
new ReplaceWildcards(),
new ReplaceSurrogateFunction(),
new ReplaceRegexMatch());
@ -108,51 +102,6 @@ public class Optimizer extends RuleExecutor<LogicalPlan> {
return Arrays.asList(substitutions, operators, constraints, operators, ordering, local, label);
}
private static class ReplaceWildcards extends OptimizerRule<Filter> {
@Override
protected LogicalPlan rule(Filter filter) {
return filter.transformExpressionsUp(e -> {
// expr : "wildcard*phrase" || expr !: "wildcard*phrase"
if (e instanceof InsensitiveBinaryComparison) {
InsensitiveBinaryComparison cmp = (InsensitiveBinaryComparison) e;
Expression target = null;
String wildString = null;
// check either side since the literals can be on both sides
// "a" : "*" is the same as "*" : "a"
if (isWildcard(cmp.left())) {
wildString = (String) cmp.left().fold();
target = cmp.right();
} else if (isWildcard(cmp.right())) {
wildString = (String) cmp.right().fold();
target = cmp.left();
}
if (target != null) {
Expression like = new Like(e.source(), target, StringUtils.toLikePattern(wildString), true);
if (e instanceof InsensitiveNotEquals) {
like = new Not(e.source(), like);
}
e = like;
}
}
return e;
});
}
private static boolean isWildcard(Expression expr) {
if (expr instanceof Literal) {
Object value = expr.fold();
return value instanceof String && ((String) value).contains("*");
}
return false;
}
}
private static class ReplaceNullChecks extends OptimizerRule<Filter> {
@Override

55
x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/optimizer/OptimizerTests.java
View File

@ -36,7 +36,6 @@ import org.elasticsearch.xpack.ql.expression.predicate.nulls.IsNull;
import org.elasticsearch.xpack.ql.expression.predicate.operator.comparison.Equals;
import org.elasticsearch.xpack.ql.expression.predicate.operator.comparison.GreaterThan;
import org.elasticsearch.xpack.ql.expression.predicate.operator.comparison.LessThan;
import org.elasticsearch.xpack.ql.expression.predicate.regex.Like;
import org.elasticsearch.xpack.ql.index.EsIndex;
import org.elasticsearch.xpack.ql.index.IndexResolution;
import org.elasticsearch.xpack.ql.plan.TableIdentifier;
@ -131,60 +130,6 @@ public class OptimizerTests extends ESTestCase {
}
}
public void testEqualsWildcard() {
List<String> tests = Arrays.asList(
"foo where command_line : \"* bar *\"",
"foo where \"* bar *\" : command_line"
);
for (String q : tests) {
LogicalPlan plan = defaultPipes(accept(q));
assertTrue(plan instanceof Filter);
Filter filter = (Filter) plan;
And condition = (And) filter.condition();
assertTrue(condition.right() instanceof Like);
Like like = (Like) condition.right();
assertEquals(((FieldAttribute) like.field()).name(), "command_line");
assertEquals(like.pattern().asJavaRegex(), "^.* bar .*$");
assertEquals(like.pattern().asLuceneWildcard(), "* bar *");
assertEquals(like.pattern().asIndexNameWildcard(), "* bar *");
}
}
// test wildcard gets applied for literals as well regardless of the side used
public void testEqualsWildcardWithLiterals() {
List<String> tests = Arrays.asList(
"foo where \"abc\": \"*b*\"",
"foo where \"*b*\" : \"abc\""
);
for (String q : tests) {
LogicalPlan plan = defaultPipes(accept(q));
assertTrue(plan instanceof Filter);
// check the optimizer kicked in and folding was applied
Filter filter = (Filter) plan;
Equals condition = (Equals) filter.condition();
assertEquals("foo", condition.right().fold());
}
}
public void testWildcardEscapes() {
LogicalPlan plan = defaultPipes(accept("foo where command_line : \"* %bar_ * \\\\ \\n \\r \\t\""));
assertTrue(plan instanceof Filter);
Filter filter = (Filter) plan;
And condition = (And) filter.condition();
assertTrue(condition.right() instanceof Like);
Like like = (Like) condition.right();
assertEquals(((FieldAttribute) like.field()).name(), "command_line");
assertEquals(like.pattern().asJavaRegex(), "^.* %bar_ .* \\\\ \n \r \t$");
assertEquals(like.pattern().asLuceneWildcard(), "* %bar_ * \\\\ \n \r \t");
assertEquals(like.pattern().asIndexNameWildcard(), "* %bar_ * \\ \n \r \t");
}
public void testCombineHeadBigHeadSmall() {
checkOffsetAndLimit(accept("process where true | head 10 | head 1"), 0, 1);
}

1
x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/planner/QueryTranslationTests.java
View File

@ -17,6 +17,7 @@ public class QueryTranslationTests extends AbstractQueryFolderTestCase {
assertThat(asQuery(plan), containsString("\"term\":{\"process_name\""));
}
@AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/63262")
public void testLikeOptimization() throws Exception {
PhysicalPlan plan = plan("process where process_name : \"*\" ");
assertThat(asQuery(plan), containsString("\"exists\":{\"field\":\"process_name\""));

12
x-pack/plugin/eql/src/test/resources/queryfolder_tests.txt
View File

@ -75,12 +75,12 @@ process where process_name in ("python.exe", "SMSS.exe", "explorer.exe")
"terms":{"process_name":["python.exe","SMSS.exe","explorer.exe"],
;
equalsAndInFilter
process where process_path : "*\\red_ttp\\wininit.*" and opcode in (0,1,2,3)
;
"wildcard":{"process_path":{"wildcard":"*\\\\red_ttp\\\\wininit.*","case_insensitive":true,"boost":1.0}}},
{"terms":{"opcode":[0,1,2,3],"boost":1.0}}
;
// equalsAndInFilter
// process where process_path : "*\\red_ttp\\wininit.*" and opcode in (0,1,2,3)
// ;
// "wildcard":{"process_path":{"wildcard":"*\\\\red_ttp\\\\wininit.*","case_insensitive":true,"boost":1.0}}},
// {"terms":{"opcode":[0,1,2,3],"boost":1.0}}
// ;
functionEqualsTrue
process where cidrMatch(source_address, "10.0.0.0/8") == true