honeymoose
/
OpenSearch
mirror of https://github.com/honeymoose/OpenSearch.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

EQL: implement cidrMatch function (#54186) (#54928) 

Related to https://github.com/elastic/elasticsearch/issues/54132
This commit is contained in:
Aleksandr Maus 2020-04-07 22:07:28 -04:00 committed by GitHub
parent 65713743c2
commit d02f774cb6
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
9 changed files with 286 additions and 42 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
x-pack/plugin/eql/qa/common/src/main/java/org/elasticsearch/test/eql/CommonEqlActionTestCase.java
View File

@ -16,7 +16,9 @@ import org.elasticsearch.client.RequestOptions;
import org.elasticsearch.client.RestHighLevelClient;
import org.elasticsearch.client.eql.EqlSearchRequest;
import org.elasticsearch.client.eql.EqlSearchResponse;
import org.elasticsearch.client.indices.CreateIndexRequest;
import org.elasticsearch.common.Strings;
import org.elasticsearch.common.io.Streams;
import org.elasticsearch.common.xcontent.XContentParser;
import org.elasticsearch.common.xcontent.XContentType;
import org.elasticsearch.common.xcontent.json.JsonXContent;
@ -56,6 +58,12 @@ public abstract class CommonEqlActionTestCase extends ESRestTestCase {
return;
}
CreateIndexRequest request = new CreateIndexRequest(testIndexName)
.mapping(Streams.readFully(CommonEqlActionTestCase.class.getResourceAsStream("/mapping-default.json")),
XContentType.JSON);
tc.highLevelClient().indices().create(request, RequestOptions.DEFAULT);
BulkRequest bulk = new BulkRequest();
bulk.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE);

80
x-pack/plugin/eql/qa/common/src/main/resources/mapping-default.json Normal file
View File

@ -0,0 +1,80 @@
{
"properties" : {
"command_line" : {
"type" : "keyword"
},
"event" : {
"properties" : {
"category" : {
"type" : "keyword"
}
}
},
"md5" : {
"type" : "keyword"
},
"parent_process_name": {
"type" : "keyword"
},
"parent_process_path": {
"type" : "keyword"
},
"pid" : {
"type" : "long"
},
"ppid" : {
"type" : "long"
},
"process_name": {
"type" : "keyword"
},
"process_path": {
"type" : "keyword"
},
"subtype" : {
"type" : "keyword"
},
"@timestamp" : {
"type" : "date"
},
"user" : {
"type" : "keyword"
},
"user_name" : {
"type" : "keyword"
},
"user_domain": {
"type" : "keyword"
},
"hostname" : {
"type" : "text",
"fields" : {
"keyword" : {
"type" : "keyword",
"ignore_above" : 256
}
}
},
"opcode" : {
"type" : "long"
},
"file_name" : {
"type" : "text",
"fields" : {
"keyword" : {
"type" : "keyword",
"ignore_above" : 256
}
}
},
"serial_event_id" : {
"type" : "long"
},
"source_address" : {
"type" : "ip"
},
"exit_code" : {
"type" : "long"
}
}
}

24
x-pack/plugin/eql/qa/common/src/main/resources/test_queries_unsupported.toml
View File

@ -1057,30 +1057,6 @@ query = '''
file where between(file_path, "dev", ".json", true) == "\\testlogs\\something"
'''
[[queries]]
expected_event_ids = [75304, 75305]
query = '''
network where cidrMatch(source_address, "10.6.48.157/8")
'''
[[queries]]
expected_event_ids = []
query = '''
network where cidrMatch(source_address, "192.168.0.0/16")
'''
[[queries]]
expected_event_ids = [75304, 75305]
query = '''
network where cidrMatch(source_address, "192.168.0.0/16", "10.6.48.157/8")
'''
[[queries]]
expected_event_ids = [75304, 75305]
query = '''
network where cidrMatch(source_address, "0.0.0.0/0")
'''
[[queries]]
expected_event_ids = [7, 14, 22, 29, 44]
query = '''

2
x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/expression/function/EqlFunctionRegistry.java
View File

@ -6,6 +6,7 @@
package org.elasticsearch.xpack.eql.expression.function;
import org.elasticsearch.xpack.eql.expression.function.scalar.string.CIDRMatch;
import org.elasticsearch.xpack.eql.expression.function.scalar.string.Between;
import org.elasticsearch.xpack.eql.expression.function.scalar.string.EndsWith;
import org.elasticsearch.xpack.eql.expression.function.scalar.string.Length;
@ -30,6 +31,7 @@ public class EqlFunctionRegistry extends FunctionRegistry {
// String
new FunctionDefinition[] {
def(Between.class, Between::new, 2, "between"),
def(CIDRMatch.class, CIDRMatch::new, "cidrmatch"),
def(EndsWith.class, EndsWith::new, "endswith"),
def(Length.class, Length::new, "length"),
def(StartsWith.class, StartsWith::new, "startswith"),

114
x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/expression/function/scalar/string/CIDRMatch.java Normal file
View File

@ -0,0 +1,114 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License;
* you may not use this file except in compliance with the Elastic License.
*/
package org.elasticsearch.xpack.eql.expression.function.scalar.string;
import org.elasticsearch.xpack.ql.expression.Expression;
import org.elasticsearch.xpack.ql.expression.Expressions;
import org.elasticsearch.xpack.ql.expression.Expressions.ParamOrdinal;
import org.elasticsearch.xpack.ql.expression.function.scalar.BaseSurrogateFunction;
import org.elasticsearch.xpack.ql.expression.function.scalar.ScalarFunction;
import org.elasticsearch.xpack.ql.expression.predicate.logical.Or;
import org.elasticsearch.xpack.ql.expression.predicate.operator.comparison.Equals;
import org.elasticsearch.xpack.ql.tree.NodeInfo;
import org.elasticsearch.xpack.ql.tree.Source;
import org.elasticsearch.xpack.ql.type.DataType;
import org.elasticsearch.xpack.ql.type.DataTypes;
import org.elasticsearch.xpack.ql.util.CollectionUtils;
import java.util.List;
import static java.util.Collections.singletonList;
import static org.elasticsearch.xpack.ql.expression.TypeResolutions.isFoldable;
import static org.elasticsearch.xpack.ql.expression.TypeResolutions.isIPAndExact;
import static org.elasticsearch.xpack.ql.expression.TypeResolutions.isStringAndExact;
/**
* EQL specific cidrMatch function
* Returns true if the source address matches any of the provided CIDR blocks.
* Refer to: https://eql.readthedocs.io/en/latest/query-guide/functions.html#cidrMatch
*/
public class CIDRMatch extends BaseSurrogateFunction {
private final Expression field;
private final List<Expression> addresses;
public CIDRMatch(Source source, Expression field, List<Expression> addresses) {
super(source, CollectionUtils.combine(singletonList(field), addresses));
this.field = field;
this.addresses = addresses;
}
@Override
protected NodeInfo<? extends Expression> info() {
return NodeInfo.create(this, CIDRMatch::new, field, addresses);
}
@Override
public Expression replaceChildren(List<Expression> newChildren) {
if (newChildren.size() < 2) {
throw new IllegalArgumentException("expected at least [2] children but received [" + newChildren.size() + "]");
}
return new CIDRMatch(source(), newChildren.get(0), newChildren.subList(1, newChildren.size()));
}
@Override
public DataType dataType() {
return DataTypes.BOOLEAN;
}
@Override
protected TypeResolution resolveType() {
if (!childrenResolved()) {
return new TypeResolution("Unresolved children");
}
TypeResolution resolution = isIPAndExact(field, sourceText(), Expressions.ParamOrdinal.FIRST);
if (resolution.unresolved()) {
return resolution;
}
for (Expression addr : addresses) {
// Currently we have limited enum for ordinal numbers
// So just using default here for error messaging
resolution = isStringAndExact(addr, sourceText(), ParamOrdinal.DEFAULT);
if (resolution.unresolved()) {
return resolution;
}
}
int index = 1;
for (Expression addr : addresses) {
resolution = isFoldable(addr, sourceText(), ParamOrdinal.fromIndex(index));
if (resolution.unresolved()) {
break;
}
resolution = isStringAndExact(addr, sourceText(), ParamOrdinal.fromIndex(index));
if (resolution.unresolved()) {
break;
}
index++;
}
return resolution;
}
@Override
public ScalarFunction makeSubstitute() {
ScalarFunction func = null;
for (Expression address : addresses) {
final Equals eq = new Equals(source(), field, address);
func = (func == null) ? eq : new Or(source(), func, eq);
}
return func;
}
}

2
x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/analysis/VerifierTests.java
View File

@ -133,8 +133,6 @@ public class VerifierTests extends ESTestCase {
error("process where serial_event_id == number('5')"));
assertEquals("1:15: Unknown function [concat]",
error("process where concat(serial_event_id, ':', process_name, opcode) == '5:winINIT.exe3'"));
assertEquals("1:15: Unknown function [cidrMatch]",
error("network where cidrMatch(source_address, \"192.168.0.0/16\", \"10.6.48.157/8\")"));
}
// Test unsupported array indexes

63
x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/planner/QueryFolderFailTests.java
View File

@ -62,6 +62,53 @@ public class QueryFolderFailTests extends AbstractQueryFolderTestCase {
error("process where between(process_name, \"s\", \"e\", false, 2)"));
}
public void testCIDRMatchNonIPField() {
VerificationException e = expectThrows(VerificationException.class,
() -> plan("process where cidrMatch(hostname, \"10.0.0.0/8\")"));
String msg = e.getMessage();
assertEquals("Found 1 problem\n" +
"line 1:15: first argument of [cidrMatch(hostname, \"10.0.0.0/8\")] must be [ip], found value [hostname] type [text]", msg);
}
public void testCIDRMatchMissingValue() {
ParsingException e = expectThrows(ParsingException.class,
() -> plan("process where cidrMatch(source_address)"));
String msg = e.getMessage();
assertEquals("line 1:16: error building [cidrmatch]: expects at least two arguments", msg);
}
public void testCIDRMatchAgainstField() {
VerificationException e = expectThrows(VerificationException.class,
() -> plan("process where cidrMatch(source_address, hostname)"));
String msg = e.getMessage();
assertEquals("Found 1 problem\n" +
"line 1:15: second argument of [cidrMatch(source_address, hostname)] must be a constant, received [hostname]", msg);
}
public void testCIDRMatchNonString() {
VerificationException e = expectThrows(VerificationException.class,
() -> plan("process where cidrMatch(source_address, 12345)"));
String msg = e.getMessage();
assertEquals("Found 1 problem\n" +
"line 1:15: argument of [cidrMatch(source_address, 12345)] must be [string], found value [12345] type [integer]", msg);
}
public void testEndsWithFunctionWithInexact() {
VerificationException e = expectThrows(VerificationException.class,
() -> plan("process where endsWith(plain_text, \"foo\") == true"));
String msg = e.getMessage();
assertEquals("Found 1 problem\nline 1:15: [endsWith(plain_text, \"foo\")] cannot operate on first argument field of data type "
+ "[text]: No keyword/multi-field defined exact matches for [plain_text]; define one or use MATCH/QUERY instead", msg);
}
public void testLengthFunctionWithInexact() {
VerificationException e = expectThrows(VerificationException.class,
() -> plan("process where length(plain_text) > 0"));
String msg = e.getMessage();
assertEquals("Found 1 problem\nline 1:15: [length(plain_text)] cannot operate on field of data type [text]: No keyword/multi-field "
+ "defined exact matches for [plain_text]; define one or use MATCH/QUERY instead", msg);
}
public void testPropertyEquationFilterUnsupported() {
QlIllegalArgumentException e = expectThrows(QlIllegalArgumentException.class,
() -> plan("process where (serial_event_id<9 and serial_event_id >= 7) or (opcode == pid)"));
@ -77,22 +124,6 @@ public class QueryFolderFailTests extends AbstractQueryFolderTestCase {
"offender [parent_process_name] in [process_name in (parent_process_name, \"SYSTEM\")]", msg);
}
public void testLengthFunctionWithInexact() {
VerificationException e = expectThrows(VerificationException.class,
() -> plan("process where length(plain_text) > 0"));
String msg = e.getMessage();
assertEquals("Found 1 problem\nline 1:15: [length(plain_text)] cannot operate on field of data type [text]: No keyword/multi-field "
+ "defined exact matches for [plain_text]; define one or use MATCH/QUERY instead", msg);
}
public void testEndsWithFunctionWithInexact() {
VerificationException e = expectThrows(VerificationException.class,
() -> plan("process where endsWith(plain_text, \"foo\") == true"));
String msg = e.getMessage();
assertEquals("Found 1 problem\nline 1:15: [endsWith(plain_text, \"foo\")] cannot operate on first argument field of data type "
+ "[text]: No keyword/multi-field defined exact matches for [plain_text]; define one or use MATCH/QUERY instead", msg);
}
public void testStartsWithFunctionWithInexact() {
VerificationException e = expectThrows(VerificationException.class,
() -> plan("process where startsWith(plain_text, \"foo\") == true"));

21
x-pack/plugin/eql/src/test/resources/queryfolder_tests.txt
View File

@ -129,6 +129,27 @@ InternalEqlScriptUtils.between(InternalQlScriptUtils.docValue(doc,params.v0),par
"params":{"v0":"process_name","v1":"s","v2":"e","v3":false,"v4":false,"v5":"yst"}
;
cidrMatchFunctionOne
process where cidrMatch(source_address, "10.0.0.0/8")
;
"term":{"source_address":{"value":"10.0.0.0/8"
;
cidrMatchFunctionTwo
process where cidrMatch(source_address, "10.0.0.0/8", "192.168.0.0/16")
;
"term":{"source_address":{"value":"10.0.0.0/8"
"term":{"source_address":{"value":"192.168.0.0/16"
;
cidrMatchFunctionThree
process where cidrMatch(source_address, "10.0.0.0/8", "192.168.0.0/16", "2001:db8::/32")
;
"term":{"source_address":{"value":"10.0.0.0/8"
"term":{"source_address":{"value":"192.168.0.0/16"
"term":{"source_address":{"value":"2001:db8::/32"
;
wildcardFunctionSingleArgument
process where wildcard(process_path, "*\\red_ttp\\wininit.*")
;

14
x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/expression/TypeResolutions.java
View File

@ -18,6 +18,7 @@ import java.util.function.Predicate;
import static org.elasticsearch.common.logging.LoggerMessageFormat.format;
import static org.elasticsearch.xpack.ql.expression.Expressions.name;
import static org.elasticsearch.xpack.ql.type.DataTypes.BOOLEAN;
import static org.elasticsearch.xpack.ql.type.DataTypes.IP;
import static org.elasticsearch.xpack.ql.type.DataTypes.NULL;
public final class TypeResolutions {
@ -40,6 +41,10 @@ public final class TypeResolutions {
return isType(e, DataTypes::isString, operationName, paramOrd, "string");
}
public static TypeResolution isIP(Expression e, String operationName, ParamOrdinal paramOrd) {
return isType(e, dt -> dt == IP, operationName, paramOrd, "ip");
}
public static TypeResolution isExact(Expression e, String message) {
if (e instanceof FieldAttribute) {
EsField.Exact exact = ((FieldAttribute) e).getExactInfo();
@ -73,6 +78,15 @@ public final class TypeResolutions {
return isExact(e, operationName, paramOrd);
}
public static TypeResolution isIPAndExact(Expression e, String operationName, ParamOrdinal paramOrd) {
TypeResolution resolution = isIP(e, operationName, paramOrd);
if (resolution.unresolved()) {
return resolution;
}
return isExact(e, operationName, paramOrd);
}
public static TypeResolution isFoldable(Expression e, String operationName, ParamOrdinal paramOrd) {
if (!e.foldable()) {
return new TypeResolution(format(null, "{}argument of [{}] must be a constant, received [{}]",