honeymoose
/
OpenSearch
mirror of https://github.com/honeymoose/OpenSearch.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

EQL: Fix cidrMatch function fails to match when used in scripts (#56246) (#56735) 

EQL: Fix cidrMatch function fails to match when used in scripts (#56246)

Addresses https://github.com/elastic/elasticsearch/issues/55709
This commit is contained in:
Aleksandr Maus 2020-05-13 22:41:24 -04:00 committed by GitHub
parent 83e9ff42da
commit 87a10806ab
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
15 changed files with 748 additions and 51 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

38
x-pack/plugin/eql/qa/common/src/main/resources/test_queries_supported.toml
View File

@ -14,6 +14,43 @@ query = '''
file where between(file_path, "dev", ".json", true) == "\\TestLogs\\something"
'''
[[queries]]
expected_event_ids = [75304, 75305]
query = '''
network where cidrMatch(source_address, "10.6.48.157/8") == true
'''
[[queries]]
expected_event_ids = [75304, 75305]
query = '''
network where string(cidrMatch(source_address, "10.6.48.157/8")) == "true"
'''
[[queries]]
expected_event_ids = [75304, 75305]
query = '''
network where true == cidrMatch(source_address, "10.6.48.157/8")
'''
[[queries]]
expected_event_ids = []
query = '''
network where cidrMatch(source_address, "192.168.0.0/16") == true
'''
[[queries]]
expected_event_ids = [75304, 75305]
query = '''
network where cidrMatch(source_address, "192.168.0.0/16", "10.6.48.157/8") == true
'''
[[queries]]
expected_event_ids = [75304, 75305]
query = '''
network where cidrMatch(source_address, "0.0.0.0/0") == true
'''
[[queries]]
description = "test string concatenation. update test to avoid case-sensitivity issues"
query = '''
@ -41,7 +78,6 @@ query = 'process where serial_event_id < 5 and concat(process_name, null, null)
expected_event_ids = [1, 2, 3, 4]
[[queries]]
query = 'process where serial_event_id < 5 and concat(parent_process_name, null) == null'
expected_event_ids = [1, 2, 3, 4]

104
x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/expression/function/scalar/string/CIDRMatch.java
View File

@ -6,58 +6,55 @@
package org.elasticsearch.xpack.eql.expression.function.scalar.string;
import org.elasticsearch.common.logging.LoggerMessageFormat;
import org.elasticsearch.xpack.ql.expression.Expression;
import org.elasticsearch.xpack.ql.expression.Expressions;
import org.elasticsearch.xpack.ql.expression.Expressions.ParamOrdinal;
import org.elasticsearch.xpack.ql.expression.function.scalar.BaseSurrogateFunction;
import org.elasticsearch.xpack.ql.expression.FieldAttribute;
import org.elasticsearch.xpack.ql.expression.function.scalar.ScalarFunction;
import org.elasticsearch.xpack.ql.expression.predicate.logical.Or;
import org.elasticsearch.xpack.ql.expression.predicate.operator.comparison.Equals;
import org.elasticsearch.xpack.ql.expression.gen.pipeline.Pipe;
import org.elasticsearch.xpack.ql.expression.gen.script.ScriptTemplate;
import org.elasticsearch.xpack.ql.expression.gen.script.Scripts;
import org.elasticsearch.xpack.ql.tree.NodeInfo;
import org.elasticsearch.xpack.ql.tree.Source;
import org.elasticsearch.xpack.ql.type.DataType;
import org.elasticsearch.xpack.ql.type.DataTypes;
import org.elasticsearch.xpack.ql.util.CollectionUtils;
import java.util.ArrayList;
import java.util.LinkedHashSet;
import java.util.List;
import static java.util.Collections.emptyList;
import static java.util.Collections.singletonList;
import static org.elasticsearch.xpack.eql.expression.function.scalar.string.CIDRMatchFunctionProcessor.doProcess;
import static org.elasticsearch.xpack.ql.expression.TypeResolutions.isFoldable;
import static org.elasticsearch.xpack.ql.expression.TypeResolutions.isIPAndExact;
import static org.elasticsearch.xpack.ql.expression.TypeResolutions.isStringAndExact;
import static org.elasticsearch.xpack.ql.expression.gen.script.ParamsBuilder.paramsBuilder;
/**
* EQL specific cidrMatch function
* Returns true if the source address matches any of the provided CIDR blocks.
* Refer to: https://eql.readthedocs.io/en/latest/query-guide/functions.html#cidrMatch
*/
public class CIDRMatch extends BaseSurrogateFunction {
public class CIDRMatch extends ScalarFunction {
private final Expression field;
private final List<Expression> addresses;
public CIDRMatch(Source source, Expression field, List<Expression> addresses) {
super(source, CollectionUtils.combine(singletonList(field), addresses));
super(source, CollectionUtils.combine(singletonList(field), addresses == null ? emptyList() : addresses));
this.field = field;
this.addresses = addresses;
this.addresses = addresses == null ? emptyList() : addresses;
}
@Override
protected NodeInfo<? extends Expression> info() {
return NodeInfo.create(this, CIDRMatch::new, field, addresses);
public Expression field() {
return field;
}
@Override
public Expression replaceChildren(List<Expression> newChildren) {
if (newChildren.size() < 2) {
throw new IllegalArgumentException("expected at least [2] children but received [" + newChildren.size() + "]");
}
return new CIDRMatch(source(), newChildren.get(0), newChildren.subList(1, newChildren.size()));
}
@Override
public DataType dataType() {
return DataTypes.BOOLEAN;
public List<Expression> addresses() {
return addresses;
}
@Override
@ -71,15 +68,6 @@ public class CIDRMatch extends BaseSurrogateFunction {
return resolution;
}
for (Expression addr : addresses) {
// Currently we have limited enum for ordinal numbers
// So just using default here for error messaging
resolution = isStringAndExact(addr, sourceText(), ParamOrdinal.DEFAULT);
if (resolution.unresolved()) {
return resolution;
}
}
int index = 1;
for (Expression addr : addresses) {
@ -101,14 +89,60 @@ public class CIDRMatch extends BaseSurrogateFunction {
}
@Override
public ScalarFunction makeSubstitute() {
ScalarFunction func = null;
protected Pipe makePipe() {
ArrayList<Pipe> arr = new ArrayList<>(addresses.size());
for (Expression address : addresses) {
final Equals eq = new Equals(source(), field, address);
func = (func == null) ? eq : new Or(source(), func, eq);
arr.add(Expressions.pipe(address));
}
return new CIDRMatchFunctionPipe(source(), this, Expressions.pipe(field), arr);
}
return func;
@Override
public boolean foldable() {
return field.foldable() && Expressions.foldable(addresses);
}
@Override
public Object fold() {
return doProcess(field.fold(), Expressions.fold(addresses));
}
@Override
protected NodeInfo<? extends Expression> info() {
return NodeInfo.create(this, CIDRMatch::new, field, addresses);
}
@Override
public ScriptTemplate asScript() {
ScriptTemplate leftScript = asScript(field);
List<Object> values = new ArrayList<>(new LinkedHashSet<>(Expressions.fold(addresses)));
return new ScriptTemplate(
formatTemplate(LoggerMessageFormat.format("{eql}.","cidrMatch({}, {})", leftScript.template())),
paramsBuilder()
.script(leftScript.params())
.variable(values)
.build(),
dataType());
}
@Override
public ScriptTemplate scriptWithField(FieldAttribute field) {
return new ScriptTemplate(processScript(Scripts.DOC_VALUE),
paramsBuilder().variable(field.exactAttribute().name()).build(),
dataType());
}
@Override
public DataType dataType() {
return DataTypes.BOOLEAN;
}
@Override
public Expression replaceChildren(List<Expression> newChildren) {
if (newChildren.size() < 2) {
throw new IllegalArgumentException("expected at least [2] children but received [" + newChildren.size() + "]");
}
return new CIDRMatch(source(), newChildren.get(0), newChildren.subList(1, newChildren.size()));
}
}

136
x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/expression/function/scalar/string/CIDRMatchFunctionPipe.java Normal file
View File

@ -0,0 +1,136 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License;
* you may not use this file except in compliance with the Elastic License.
*/
package org.elasticsearch.xpack.eql.expression.function.scalar.string;
import org.elasticsearch.xpack.ql.execution.search.QlSourceBuilder;
import org.elasticsearch.xpack.ql.expression.Expression;
import org.elasticsearch.xpack.ql.expression.gen.pipeline.Pipe;
import org.elasticsearch.xpack.ql.expression.gen.processor.Processor;
import org.elasticsearch.xpack.ql.tree.NodeInfo;
import org.elasticsearch.xpack.ql.tree.Source;
import org.elasticsearch.xpack.ql.util.CollectionUtils;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import java.util.Objects;
public class CIDRMatchFunctionPipe extends Pipe {
private final Pipe source;
private final List<Pipe> addresses;
public CIDRMatchFunctionPipe(Source source, Expression expression, Pipe src, List<Pipe> addresses) {
super(source, expression, CollectionUtils.combine(Collections.singletonList(src), addresses));
this.source = src;
this.addresses = addresses;
}
@Override
public final Pipe replaceChildren(List<Pipe> newChildren) {
if (newChildren.size() < 2) {
throw new IllegalArgumentException("expected at least [2] children but received [" + newChildren.size() + "]");
}
return replaceChildren(newChildren.get(0), newChildren.subList(1, newChildren.size()));
}
@Override
public final Pipe resolveAttributes(AttributeResolver resolver) {
Pipe newSource = source.resolveAttributes(resolver);
boolean same = (newSource == source);
ArrayList<Pipe> newAddresses = new ArrayList<Pipe>(addresses.size());
for (Pipe address : addresses) {
Pipe newAddress = address.resolveAttributes(resolver);
newAddresses.add(newAddress);
same = same && (address == newAddress);
}
if (same) {
return this;
}
return replaceChildren(newSource, newAddresses);
}
@Override
public boolean supportedByAggsOnlyQuery() {
if (source.supportedByAggsOnlyQuery() == false) {
return false;
}
for (Pipe address : addresses) {
if (address.supportedByAggsOnlyQuery() == false) {
return false;
}
}
return true;
}
@Override
public boolean resolved() {
if (source.resolved() == false) {
return false;
}
for (Pipe address : addresses) {
if (address.resolved() == false) {
return false;
}
}
return true;
}
protected Pipe replaceChildren(Pipe newSource, List<Pipe> newAddresses) {
return new CIDRMatchFunctionPipe(source(), expression(), newSource, newAddresses);
}
@Override
public final void collectFields(QlSourceBuilder sourceBuilder) {
source.collectFields(sourceBuilder);
for (Pipe address : addresses) {
address.collectFields(sourceBuilder);
}
}
@Override
protected NodeInfo<CIDRMatchFunctionPipe> info() {
return NodeInfo.create(this, CIDRMatchFunctionPipe::new, expression(), source, addresses);
}
@Override
public CIDRMatchFunctionProcessor asProcessor() {
ArrayList<Processor> processors = new ArrayList<>(addresses.size());
for (Pipe address: addresses) {
processors.add(address.asProcessor());
}
return new CIDRMatchFunctionProcessor(source.asProcessor(), processors);
}
public Pipe src() {
return source;
}
public List<Pipe> addresses() {
return addresses;
}
@Override
public int hashCode() {
return Objects.hash(source(), addresses());
}
@Override
public boolean equals(Object obj) {
if (this == obj) {
return true;
}
if (obj == null || getClass() != obj.getClass()) {
return false;
}
CIDRMatchFunctionPipe other = (CIDRMatchFunctionPipe) obj;
return Objects.equals(source(), other.source())
&& Objects.equals(addresses(), other.addresses());
}
}

100
x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/expression/function/scalar/string/CIDRMatchFunctionProcessor.java Normal file
View File

@ -0,0 +1,100 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License;
* you may not use this file except in compliance with the Elastic License.
*/
package org.elasticsearch.xpack.eql.expression.function.scalar.string;
import org.elasticsearch.common.io.stream.StreamInput;
import org.elasticsearch.common.io.stream.StreamOutput;
import org.elasticsearch.xpack.ql.expression.gen.processor.Processor;
import org.elasticsearch.xpack.ql.util.Check;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
import java.util.Objects;
public class CIDRMatchFunctionProcessor implements Processor {
public static final String NAME = "cdrm";
private final Processor source;
private final List<Processor> addresses;
public CIDRMatchFunctionProcessor(Processor source, List<Processor> addresses) {
this.source = source;
this.addresses = addresses;
}
public CIDRMatchFunctionProcessor(StreamInput in) throws IOException {
source = in.readNamedWriteable(Processor.class);
addresses = in.readNamedWriteableList(Processor.class);
}
@Override
public final void writeTo(StreamOutput out) throws IOException {
out.writeNamedWriteable(source);
out.writeNamedWriteableList(addresses);
}
@Override
public String getWriteableName() {
return NAME;
}
@Override
public Object process(Object input) {
Object src = source.process(input);
ArrayList<Object> arr = new ArrayList<>(addresses.size());
for (Processor address : addresses) {
arr.add(address.process(input));
}
return doProcess(src, arr);
}
public static Object doProcess(Object source, List<Object> addresses) {
if (source == null) {
return null;
}
Check.isString(source);
String[] arr = new String[addresses.size()];
int i = 0;
for (Object address: addresses) {
Check.isString(address);
arr[i++] = (String)address;
}
return CIDRUtils.isInRange((String)source, arr);
}
protected Processor source() {
return source;
}
public List<Processor> addresses() {
return addresses;
}
@Override
public int hashCode() {
return Objects.hash(source(), addresses());
}
@Override
public boolean equals(Object obj) {
if (this == obj) {
return true;
}
if (obj == null || getClass() != obj.getClass()) {
return false;
}
CIDRMatchFunctionProcessor other = (CIDRMatchFunctionProcessor) obj;
return Objects.equals(source(), other.source())
&& Objects.equals(addresses(), other.addresses());
}
}

96
x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/expression/function/scalar/string/CIDRUtils.java Normal file
View File

@ -0,0 +1,96 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License;
* you may not use this file except in compliance with the Elastic License.
*/
package org.elasticsearch.xpack.eql.expression.function.scalar.string;
import org.apache.lucene.util.FutureArrays;
import org.elasticsearch.common.collect.Tuple;
import org.elasticsearch.common.network.InetAddresses;
import org.elasticsearch.xpack.eql.EqlIllegalArgumentException;
import java.net.InetAddress;
public class CIDRUtils {
// Borrowed from Lucene, rfc4291 prefix
static final byte[] IPV4_PREFIX = new byte[]{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, -1, -1};
private CIDRUtils() {
}
public static boolean isInRange(String address, String... cidrAddresses) {
try {
// Check if address is parsable first
byte[] addr = InetAddresses.forString(address).getAddress();
if (cidrAddresses == null || cidrAddresses.length == 0) {
return false;
}
for (String cidrAddress : cidrAddresses) {
if (cidrAddress == null) continue;
byte[] lower, upper;
if (cidrAddress.contains("/")) {
final Tuple<byte[], byte[]> range = getLowerUpper(InetAddresses.parseCidr(cidrAddress));
lower = range.v1();
upper = range.v2();
} else {
lower = InetAddresses.forString(cidrAddress).getAddress();
upper = lower;
}
if (isBetween(addr, lower, upper)) return true;
}
} catch (IllegalArgumentException e) {
throw new EqlIllegalArgumentException(e.getMessage());
}
return false;
}
private static Tuple<byte[], byte[]> getLowerUpper(Tuple<InetAddress, Integer> cidr) {
final InetAddress value = cidr.v1();
final Integer prefixLength = cidr.v2();
if (prefixLength < 0 || prefixLength > 8 * value.getAddress().length) {
throw new IllegalArgumentException("illegal prefixLength '" + prefixLength +
"'. Must be 0-32 for IPv4 ranges, 0-128 for IPv6 ranges");
}
byte[] lower = value.getAddress();
byte[] upper = value.getAddress();
// Borrowed from Lucene
for (int i = prefixLength; i < 8 * lower.length; i++) {
int m = 1 << (7 - (i & 7));
lower[i >> 3] &= ~m;
upper[i >> 3] |= m;
}
return new Tuple<>(lower, upper);
}
private static boolean isBetween(byte[] addr, byte[] lower, byte[] upper) {
// Encode the addresses bytes if lengths do not match
if (addr.length != lower.length) {
addr = encode(addr);
lower = encode(lower);
upper = encode(upper);
}
return FutureArrays.compareUnsigned(lower, 0, lower.length, addr, 0, addr.length) <= 0 &&
FutureArrays.compareUnsigned(upper, 0, upper.length, addr, 0, addr.length) >= 0;
}
// Borrowed from Lucene to make this consistent IP fields matching for the mix of IPv4 and IPv6 values
// Modified signature to avoid extra conversions
private static byte[] encode(byte[] address) {
if (address.length == 4) {
byte[] mapped = new byte[16];
System.arraycopy(IPV4_PREFIX, 0, mapped, 0, IPV4_PREFIX.length);
System.arraycopy(address, 0, mapped, IPV4_PREFIX.length, address.length);
address = mapped;
} else if (address.length != 16) {
throw new UnsupportedOperationException("Only IPv4 and IPv6 addresses are supported");
}
return address;
}
}

5
x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/expression/function/scalar/whitelist/InternalEqlScriptUtils.java
View File

@ -7,6 +7,7 @@
package org.elasticsearch.xpack.eql.expression.function.scalar.whitelist;
import org.elasticsearch.xpack.eql.expression.function.scalar.string.BetweenFunctionProcessor;
import org.elasticsearch.xpack.eql.expression.function.scalar.string.CIDRMatchFunctionProcessor;
import org.elasticsearch.xpack.eql.expression.function.scalar.string.ConcatFunctionProcessor;
import org.elasticsearch.xpack.eql.expression.function.scalar.string.EndsWithFunctionProcessor;
import org.elasticsearch.xpack.eql.expression.function.scalar.string.IndexOfFunctionProcessor;
@ -32,6 +33,10 @@ public class InternalEqlScriptUtils extends InternalQlScriptUtils {
return (String) BetweenFunctionProcessor.doProcess(s, left, right, greedy, caseSensitive);
}
public static Boolean cidrMatch(String s, List<Object> addresses) {
return (Boolean) CIDRMatchFunctionProcessor.doProcess(s, addresses);
}
public static String concat(List<Object> values) {
return (String) ConcatFunctionProcessor.doProcess(values);
}

19
x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/planner/EqlTranslatorHandler.java Normal file
View File

@ -0,0 +1,19 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License;
* you may not use this file except in compliance with the Elastic License.
*/
package org.elasticsearch.xpack.eql.planner;
import org.elasticsearch.xpack.ql.expression.Expression;
import org.elasticsearch.xpack.ql.planner.QlTranslatorHandler;
import org.elasticsearch.xpack.ql.querydsl.query.Query;
public class EqlTranslatorHandler extends QlTranslatorHandler {
@Override
public Query asQuery(Expression e) {
return QueryTranslator.toQuery(e, this);
}
}

4
x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/planner/QueryFolder.java
View File

@ -69,7 +69,7 @@ class QueryFolder extends RuleExecutor<PhysicalPlan> {
EsQueryExec exec = (EsQueryExec) plan.child();
QueryContainer qContainer = exec.queryContainer();
Query query = ExpressionTranslators.toQuery(plan.condition());
Query query = QueryTranslator.toQuery(plan.condition());
if (qContainer.query() != null || query != null) {
query = ExpressionTranslators.and(plan.source(), qContainer.query(), query);
@ -139,4 +139,4 @@ class QueryFolder extends RuleExecutor<PhysicalPlan> {
@Override
protected abstract PhysicalPlan rule(SubPlan plan);
}
}
}

116
x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/planner/QueryTranslator.java Normal file
View File

@ -0,0 +1,116 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License;
* you may not use this file except in compliance with the Elastic License.
*/
package org.elasticsearch.xpack.eql.planner;
import org.elasticsearch.xpack.eql.expression.function.scalar.string.CIDRMatch;
import org.elasticsearch.xpack.ql.QlIllegalArgumentException;
import org.elasticsearch.xpack.ql.expression.Expression;
import org.elasticsearch.xpack.ql.expression.Expressions;
import org.elasticsearch.xpack.ql.expression.FieldAttribute;
import org.elasticsearch.xpack.ql.expression.function.scalar.ScalarFunction;
import org.elasticsearch.xpack.ql.expression.predicate.logical.And;
import org.elasticsearch.xpack.ql.expression.predicate.logical.Or;
import org.elasticsearch.xpack.ql.planner.ExpressionTranslator;
import org.elasticsearch.xpack.ql.planner.ExpressionTranslators;
import org.elasticsearch.xpack.ql.planner.TranslatorHandler;
import org.elasticsearch.xpack.ql.querydsl.query.Query;
import org.elasticsearch.xpack.ql.querydsl.query.ScriptQuery;
import org.elasticsearch.xpack.ql.querydsl.query.TermsQuery;
import org.elasticsearch.xpack.ql.util.CollectionUtils;
import java.util.Arrays;
import java.util.LinkedHashSet;
import java.util.List;
import java.util.Set;
import static org.elasticsearch.xpack.ql.planner.ExpressionTranslators.and;
import static org.elasticsearch.xpack.ql.planner.ExpressionTranslators.or;
final class QueryTranslator {
public static final List<ExpressionTranslator<?>> QUERY_TRANSLATORS = Arrays.asList(
new ExpressionTranslators.BinaryComparisons(),
new ExpressionTranslators.Ranges(),
new BinaryLogic(),
new ExpressionTranslators.Nots(),
new ExpressionTranslators.Likes(),
new ExpressionTranslators.InComparisons(),
new ExpressionTranslators.StringQueries(),
new ExpressionTranslators.Matches(),
new ExpressionTranslators.MultiMatches(),
new Scalars()
);
public static Query toQuery(Expression e) {
return toQuery(e, new EqlTranslatorHandler());
}
public static Query toQuery(Expression e, TranslatorHandler handler) {
Query translation = null;
for (ExpressionTranslator<?> translator : QUERY_TRANSLATORS) {
translation = translator.translate(e, handler);
if (translation != null) {
return translation;
}
}
throw new QlIllegalArgumentException("Don't know how to translate {} {}", e.nodeName(), e);
}
public static class BinaryLogic extends ExpressionTranslator<org.elasticsearch.xpack.ql.expression.predicate.logical.BinaryLogic> {
@Override
protected Query asQuery(org.elasticsearch.xpack.ql.expression.predicate.logical.BinaryLogic e, TranslatorHandler handler) {
if (e instanceof And) {
return and(e.source(), toQuery(e.left(), handler), toQuery(e.right(), handler));
}
if (e instanceof Or) {
return or(e.source(), toQuery(e.left(), handler), toQuery(e.right(), handler));
}
return null;
}
}
public static Object valueOf(Expression e) {
if (e.foldable()) {
return e.fold();
}
throw new QlIllegalArgumentException("Cannot determine value for {}", e);
}
public static class Scalars extends ExpressionTranslator<ScalarFunction> {
@Override
protected Query asQuery(ScalarFunction f, TranslatorHandler handler) {
return doTranslate(f, handler);
}
public static Query doTranslate(ScalarFunction f, TranslatorHandler handler) {
Query q = ExpressionTranslators.Scalars.doKnownTranslate(f, handler);
if (q != null) {
return q;
}
if (f instanceof CIDRMatch) {
CIDRMatch cm = (CIDRMatch) f;
if (cm.field() instanceof FieldAttribute && Expressions.foldable(cm.addresses())) {
String targetFieldName = handler.nameOf(((FieldAttribute) cm.field()).exactAttribute());
Set<Object> set = new LinkedHashSet<>(CollectionUtils.mapSize(cm.addresses().size()));
for (Expression e : cm.addresses()) {
set.add(valueOf(e));
}
return new TermsQuery(f.source(), targetFieldName, set);
}
}
return handler.wrapFunctionQuery(f, f, new ScriptQuery(f.source(), f.asScript()));
}
}
}

1
x-pack/plugin/eql/src/main/resources/org/elasticsearch/xpack/eql/plugin/eql_whitelist.txt
View File

@ -66,6 +66,7 @@ class org.elasticsearch.xpack.eql.expression.function.scalar.whitelist.InternalE
# ASCII Functions
#
String between(String, String, String, Boolean, Boolean)
Boolean cidrMatch(String, java.util.List)
String concat(java.util.List)
Boolean endsWith(String, String)
Integer indexOf(String, String, Number)

73
x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/expression/function/scalar/string/CIDRMatchProcessorTests.java Normal file
View File

@ -0,0 +1,73 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License;
* you may not use this file except in compliance with the Elastic License.
*/
package org.elasticsearch.xpack.eql.expression.function.scalar.string;
import org.elasticsearch.test.ESTestCase;
import org.elasticsearch.xpack.eql.EqlIllegalArgumentException;
import org.elasticsearch.xpack.ql.QlIllegalArgumentException;
import org.elasticsearch.xpack.ql.expression.Expression;
import java.util.ArrayList;
import static org.elasticsearch.xpack.ql.expression.function.scalar.FunctionTestUtils.l;
import static org.elasticsearch.xpack.ql.tree.Source.EMPTY;
public class CIDRMatchProcessorTests extends ESTestCase {
public void testCIDRMatchFunctionValidInput() {
// Expects null if source was null
assertNull(new CIDRMatch(EMPTY, l(null), null).makePipe().asProcessor().process(null));
ArrayList<Expression> addresses = new ArrayList<>();
assertNull(new CIDRMatch(EMPTY, l(null), addresses).makePipe().asProcessor().process(null));
assertFalse((Boolean) new CIDRMatch(EMPTY, l("10.6.48.157"), addresses).makePipe().asProcessor().process(null));
addresses.add(l("10.6.48.157/8"));
assertTrue((Boolean) new CIDRMatch(EMPTY, l("10.6.48.157"), addresses).makePipe().asProcessor().process(null));
}
public void testCIDRMatchFunctionInvalidInput() {
ArrayList<Expression> addresses = new ArrayList<>();
// Invalid source address
EqlIllegalArgumentException e = expectThrows(EqlIllegalArgumentException.class,
() -> new CIDRMatch(EMPTY, l("10.6.48"), addresses).makePipe().asProcessor().process(null));
assertEquals("'10.6.48' is not an IP string literal.", e.getMessage());
// Invalid match ip address
addresses.add(l("10.6.48"));
e = expectThrows(EqlIllegalArgumentException.class,
() -> new CIDRMatch(EMPTY, l("10.6.48.157"), addresses).makePipe().asProcessor().process(null));
assertEquals("'10.6.48' is not an IP string literal.", e.getMessage());
addresses.clear();
// Invalid CIDR
addresses.add(l("10.6.12/12"));
e = expectThrows(EqlIllegalArgumentException.class,
() -> new CIDRMatch(EMPTY, l("10.6.48.157"), addresses).makePipe().asProcessor().process(null));
assertEquals("'10.6.12' is not an IP string literal.", e.getMessage());
addresses.clear();
// Invalid source type
QlIllegalArgumentException eqe = expectThrows(QlIllegalArgumentException.class,
() -> new CIDRMatch(EMPTY, l(12345), addresses).makePipe().asProcessor().process(null));
assertEquals("A string/char is required; received [12345]", eqe.getMessage());
// Invalid cidr type
addresses.add(l(5678));
eqe = expectThrows(QlIllegalArgumentException.class,
() -> new CIDRMatch(EMPTY, l("10.6.48.157"), addresses).makePipe().asProcessor().process(null));
assertEquals("A string/char is required; received [5678]", eqe.getMessage());
}
}

52
x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/expression/function/scalar/string/CIDRUtilsTests.java Normal file
View File

@ -0,0 +1,52 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License;
* you may not use this file except in compliance with the Elastic License.
*/
package org.elasticsearch.xpack.eql.expression.function.scalar.string;
import org.elasticsearch.test.ESTestCase;
public class CIDRUtilsTests extends ESTestCase {
public void testCIDRUtils() {
// Missing or empty param
assertFalse(CIDRUtils.isInRange("10.6.48.157"));
assertFalse(CIDRUtils.isInRange("10.6.48.157", (String)null));
// EQL tests matches
assertTrue(CIDRUtils.isInRange("10.6.48.157", "10.6.48.157/8"));
assertFalse(CIDRUtils.isInRange("10.6.48.157", "192.168.0.0/16"));
assertTrue(CIDRUtils.isInRange("10.6.48.157", "192.168.0.0/16", "10.6.48.157/8"));
assertTrue(CIDRUtils.isInRange("10.6.48.157", "0.0.0.0/0"));
assertFalse(CIDRUtils.isInRange("10.6.48.157", "0.0.0.0"));
// Random things
assertTrue(CIDRUtils.isInRange("192.168.2.1", "192.168.2.1"));
assertFalse(CIDRUtils.isInRange("192.168.2.1", "192.168.2.0/32"));
assertTrue(CIDRUtils.isInRange("192.168.2.5", "192.168.2.0/24"));
assertFalse(CIDRUtils.isInRange("92.168.2.1", "fe80:0:0:0:0:0:c0a8:1/120"));
assertTrue(CIDRUtils.isInRange("192.168.2.5", "192.168.2.0/24"));
assertTrue(CIDRUtils.isInRange("fe80:0:0:0:0:0:c0a8:11", "fe80:0:0:0:0:0:c0a8:1/120"));
assertFalse(CIDRUtils.isInRange("fe80:0:0:0:0:0:c0a8:11", "fe80:0:0:0:0:0:c0a8:1/128"));
assertFalse(CIDRUtils.isInRange("fe80:0:0:0:0:0:c0a8:11", "192.168.2.0/32"));
assertTrue(CIDRUtils.isInRange("2001:db8:3c0d:5b6d:0:0:42:8329", "2001:db8:3c0d:5b6d:0:0:42:8329/58"));
assertTrue(CIDRUtils.isInRange("2001:db8:3c0d:5b40::", "2001:db8:3c0d:5b6d:0:0:42:8329/58"));
assertTrue(CIDRUtils.isInRange("2001:db8:3c0d:5b7f:ffff:ffff:ffff:ffff", "2001:db8:3c0d:5b6d:0:0:42:8329/58"));
assertTrue(CIDRUtils.isInRange("2001:db8:3c0d:5b53:0:0:0:1", "2001:db8:3c0d:5b6d:0:0:42:8329/58"));
assertFalse(CIDRUtils.isInRange("2001:db8:3c0d:5b3f:ffff:ffff:ffff:ffff", "2001:db8:3c0d:5b6d:0:0:42:8329/58"));
assertFalse(CIDRUtils.isInRange("2001:db8:3c0d:5b80::", "2001:db8:3c0d:5b6d:0:0:42:8329/58"));
assertTrue(CIDRUtils.isInRange("2001:db8:3c0d:5b6d:0:0:42:8329", "2001:db8:3c0d:5b6d:0:0:42:8329/128"));
assertFalse(CIDRUtils.isInRange("2001:db8:3c0d:5b6d:0:0:42:8329", "192.168.2.0/32"));
assertTrue(CIDRUtils.isInRange("127.0.0.1", "127.0.0.1/8"));
assertTrue(CIDRUtils.isInRange("127.0.0.1", "::1/64"));
}
}

2
x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/planner/QueryFolderFailTests.java
View File

@ -90,7 +90,7 @@ public class QueryFolderFailTests extends AbstractQueryFolderTestCase {
() -> plan("process where cidrMatch(source_address, 12345)"));
String msg = e.getMessage();
assertEquals("Found 1 problem\n" +
"line 1:15: argument of [cidrMatch(source_address, 12345)] must be [string], found value [12345] type [integer]", msg);
"line 1:15: second argument of [cidrMatch(source_address, 12345)] must be [string], found value [12345] type [integer]", msg);
}
public void testConcatWithInexact() {

42
x-pack/plugin/eql/src/test/resources/queryfolder_tests.txt
View File

@ -78,28 +78,28 @@ functionEqualsTrue
process where cidrMatch(source_address, "10.0.0.0/8") == true
;
{"bool":{"must":[{"term":{"event.category":{"value":"process"
{"term":{"source_address":{"value":"10.0.0.0/8"
{"terms":{"source_address":["10.0.0.0/8"]
;
functionEqualsFalse
process where cidrMatch(source_address, "10.0.0.0/8") == false
;
{"bool":{"must":[{"term":{"event.category":{"value":"process"
{"bool":{"must_not":[{"term":{"source_address":{"value":"10.0.0.0/8"
{"bool":{"must_not":[{"terms":{"source_address":["10.0.0.0/8"]
;
functionNotEqualsTrue
process where cidrMatch(source_address, "10.0.0.0/8") != true
;
{"bool":{"must":[{"term":{"event.category":{"value":"process"
{"bool":{"must_not":[{"term":{"source_address":{"value":"10.0.0.0/8"
{"bool":{"must_not":[{"terms":{"source_address":["10.0.0.0/8"]
;
functionNotEqualsFalse
process where cidrMatch(source_address, "10.0.0.0/8") != false
;
{"bool":{"must":[{"term":{"event.category":{"value":"process"
{"term":{"source_address":{"value":"10.0.0.0/8"
{"terms":{"source_address":["10.0.0.0/8"]
;
twoFunctionsEqualsBooleanLiterals
@ -221,22 +221,44 @@ InternalEqlScriptUtils.concat([InternalQlScriptUtils.docValue(doc,params.v0),par
cidrMatchFunctionOne
process where cidrMatch(source_address, "10.0.0.0/8")
;
"term":{"source_address":{"value":"10.0.0.0/8"
{"bool":{"must":[{"term":{"event.category":{"value":"process"
{"terms":{"source_address":["10.0.0.0/8"]
;
cidrMatchFunctionOneBool
process where cidrMatch(source_address, "10.0.0.0/8") == true
;
{"bool":{"must":[{"term":{"event.category":{"value":"process"
{"terms":{"source_address":["10.0.0.0/8"]
;
cidrMatchFunctionTwo
process where cidrMatch(source_address, "10.0.0.0/8", "192.168.0.0/16")
;
"term":{"source_address":{"value":"10.0.0.0/8"
"term":{"source_address":{"value":"192.168.0.0/16"
{"bool":{"must":[{"term":{"event.category":{"value":"process"
{"terms":{"source_address":["10.0.0.0/8","192.168.0.0/16"]
;
cidrMatchFunctionTwoWithOr
process where cidrMatch(source_address, "10.0.0.0/8") or cidrMatch(source_address, "192.168.0.0/16")
;
{"bool":{"must":[{"term":{"event.category":{"value":"process"
{"bool":{"should":[{"terms":{"source_address":["10.0.0.0/8"],"boost":1.0}},{"terms":{"source_address":["192.168.0.0/16"],"boost":1.0}}
;
cidrMatchFunctionThree
process where cidrMatch(source_address, "10.0.0.0/8", "192.168.0.0/16", "2001:db8::/32")
;
"term":{"source_address":{"value":"10.0.0.0/8"
"term":{"source_address":{"value":"192.168.0.0/16"
"term":{"source_address":{"value":"2001:db8::/32"
{"bool":{"must":[{"term":{"event.category":{"value":"process"
{"terms":{"source_address":["10.0.0.0/8","192.168.0.0/16","2001:db8::/32"]
;
cidrMatchFunctionWrapped
process where string(cidrMatch(source_address, "10.6.48.157/8")) == "true"
;
{"script":{"source":"InternalQlScriptUtils.nullSafeFilter(InternalQlScriptUtils.eq(InternalEqlScriptUtils.string(
InternalEqlScriptUtils.cidrMatch(InternalQlScriptUtils.docValue(doc,params.v0),params.v1)),params.v2))"
"params":{"v0":"source_address","v1":["10.6.48.157/8"],"v2":"true"}
;
matchFunctionOne

11
x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/planner/ExpressionTranslators.java
View File

@ -375,6 +375,14 @@ public final class ExpressionTranslators {
}
public static Query doTranslate(ScalarFunction f, TranslatorHandler handler) {
Query q = doKnownTranslate(f, handler);
if (q != null) {
return q;
}
return handler.wrapFunctionQuery(f, f, new ScriptQuery(f.source(), f.asScript()));
}
public static Query doKnownTranslate(ScalarFunction f, TranslatorHandler handler) {
if (f instanceof StartsWith) {
StartsWith sw = (StartsWith) f;
if (sw.isCaseSensitive() && sw.field() instanceof FieldAttribute && sw.pattern().foldable()) {
@ -384,8 +392,7 @@ public final class ExpressionTranslators {
return new PrefixQuery(f.source(), targetFieldName, pattern);
}
}
return handler.wrapFunctionQuery(f, f, new ScriptQuery(f.source(), f.asScript()));
return null;
}
}