From 87a10806ab7713f403f160097bb93613aac76933 Mon Sep 17 00:00:00 2001 From: Aleksandr Maus Date: Wed, 13 May 2020 22:41:24 -0400 Subject: [PATCH] EQL: Fix cidrMatch function fails to match when used in scripts (#56246) (#56735) EQL: Fix cidrMatch function fails to match when used in scripts (#56246) Addresses https://github.com/elastic/elasticsearch/issues/55709 --- .../resources/test_queries_supported.toml | 38 ++++- .../function/scalar/string/CIDRMatch.java | 104 +++++++++----- .../scalar/string/CIDRMatchFunctionPipe.java | 136 ++++++++++++++++++ .../string/CIDRMatchFunctionProcessor.java | 100 +++++++++++++ .../function/scalar/string/CIDRUtils.java | 96 +++++++++++++ .../whitelist/InternalEqlScriptUtils.java | 5 + .../eql/planner/EqlTranslatorHandler.java | 19 +++ .../xpack/eql/planner/QueryFolder.java | 4 +- .../xpack/eql/planner/QueryTranslator.java | 116 +++++++++++++++ .../xpack/eql/plugin/eql_whitelist.txt | 1 + .../string/CIDRMatchProcessorTests.java | 73 ++++++++++ .../scalar/string/CIDRUtilsTests.java | 52 +++++++ .../eql/planner/QueryFolderFailTests.java | 2 +- .../src/test/resources/queryfolder_tests.txt | 42 ++++-- .../ql/planner/ExpressionTranslators.java | 11 +- 15 files changed, 748 insertions(+), 51 deletions(-) create mode 100644 x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/expression/function/scalar/string/CIDRMatchFunctionPipe.java create mode 100644 x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/expression/function/scalar/string/CIDRMatchFunctionProcessor.java create mode 100644 x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/expression/function/scalar/string/CIDRUtils.java create mode 100644 x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/planner/EqlTranslatorHandler.java create mode 100644 x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/planner/QueryTranslator.java create mode 100644 x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/expression/function/scalar/string/CIDRMatchProcessorTests.java create mode 100644 x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/expression/function/scalar/string/CIDRUtilsTests.java diff --git a/x-pack/plugin/eql/qa/common/src/main/resources/test_queries_supported.toml b/x-pack/plugin/eql/qa/common/src/main/resources/test_queries_supported.toml index 546f7dede33..94e53185f40 100644 --- a/x-pack/plugin/eql/qa/common/src/main/resources/test_queries_supported.toml +++ b/x-pack/plugin/eql/qa/common/src/main/resources/test_queries_supported.toml @@ -14,6 +14,43 @@ query = ''' file where between(file_path, "dev", ".json", true) == "\\TestLogs\\something" ''' +[[queries]] +expected_event_ids = [75304, 75305] +query = ''' +network where cidrMatch(source_address, "10.6.48.157/8") == true +''' + +[[queries]] +expected_event_ids = [75304, 75305] +query = ''' +network where string(cidrMatch(source_address, "10.6.48.157/8")) == "true" +''' + +[[queries]] +expected_event_ids = [75304, 75305] +query = ''' +network where true == cidrMatch(source_address, "10.6.48.157/8") +''' + +[[queries]] +expected_event_ids = [] +query = ''' +network where cidrMatch(source_address, "192.168.0.0/16") == true +''' + +[[queries]] +expected_event_ids = [75304, 75305] +query = ''' +network where cidrMatch(source_address, "192.168.0.0/16", "10.6.48.157/8") == true +''' + +[[queries]] +expected_event_ids = [75304, 75305] +query = ''' +network where cidrMatch(source_address, "0.0.0.0/0") == true +''' + + [[queries]] description = "test string concatenation. update test to avoid case-sensitivity issues" query = ''' @@ -41,7 +78,6 @@ query = 'process where serial_event_id < 5 and concat(process_name, null, null) expected_event_ids = [1, 2, 3, 4] - [[queries]] query = 'process where serial_event_id < 5 and concat(parent_process_name, null) == null' expected_event_ids = [1, 2, 3, 4] diff --git a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/expression/function/scalar/string/CIDRMatch.java b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/expression/function/scalar/string/CIDRMatch.java index b7d5ea88ba3..c192437441d 100644 --- a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/expression/function/scalar/string/CIDRMatch.java +++ b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/expression/function/scalar/string/CIDRMatch.java @@ -6,58 +6,55 @@ package org.elasticsearch.xpack.eql.expression.function.scalar.string; +import org.elasticsearch.common.logging.LoggerMessageFormat; import org.elasticsearch.xpack.ql.expression.Expression; import org.elasticsearch.xpack.ql.expression.Expressions; import org.elasticsearch.xpack.ql.expression.Expressions.ParamOrdinal; -import org.elasticsearch.xpack.ql.expression.function.scalar.BaseSurrogateFunction; +import org.elasticsearch.xpack.ql.expression.FieldAttribute; import org.elasticsearch.xpack.ql.expression.function.scalar.ScalarFunction; -import org.elasticsearch.xpack.ql.expression.predicate.logical.Or; -import org.elasticsearch.xpack.ql.expression.predicate.operator.comparison.Equals; +import org.elasticsearch.xpack.ql.expression.gen.pipeline.Pipe; +import org.elasticsearch.xpack.ql.expression.gen.script.ScriptTemplate; +import org.elasticsearch.xpack.ql.expression.gen.script.Scripts; import org.elasticsearch.xpack.ql.tree.NodeInfo; import org.elasticsearch.xpack.ql.tree.Source; import org.elasticsearch.xpack.ql.type.DataType; import org.elasticsearch.xpack.ql.type.DataTypes; import org.elasticsearch.xpack.ql.util.CollectionUtils; +import java.util.ArrayList; +import java.util.LinkedHashSet; import java.util.List; +import static java.util.Collections.emptyList; import static java.util.Collections.singletonList; +import static org.elasticsearch.xpack.eql.expression.function.scalar.string.CIDRMatchFunctionProcessor.doProcess; import static org.elasticsearch.xpack.ql.expression.TypeResolutions.isFoldable; import static org.elasticsearch.xpack.ql.expression.TypeResolutions.isIPAndExact; import static org.elasticsearch.xpack.ql.expression.TypeResolutions.isStringAndExact; +import static org.elasticsearch.xpack.ql.expression.gen.script.ParamsBuilder.paramsBuilder; /** * EQL specific cidrMatch function * Returns true if the source address matches any of the provided CIDR blocks. * Refer to: https://eql.readthedocs.io/en/latest/query-guide/functions.html#cidrMatch */ -public class CIDRMatch extends BaseSurrogateFunction { +public class CIDRMatch extends ScalarFunction { private final Expression field; private final List addresses; public CIDRMatch(Source source, Expression field, List addresses) { - super(source, CollectionUtils.combine(singletonList(field), addresses)); + super(source, CollectionUtils.combine(singletonList(field), addresses == null ? emptyList() : addresses)); this.field = field; - this.addresses = addresses; + this.addresses = addresses == null ? emptyList() : addresses; } - @Override - protected NodeInfo info() { - return NodeInfo.create(this, CIDRMatch::new, field, addresses); + public Expression field() { + return field; } - @Override - public Expression replaceChildren(List newChildren) { - if (newChildren.size() < 2) { - throw new IllegalArgumentException("expected at least [2] children but received [" + newChildren.size() + "]"); - } - return new CIDRMatch(source(), newChildren.get(0), newChildren.subList(1, newChildren.size())); - } - - @Override - public DataType dataType() { - return DataTypes.BOOLEAN; + public List addresses() { + return addresses; } @Override @@ -71,15 +68,6 @@ public class CIDRMatch extends BaseSurrogateFunction { return resolution; } - for (Expression addr : addresses) { - // Currently we have limited enum for ordinal numbers - // So just using default here for error messaging - resolution = isStringAndExact(addr, sourceText(), ParamOrdinal.DEFAULT); - if (resolution.unresolved()) { - return resolution; - } - } - int index = 1; for (Expression addr : addresses) { @@ -101,14 +89,60 @@ public class CIDRMatch extends BaseSurrogateFunction { } @Override - public ScalarFunction makeSubstitute() { - ScalarFunction func = null; - + protected Pipe makePipe() { + ArrayList arr = new ArrayList<>(addresses.size()); for (Expression address : addresses) { - final Equals eq = new Equals(source(), field, address); - func = (func == null) ? eq : new Or(source(), func, eq); + arr.add(Expressions.pipe(address)); } + return new CIDRMatchFunctionPipe(source(), this, Expressions.pipe(field), arr); + } - return func; + @Override + public boolean foldable() { + return field.foldable() && Expressions.foldable(addresses); + } + + @Override + public Object fold() { + return doProcess(field.fold(), Expressions.fold(addresses)); + } + + @Override + protected NodeInfo info() { + return NodeInfo.create(this, CIDRMatch::new, field, addresses); + } + + @Override + public ScriptTemplate asScript() { + ScriptTemplate leftScript = asScript(field); + + List values = new ArrayList<>(new LinkedHashSet<>(Expressions.fold(addresses))); + return new ScriptTemplate( + formatTemplate(LoggerMessageFormat.format("{eql}.","cidrMatch({}, {})", leftScript.template())), + paramsBuilder() + .script(leftScript.params()) + .variable(values) + .build(), + dataType()); + } + + @Override + public ScriptTemplate scriptWithField(FieldAttribute field) { + return new ScriptTemplate(processScript(Scripts.DOC_VALUE), + paramsBuilder().variable(field.exactAttribute().name()).build(), + dataType()); + } + + @Override + public DataType dataType() { + return DataTypes.BOOLEAN; + } + + @Override + public Expression replaceChildren(List newChildren) { + if (newChildren.size() < 2) { + throw new IllegalArgumentException("expected at least [2] children but received [" + newChildren.size() + "]"); + } + return new CIDRMatch(source(), newChildren.get(0), newChildren.subList(1, newChildren.size())); } } diff --git a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/expression/function/scalar/string/CIDRMatchFunctionPipe.java b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/expression/function/scalar/string/CIDRMatchFunctionPipe.java new file mode 100644 index 00000000000..0936b4f97d6 --- /dev/null +++ b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/expression/function/scalar/string/CIDRMatchFunctionPipe.java @@ -0,0 +1,136 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.eql.expression.function.scalar.string; + +import org.elasticsearch.xpack.ql.execution.search.QlSourceBuilder; +import org.elasticsearch.xpack.ql.expression.Expression; +import org.elasticsearch.xpack.ql.expression.gen.pipeline.Pipe; +import org.elasticsearch.xpack.ql.expression.gen.processor.Processor; +import org.elasticsearch.xpack.ql.tree.NodeInfo; +import org.elasticsearch.xpack.ql.tree.Source; +import org.elasticsearch.xpack.ql.util.CollectionUtils; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Objects; + +public class CIDRMatchFunctionPipe extends Pipe { + + private final Pipe source; + private final List addresses; + + public CIDRMatchFunctionPipe(Source source, Expression expression, Pipe src, List addresses) { + super(source, expression, CollectionUtils.combine(Collections.singletonList(src), addresses)); + this.source = src; + this.addresses = addresses; + } + + @Override + public final Pipe replaceChildren(List newChildren) { + if (newChildren.size() < 2) { + throw new IllegalArgumentException("expected at least [2] children but received [" + newChildren.size() + "]"); + } + return replaceChildren(newChildren.get(0), newChildren.subList(1, newChildren.size())); + } + + @Override + public final Pipe resolveAttributes(AttributeResolver resolver) { + Pipe newSource = source.resolveAttributes(resolver); + boolean same = (newSource == source); + + ArrayList newAddresses = new ArrayList(addresses.size()); + for (Pipe address : addresses) { + Pipe newAddress = address.resolveAttributes(resolver); + newAddresses.add(newAddress); + same = same && (address == newAddress); + } + if (same) { + return this; + } + return replaceChildren(newSource, newAddresses); + } + + @Override + public boolean supportedByAggsOnlyQuery() { + if (source.supportedByAggsOnlyQuery() == false) { + return false; + } + for (Pipe address : addresses) { + if (address.supportedByAggsOnlyQuery() == false) { + return false; + } + } + return true; + } + + @Override + public boolean resolved() { + if (source.resolved() == false) { + return false; + } + for (Pipe address : addresses) { + if (address.resolved() == false) { + return false; + } + } + return true; + } + + protected Pipe replaceChildren(Pipe newSource, List newAddresses) { + return new CIDRMatchFunctionPipe(source(), expression(), newSource, newAddresses); + } + + @Override + public final void collectFields(QlSourceBuilder sourceBuilder) { + source.collectFields(sourceBuilder); + for (Pipe address : addresses) { + address.collectFields(sourceBuilder); + } + } + + @Override + protected NodeInfo info() { + return NodeInfo.create(this, CIDRMatchFunctionPipe::new, expression(), source, addresses); + } + + @Override + public CIDRMatchFunctionProcessor asProcessor() { + ArrayList processors = new ArrayList<>(addresses.size()); + for (Pipe address: addresses) { + processors.add(address.asProcessor()); + } + return new CIDRMatchFunctionProcessor(source.asProcessor(), processors); + } + + public Pipe src() { + return source; + } + + public List addresses() { + return addresses; + } + + @Override + public int hashCode() { + return Objects.hash(source(), addresses()); + } + + @Override + public boolean equals(Object obj) { + if (this == obj) { + return true; + } + + if (obj == null || getClass() != obj.getClass()) { + return false; + } + + CIDRMatchFunctionPipe other = (CIDRMatchFunctionPipe) obj; + return Objects.equals(source(), other.source()) + && Objects.equals(addresses(), other.addresses()); + } +} diff --git a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/expression/function/scalar/string/CIDRMatchFunctionProcessor.java b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/expression/function/scalar/string/CIDRMatchFunctionProcessor.java new file mode 100644 index 00000000000..fe23d607f1f --- /dev/null +++ b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/expression/function/scalar/string/CIDRMatchFunctionProcessor.java @@ -0,0 +1,100 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.eql.expression.function.scalar.string; + +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.xpack.ql.expression.gen.processor.Processor; +import org.elasticsearch.xpack.ql.util.Check; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import java.util.Objects; + +public class CIDRMatchFunctionProcessor implements Processor { + + public static final String NAME = "cdrm"; + + private final Processor source; + private final List addresses; + + public CIDRMatchFunctionProcessor(Processor source, List addresses) { + this.source = source; + this.addresses = addresses; + } + + public CIDRMatchFunctionProcessor(StreamInput in) throws IOException { + source = in.readNamedWriteable(Processor.class); + addresses = in.readNamedWriteableList(Processor.class); + } + + @Override + public final void writeTo(StreamOutput out) throws IOException { + out.writeNamedWriteable(source); + out.writeNamedWriteableList(addresses); + } + + @Override + public String getWriteableName() { + return NAME; + } + + @Override + public Object process(Object input) { + Object src = source.process(input); + ArrayList arr = new ArrayList<>(addresses.size()); + for (Processor address : addresses) { + arr.add(address.process(input)); + } + return doProcess(src, arr); + } + + public static Object doProcess(Object source, List addresses) { + if (source == null) { + return null; + } + + Check.isString(source); + + String[] arr = new String[addresses.size()]; + int i = 0; + for (Object address: addresses) { + Check.isString(address); + arr[i++] = (String)address; + } + return CIDRUtils.isInRange((String)source, arr); + } + + protected Processor source() { + return source; + } + + public List addresses() { + return addresses; + } + + + @Override + public int hashCode() { + return Objects.hash(source(), addresses()); + } + + @Override + public boolean equals(Object obj) { + if (this == obj) { + return true; + } + + if (obj == null || getClass() != obj.getClass()) { + return false; + } + + CIDRMatchFunctionProcessor other = (CIDRMatchFunctionProcessor) obj; + return Objects.equals(source(), other.source()) + && Objects.equals(addresses(), other.addresses()); + } +} diff --git a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/expression/function/scalar/string/CIDRUtils.java b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/expression/function/scalar/string/CIDRUtils.java new file mode 100644 index 00000000000..db5f6fdb368 --- /dev/null +++ b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/expression/function/scalar/string/CIDRUtils.java @@ -0,0 +1,96 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.eql.expression.function.scalar.string; + +import org.apache.lucene.util.FutureArrays; +import org.elasticsearch.common.collect.Tuple; +import org.elasticsearch.common.network.InetAddresses; +import org.elasticsearch.xpack.eql.EqlIllegalArgumentException; + +import java.net.InetAddress; + +public class CIDRUtils { + // Borrowed from Lucene, rfc4291 prefix + static final byte[] IPV4_PREFIX = new byte[]{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, -1, -1}; + + private CIDRUtils() { + } + + public static boolean isInRange(String address, String... cidrAddresses) { + try { + // Check if address is parsable first + byte[] addr = InetAddresses.forString(address).getAddress(); + + if (cidrAddresses == null || cidrAddresses.length == 0) { + return false; + } + + for (String cidrAddress : cidrAddresses) { + if (cidrAddress == null) continue; + byte[] lower, upper; + if (cidrAddress.contains("/")) { + final Tuple range = getLowerUpper(InetAddresses.parseCidr(cidrAddress)); + lower = range.v1(); + upper = range.v2(); + } else { + lower = InetAddresses.forString(cidrAddress).getAddress(); + upper = lower; + } + if (isBetween(addr, lower, upper)) return true; + } + } catch (IllegalArgumentException e) { + throw new EqlIllegalArgumentException(e.getMessage()); + } + + return false; + } + + private static Tuple getLowerUpper(Tuple cidr) { + final InetAddress value = cidr.v1(); + final Integer prefixLength = cidr.v2(); + + if (prefixLength < 0 || prefixLength > 8 * value.getAddress().length) { + throw new IllegalArgumentException("illegal prefixLength '" + prefixLength + + "'. Must be 0-32 for IPv4 ranges, 0-128 for IPv6 ranges"); + } + + byte[] lower = value.getAddress(); + byte[] upper = value.getAddress(); + // Borrowed from Lucene + for (int i = prefixLength; i < 8 * lower.length; i++) { + int m = 1 << (7 - (i & 7)); + lower[i >> 3] &= ~m; + upper[i >> 3] |= m; + } + return new Tuple<>(lower, upper); + } + + private static boolean isBetween(byte[] addr, byte[] lower, byte[] upper) { + // Encode the addresses bytes if lengths do not match + if (addr.length != lower.length) { + addr = encode(addr); + lower = encode(lower); + upper = encode(upper); + } + return FutureArrays.compareUnsigned(lower, 0, lower.length, addr, 0, addr.length) <= 0 && + FutureArrays.compareUnsigned(upper, 0, upper.length, addr, 0, addr.length) >= 0; + } + + // Borrowed from Lucene to make this consistent IP fields matching for the mix of IPv4 and IPv6 values + // Modified signature to avoid extra conversions + private static byte[] encode(byte[] address) { + if (address.length == 4) { + byte[] mapped = new byte[16]; + System.arraycopy(IPV4_PREFIX, 0, mapped, 0, IPV4_PREFIX.length); + System.arraycopy(address, 0, mapped, IPV4_PREFIX.length, address.length); + address = mapped; + } else if (address.length != 16) { + throw new UnsupportedOperationException("Only IPv4 and IPv6 addresses are supported"); + } + return address; + } +} diff --git a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/expression/function/scalar/whitelist/InternalEqlScriptUtils.java b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/expression/function/scalar/whitelist/InternalEqlScriptUtils.java index 8873ba0f924..94473308e1b 100644 --- a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/expression/function/scalar/whitelist/InternalEqlScriptUtils.java +++ b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/expression/function/scalar/whitelist/InternalEqlScriptUtils.java @@ -7,6 +7,7 @@ package org.elasticsearch.xpack.eql.expression.function.scalar.whitelist; import org.elasticsearch.xpack.eql.expression.function.scalar.string.BetweenFunctionProcessor; +import org.elasticsearch.xpack.eql.expression.function.scalar.string.CIDRMatchFunctionProcessor; import org.elasticsearch.xpack.eql.expression.function.scalar.string.ConcatFunctionProcessor; import org.elasticsearch.xpack.eql.expression.function.scalar.string.EndsWithFunctionProcessor; import org.elasticsearch.xpack.eql.expression.function.scalar.string.IndexOfFunctionProcessor; @@ -32,6 +33,10 @@ public class InternalEqlScriptUtils extends InternalQlScriptUtils { return (String) BetweenFunctionProcessor.doProcess(s, left, right, greedy, caseSensitive); } + public static Boolean cidrMatch(String s, List addresses) { + return (Boolean) CIDRMatchFunctionProcessor.doProcess(s, addresses); + } + public static String concat(List values) { return (String) ConcatFunctionProcessor.doProcess(values); } diff --git a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/planner/EqlTranslatorHandler.java b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/planner/EqlTranslatorHandler.java new file mode 100644 index 00000000000..55f581b7a01 --- /dev/null +++ b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/planner/EqlTranslatorHandler.java @@ -0,0 +1,19 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.eql.planner; + +import org.elasticsearch.xpack.ql.expression.Expression; +import org.elasticsearch.xpack.ql.planner.QlTranslatorHandler; +import org.elasticsearch.xpack.ql.querydsl.query.Query; + +public class EqlTranslatorHandler extends QlTranslatorHandler { + + @Override + public Query asQuery(Expression e) { + return QueryTranslator.toQuery(e, this); + } +} diff --git a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/planner/QueryFolder.java b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/planner/QueryFolder.java index 1908601433c..c8766b5f0a9 100644 --- a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/planner/QueryFolder.java +++ b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/planner/QueryFolder.java @@ -69,7 +69,7 @@ class QueryFolder extends RuleExecutor { EsQueryExec exec = (EsQueryExec) plan.child(); QueryContainer qContainer = exec.queryContainer(); - Query query = ExpressionTranslators.toQuery(plan.condition()); + Query query = QueryTranslator.toQuery(plan.condition()); if (qContainer.query() != null || query != null) { query = ExpressionTranslators.and(plan.source(), qContainer.query(), query); @@ -139,4 +139,4 @@ class QueryFolder extends RuleExecutor { @Override protected abstract PhysicalPlan rule(SubPlan plan); } -} \ No newline at end of file +} diff --git a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/planner/QueryTranslator.java b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/planner/QueryTranslator.java new file mode 100644 index 00000000000..178b0734949 --- /dev/null +++ b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/planner/QueryTranslator.java @@ -0,0 +1,116 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.eql.planner; + +import org.elasticsearch.xpack.eql.expression.function.scalar.string.CIDRMatch; +import org.elasticsearch.xpack.ql.QlIllegalArgumentException; +import org.elasticsearch.xpack.ql.expression.Expression; +import org.elasticsearch.xpack.ql.expression.Expressions; +import org.elasticsearch.xpack.ql.expression.FieldAttribute; +import org.elasticsearch.xpack.ql.expression.function.scalar.ScalarFunction; +import org.elasticsearch.xpack.ql.expression.predicate.logical.And; +import org.elasticsearch.xpack.ql.expression.predicate.logical.Or; +import org.elasticsearch.xpack.ql.planner.ExpressionTranslator; +import org.elasticsearch.xpack.ql.planner.ExpressionTranslators; +import org.elasticsearch.xpack.ql.planner.TranslatorHandler; +import org.elasticsearch.xpack.ql.querydsl.query.Query; +import org.elasticsearch.xpack.ql.querydsl.query.ScriptQuery; +import org.elasticsearch.xpack.ql.querydsl.query.TermsQuery; +import org.elasticsearch.xpack.ql.util.CollectionUtils; + +import java.util.Arrays; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Set; + +import static org.elasticsearch.xpack.ql.planner.ExpressionTranslators.and; +import static org.elasticsearch.xpack.ql.planner.ExpressionTranslators.or; + +final class QueryTranslator { + + public static final List> QUERY_TRANSLATORS = Arrays.asList( + new ExpressionTranslators.BinaryComparisons(), + new ExpressionTranslators.Ranges(), + new BinaryLogic(), + new ExpressionTranslators.Nots(), + new ExpressionTranslators.Likes(), + new ExpressionTranslators.InComparisons(), + new ExpressionTranslators.StringQueries(), + new ExpressionTranslators.Matches(), + new ExpressionTranslators.MultiMatches(), + new Scalars() + ); + + public static Query toQuery(Expression e) { + return toQuery(e, new EqlTranslatorHandler()); + } + + public static Query toQuery(Expression e, TranslatorHandler handler) { + Query translation = null; + for (ExpressionTranslator translator : QUERY_TRANSLATORS) { + translation = translator.translate(e, handler); + if (translation != null) { + return translation; + } + } + + throw new QlIllegalArgumentException("Don't know how to translate {} {}", e.nodeName(), e); + } + + public static class BinaryLogic extends ExpressionTranslator { + + @Override + protected Query asQuery(org.elasticsearch.xpack.ql.expression.predicate.logical.BinaryLogic e, TranslatorHandler handler) { + if (e instanceof And) { + return and(e.source(), toQuery(e.left(), handler), toQuery(e.right(), handler)); + } + if (e instanceof Or) { + return or(e.source(), toQuery(e.left(), handler), toQuery(e.right(), handler)); + } + + return null; + } + } + + public static Object valueOf(Expression e) { + if (e.foldable()) { + return e.fold(); + } + throw new QlIllegalArgumentException("Cannot determine value for {}", e); + } + + public static class Scalars extends ExpressionTranslator { + + @Override + protected Query asQuery(ScalarFunction f, TranslatorHandler handler) { + return doTranslate(f, handler); + } + + public static Query doTranslate(ScalarFunction f, TranslatorHandler handler) { + Query q = ExpressionTranslators.Scalars.doKnownTranslate(f, handler); + if (q != null) { + return q; + } + if (f instanceof CIDRMatch) { + CIDRMatch cm = (CIDRMatch) f; + if (cm.field() instanceof FieldAttribute && Expressions.foldable(cm.addresses())) { + String targetFieldName = handler.nameOf(((FieldAttribute) cm.field()).exactAttribute()); + + Set set = new LinkedHashSet<>(CollectionUtils.mapSize(cm.addresses().size())); + + for (Expression e : cm.addresses()) { + set.add(valueOf(e)); + } + + return new TermsQuery(f.source(), targetFieldName, set); + } + } + + return handler.wrapFunctionQuery(f, f, new ScriptQuery(f.source(), f.asScript())); + } + } +} diff --git a/x-pack/plugin/eql/src/main/resources/org/elasticsearch/xpack/eql/plugin/eql_whitelist.txt b/x-pack/plugin/eql/src/main/resources/org/elasticsearch/xpack/eql/plugin/eql_whitelist.txt index c6fe606855b..616a19873d5 100644 --- a/x-pack/plugin/eql/src/main/resources/org/elasticsearch/xpack/eql/plugin/eql_whitelist.txt +++ b/x-pack/plugin/eql/src/main/resources/org/elasticsearch/xpack/eql/plugin/eql_whitelist.txt @@ -66,6 +66,7 @@ class org.elasticsearch.xpack.eql.expression.function.scalar.whitelist.InternalE # ASCII Functions # String between(String, String, String, Boolean, Boolean) + Boolean cidrMatch(String, java.util.List) String concat(java.util.List) Boolean endsWith(String, String) Integer indexOf(String, String, Number) diff --git a/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/expression/function/scalar/string/CIDRMatchProcessorTests.java b/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/expression/function/scalar/string/CIDRMatchProcessorTests.java new file mode 100644 index 00000000000..72842ce14ee --- /dev/null +++ b/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/expression/function/scalar/string/CIDRMatchProcessorTests.java @@ -0,0 +1,73 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.eql.expression.function.scalar.string; + +import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.xpack.eql.EqlIllegalArgumentException; +import org.elasticsearch.xpack.ql.QlIllegalArgumentException; +import org.elasticsearch.xpack.ql.expression.Expression; + +import java.util.ArrayList; + +import static org.elasticsearch.xpack.ql.expression.function.scalar.FunctionTestUtils.l; +import static org.elasticsearch.xpack.ql.tree.Source.EMPTY; + +public class CIDRMatchProcessorTests extends ESTestCase { + + public void testCIDRMatchFunctionValidInput() { + // Expects null if source was null + assertNull(new CIDRMatch(EMPTY, l(null), null).makePipe().asProcessor().process(null)); + + ArrayList addresses = new ArrayList<>(); + assertNull(new CIDRMatch(EMPTY, l(null), addresses).makePipe().asProcessor().process(null)); + + assertFalse((Boolean) new CIDRMatch(EMPTY, l("10.6.48.157"), addresses).makePipe().asProcessor().process(null)); + + addresses.add(l("10.6.48.157/8")); + assertTrue((Boolean) new CIDRMatch(EMPTY, l("10.6.48.157"), addresses).makePipe().asProcessor().process(null)); + } + + public void testCIDRMatchFunctionInvalidInput() { + ArrayList addresses = new ArrayList<>(); + + // Invalid source address + EqlIllegalArgumentException e = expectThrows(EqlIllegalArgumentException.class, + () -> new CIDRMatch(EMPTY, l("10.6.48"), addresses).makePipe().asProcessor().process(null)); + + assertEquals("'10.6.48' is not an IP string literal.", e.getMessage()); + + // Invalid match ip address + addresses.add(l("10.6.48")); + e = expectThrows(EqlIllegalArgumentException.class, + () -> new CIDRMatch(EMPTY, l("10.6.48.157"), addresses).makePipe().asProcessor().process(null)); + + assertEquals("'10.6.48' is not an IP string literal.", e.getMessage()); + addresses.clear(); + + // Invalid CIDR + addresses.add(l("10.6.12/12")); + e = expectThrows(EqlIllegalArgumentException.class, + () -> new CIDRMatch(EMPTY, l("10.6.48.157"), addresses).makePipe().asProcessor().process(null)); + + assertEquals("'10.6.12' is not an IP string literal.", e.getMessage()); + addresses.clear(); + + // Invalid source type + QlIllegalArgumentException eqe = expectThrows(QlIllegalArgumentException.class, + () -> new CIDRMatch(EMPTY, l(12345), addresses).makePipe().asProcessor().process(null)); + + assertEquals("A string/char is required; received [12345]", eqe.getMessage()); + + + // Invalid cidr type + addresses.add(l(5678)); + eqe = expectThrows(QlIllegalArgumentException.class, + () -> new CIDRMatch(EMPTY, l("10.6.48.157"), addresses).makePipe().asProcessor().process(null)); + + assertEquals("A string/char is required; received [5678]", eqe.getMessage()); + } +} diff --git a/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/expression/function/scalar/string/CIDRUtilsTests.java b/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/expression/function/scalar/string/CIDRUtilsTests.java new file mode 100644 index 00000000000..140ad8ebf38 --- /dev/null +++ b/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/expression/function/scalar/string/CIDRUtilsTests.java @@ -0,0 +1,52 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.eql.expression.function.scalar.string; + +import org.elasticsearch.test.ESTestCase; + +public class CIDRUtilsTests extends ESTestCase { + + public void testCIDRUtils() { + // Missing or empty param + assertFalse(CIDRUtils.isInRange("10.6.48.157")); + assertFalse(CIDRUtils.isInRange("10.6.48.157", (String)null)); + + // EQL tests matches + assertTrue(CIDRUtils.isInRange("10.6.48.157", "10.6.48.157/8")); + assertFalse(CIDRUtils.isInRange("10.6.48.157", "192.168.0.0/16")); + assertTrue(CIDRUtils.isInRange("10.6.48.157", "192.168.0.0/16", "10.6.48.157/8")); + assertTrue(CIDRUtils.isInRange("10.6.48.157", "0.0.0.0/0")); + assertFalse(CIDRUtils.isInRange("10.6.48.157", "0.0.0.0")); + + // Random things + assertTrue(CIDRUtils.isInRange("192.168.2.1", "192.168.2.1")); + assertFalse(CIDRUtils.isInRange("192.168.2.1", "192.168.2.0/32")); + assertTrue(CIDRUtils.isInRange("192.168.2.5", "192.168.2.0/24")); + assertFalse(CIDRUtils.isInRange("92.168.2.1", "fe80:0:0:0:0:0:c0a8:1/120")); + assertTrue(CIDRUtils.isInRange("192.168.2.5", "192.168.2.0/24")); + + assertTrue(CIDRUtils.isInRange("fe80:0:0:0:0:0:c0a8:11", "fe80:0:0:0:0:0:c0a8:1/120")); + assertFalse(CIDRUtils.isInRange("fe80:0:0:0:0:0:c0a8:11", "fe80:0:0:0:0:0:c0a8:1/128")); + assertFalse(CIDRUtils.isInRange("fe80:0:0:0:0:0:c0a8:11", "192.168.2.0/32")); + + + assertTrue(CIDRUtils.isInRange("2001:db8:3c0d:5b6d:0:0:42:8329", "2001:db8:3c0d:5b6d:0:0:42:8329/58")); + assertTrue(CIDRUtils.isInRange("2001:db8:3c0d:5b40::", "2001:db8:3c0d:5b6d:0:0:42:8329/58")); + assertTrue(CIDRUtils.isInRange("2001:db8:3c0d:5b7f:ffff:ffff:ffff:ffff", "2001:db8:3c0d:5b6d:0:0:42:8329/58")); + assertTrue(CIDRUtils.isInRange("2001:db8:3c0d:5b53:0:0:0:1", "2001:db8:3c0d:5b6d:0:0:42:8329/58")); + assertFalse(CIDRUtils.isInRange("2001:db8:3c0d:5b3f:ffff:ffff:ffff:ffff", "2001:db8:3c0d:5b6d:0:0:42:8329/58")); + assertFalse(CIDRUtils.isInRange("2001:db8:3c0d:5b80::", "2001:db8:3c0d:5b6d:0:0:42:8329/58")); + + assertTrue(CIDRUtils.isInRange("2001:db8:3c0d:5b6d:0:0:42:8329", "2001:db8:3c0d:5b6d:0:0:42:8329/128")); + + assertFalse(CIDRUtils.isInRange("2001:db8:3c0d:5b6d:0:0:42:8329", "192.168.2.0/32")); + + assertTrue(CIDRUtils.isInRange("127.0.0.1", "127.0.0.1/8")); + + assertTrue(CIDRUtils.isInRange("127.0.0.1", "::1/64")); + } +} diff --git a/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/planner/QueryFolderFailTests.java b/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/planner/QueryFolderFailTests.java index 34592f2e38a..c55f3e30c8a 100644 --- a/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/planner/QueryFolderFailTests.java +++ b/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/planner/QueryFolderFailTests.java @@ -90,7 +90,7 @@ public class QueryFolderFailTests extends AbstractQueryFolderTestCase { () -> plan("process where cidrMatch(source_address, 12345)")); String msg = e.getMessage(); assertEquals("Found 1 problem\n" + - "line 1:15: argument of [cidrMatch(source_address, 12345)] must be [string], found value [12345] type [integer]", msg); + "line 1:15: second argument of [cidrMatch(source_address, 12345)] must be [string], found value [12345] type [integer]", msg); } public void testConcatWithInexact() { diff --git a/x-pack/plugin/eql/src/test/resources/queryfolder_tests.txt b/x-pack/plugin/eql/src/test/resources/queryfolder_tests.txt index f8748a85eff..eb45d665088 100644 --- a/x-pack/plugin/eql/src/test/resources/queryfolder_tests.txt +++ b/x-pack/plugin/eql/src/test/resources/queryfolder_tests.txt @@ -78,28 +78,28 @@ functionEqualsTrue process where cidrMatch(source_address, "10.0.0.0/8") == true ; {"bool":{"must":[{"term":{"event.category":{"value":"process" -{"term":{"source_address":{"value":"10.0.0.0/8" +{"terms":{"source_address":["10.0.0.0/8"] ; functionEqualsFalse process where cidrMatch(source_address, "10.0.0.0/8") == false ; {"bool":{"must":[{"term":{"event.category":{"value":"process" -{"bool":{"must_not":[{"term":{"source_address":{"value":"10.0.0.0/8" +{"bool":{"must_not":[{"terms":{"source_address":["10.0.0.0/8"] ; functionNotEqualsTrue process where cidrMatch(source_address, "10.0.0.0/8") != true ; {"bool":{"must":[{"term":{"event.category":{"value":"process" -{"bool":{"must_not":[{"term":{"source_address":{"value":"10.0.0.0/8" +{"bool":{"must_not":[{"terms":{"source_address":["10.0.0.0/8"] ; functionNotEqualsFalse process where cidrMatch(source_address, "10.0.0.0/8") != false ; {"bool":{"must":[{"term":{"event.category":{"value":"process" -{"term":{"source_address":{"value":"10.0.0.0/8" +{"terms":{"source_address":["10.0.0.0/8"] ; twoFunctionsEqualsBooleanLiterals @@ -221,22 +221,44 @@ InternalEqlScriptUtils.concat([InternalQlScriptUtils.docValue(doc,params.v0),par cidrMatchFunctionOne process where cidrMatch(source_address, "10.0.0.0/8") ; -"term":{"source_address":{"value":"10.0.0.0/8" +{"bool":{"must":[{"term":{"event.category":{"value":"process" +{"terms":{"source_address":["10.0.0.0/8"] +; + +cidrMatchFunctionOneBool +process where cidrMatch(source_address, "10.0.0.0/8") == true +; +{"bool":{"must":[{"term":{"event.category":{"value":"process" +{"terms":{"source_address":["10.0.0.0/8"] ; cidrMatchFunctionTwo process where cidrMatch(source_address, "10.0.0.0/8", "192.168.0.0/16") ; -"term":{"source_address":{"value":"10.0.0.0/8" -"term":{"source_address":{"value":"192.168.0.0/16" +{"bool":{"must":[{"term":{"event.category":{"value":"process" +{"terms":{"source_address":["10.0.0.0/8","192.168.0.0/16"] +; + +cidrMatchFunctionTwoWithOr +process where cidrMatch(source_address, "10.0.0.0/8") or cidrMatch(source_address, "192.168.0.0/16") +; +{"bool":{"must":[{"term":{"event.category":{"value":"process" +{"bool":{"should":[{"terms":{"source_address":["10.0.0.0/8"],"boost":1.0}},{"terms":{"source_address":["192.168.0.0/16"],"boost":1.0}} ; cidrMatchFunctionThree process where cidrMatch(source_address, "10.0.0.0/8", "192.168.0.0/16", "2001:db8::/32") ; -"term":{"source_address":{"value":"10.0.0.0/8" -"term":{"source_address":{"value":"192.168.0.0/16" -"term":{"source_address":{"value":"2001:db8::/32" +{"bool":{"must":[{"term":{"event.category":{"value":"process" +{"terms":{"source_address":["10.0.0.0/8","192.168.0.0/16","2001:db8::/32"] +; + +cidrMatchFunctionWrapped +process where string(cidrMatch(source_address, "10.6.48.157/8")) == "true" +; +{"script":{"source":"InternalQlScriptUtils.nullSafeFilter(InternalQlScriptUtils.eq(InternalEqlScriptUtils.string( +InternalEqlScriptUtils.cidrMatch(InternalQlScriptUtils.docValue(doc,params.v0),params.v1)),params.v2))" +"params":{"v0":"source_address","v1":["10.6.48.157/8"],"v2":"true"} ; matchFunctionOne diff --git a/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/planner/ExpressionTranslators.java b/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/planner/ExpressionTranslators.java index fad8020f680..5d4450ea278 100644 --- a/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/planner/ExpressionTranslators.java +++ b/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/planner/ExpressionTranslators.java @@ -375,6 +375,14 @@ public final class ExpressionTranslators { } public static Query doTranslate(ScalarFunction f, TranslatorHandler handler) { + Query q = doKnownTranslate(f, handler); + if (q != null) { + return q; + } + return handler.wrapFunctionQuery(f, f, new ScriptQuery(f.source(), f.asScript())); + } + + public static Query doKnownTranslate(ScalarFunction f, TranslatorHandler handler) { if (f instanceof StartsWith) { StartsWith sw = (StartsWith) f; if (sw.isCaseSensitive() && sw.field() instanceof FieldAttribute && sw.pattern().foldable()) { @@ -384,8 +392,7 @@ public final class ExpressionTranslators { return new PrefixQuery(f.source(), targetFieldName, pattern); } } - - return handler.wrapFunctionQuery(f, f, new ScriptQuery(f.source(), f.asScript())); + return null; } }