mirror of
https://github.com/honeymoose/OpenSearch.git
synced 2025-02-23 13:26:02 +00:00
EQL: Use In from QL (#53244)
* EQL: Use In from QL * EQL: Add more In tests * EQL: Test In duplicates * EQL: Add test for In mixed types * EQL: Copy In translation to QL * SQL: Use InComparisons from QL * EQL: Remove boost checks from QueryFolderOkTests * QL: Add TranslatorHandler.convert
This commit is contained in:
Ross Wolf
parent
ce7539ce6c
commit
d11e977b1f
@ -35,6 +35,7 @@ import org.elasticsearch.xpack.ql.expression.predicate.operator.arithmetic.Sub;
|
||||
import org.elasticsearch.xpack.ql.expression.predicate.operator.comparison.Equals;
|
||||
import org.elasticsearch.xpack.ql.expression.predicate.operator.comparison.GreaterThan;
|
||||
import org.elasticsearch.xpack.ql.expression.predicate.operator.comparison.GreaterThanOrEqual;
|
||||
import org.elasticsearch.xpack.ql.expression.predicate.operator.comparison.In;
|
||||
import org.elasticsearch.xpack.ql.expression.predicate.operator.comparison.LessThan;
|
||||
import org.elasticsearch.xpack.ql.expression.predicate.operator.comparison.LessThanOrEqual;
|
||||
import org.elasticsearch.xpack.ql.expression.predicate.operator.comparison.NotEquals;
|
||||
@ -137,14 +138,7 @@ public class ExpressionBuilder extends IdentifierBuilder {
|
||||
}
|
||||
|
||||
List<Expression> container = expressions(predicate.expression());
|
||||
|
||||
// TODO: Add IN to QL and use that directly
|
||||
Expression checkInSet = null;
|
||||
|
||||
for (Expression inner : container) {
|
||||
Expression termCheck = new Equals(source, expr, inner);
|
||||
checkInSet = checkInSet == null ? termCheck : new Or(source, checkInSet, termCheck);
|
||||
}
|
||||
Expression checkInSet = new In(source, expr, container);
|
||||
|
||||
return predicate.NOT() != null ? new Not(source, checkInSet) : checkInSet;
|
||||
}
|
||||
|
||||
@ -298,6 +298,12 @@ public class VerifierTests extends ESTestCase {
|
||||
error(idxr, "foo where ip_range_field == ''"));
|
||||
}
|
||||
|
||||
public void testMixedSet() {
|
||||
final IndexResolution idxr = loadIndexResolution("mapping-numeric.json");
|
||||
assertEquals("1:11: 2nd argument of [long_field in (1, 'string')] must be [long], found value ['string'] type [keyword]",
|
||||
error(idxr, "foo where long_field in (1, 'string')"));
|
||||
}
|
||||
|
||||
public void testObject() {
|
||||
final IndexResolution idxr = loadIndexResolution("mapping-object.json");
|
||||
accept(idxr, "foo where endgame.pid == 0");
|
||||
|
||||
@ -13,16 +13,19 @@ import org.elasticsearch.xpack.ql.expression.Literal;
|
||||
import org.elasticsearch.xpack.ql.expression.UnresolvedAttribute;
|
||||
import org.elasticsearch.xpack.ql.expression.function.UnresolvedFunction;
|
||||
import org.elasticsearch.xpack.ql.expression.predicate.logical.And;
|
||||
import org.elasticsearch.xpack.ql.expression.predicate.logical.Not;
|
||||
import org.elasticsearch.xpack.ql.expression.predicate.logical.Or;
|
||||
import org.elasticsearch.xpack.ql.expression.predicate.operator.arithmetic.Neg;
|
||||
import org.elasticsearch.xpack.ql.expression.predicate.operator.comparison.Equals;
|
||||
import org.elasticsearch.xpack.ql.expression.predicate.operator.comparison.GreaterThan;
|
||||
import org.elasticsearch.xpack.ql.expression.predicate.operator.comparison.GreaterThanOrEqual;
|
||||
import org.elasticsearch.xpack.ql.expression.predicate.operator.comparison.In;
|
||||
import org.elasticsearch.xpack.ql.expression.predicate.operator.comparison.LessThan;
|
||||
import org.elasticsearch.xpack.ql.expression.predicate.operator.comparison.LessThanOrEqual;
|
||||
import org.elasticsearch.xpack.ql.expression.predicate.operator.comparison.NotEquals;
|
||||
import org.elasticsearch.xpack.ql.type.DataTypes;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.Arrays;
|
||||
import java.util.List;
|
||||
|
||||
@ -40,6 +43,14 @@ public class ExpressionTests extends ESTestCase {
|
||||
return parser.createExpression(source);
|
||||
}
|
||||
|
||||
private List<Expression> exprs(String... sources) {
|
||||
List<Expression> results = new ArrayList<Expression>(sources.length);
|
||||
for (String s : sources) {
|
||||
results.add(expr(s));
|
||||
}
|
||||
return results;
|
||||
}
|
||||
|
||||
|
||||
public void testStrings() throws Exception {
|
||||
assertEquals("hello\"world", unquoteString("'hello\"world'"));
|
||||
@ -156,19 +167,49 @@ public class ExpressionTests extends ESTestCase {
|
||||
}
|
||||
|
||||
public void testInSet() {
|
||||
assertEquals(
|
||||
expr("name in (1)"),
|
||||
new In(null, expr("name"), exprs("1"))
|
||||
);
|
||||
|
||||
assertEquals(
|
||||
expr("name in (2, 1)"),
|
||||
new In(null, expr("name"), exprs("2", "1"))
|
||||
);
|
||||
assertEquals(
|
||||
expr("name in ('net.exe')"),
|
||||
expr("name == 'net.exe'")
|
||||
new In(null, expr("name"), exprs("'net.exe'"))
|
||||
);
|
||||
|
||||
assertEquals(
|
||||
expr("name in ('net.exe', 'whoami.exe', 'hostname.exe')"),
|
||||
expr("name == 'net.exe' or name == 'whoami.exe' or name == 'hostname.exe'")
|
||||
new In(null, expr("name"), exprs("'net.exe'", "'whoami.exe'", "'hostname.exe'"))
|
||||
);
|
||||
}
|
||||
|
||||
public void testInSetDuplicates() {
|
||||
assertEquals(
|
||||
expr("name in (1, 1)"),
|
||||
new In(null, expr("name"), exprs("1", "1"))
|
||||
);
|
||||
|
||||
assertEquals(
|
||||
expr("name not in ('net.exe', 'whoami.exe', 'hostname.exe')"),
|
||||
expr("not (name == 'net.exe' or name == 'whoami.exe' or name == 'hostname.exe')")
|
||||
expr("name in ('net.exe', 'net.exe')"),
|
||||
new In(null, expr("name"), exprs("'net.exe'", "'net.exe'"))
|
||||
);
|
||||
}
|
||||
|
||||
public void testNotInSet() {
|
||||
assertEquals(
|
||||
expr("name not in ('net.exe', 'whoami.exe', 'hostname.exe')"),
|
||||
new Not(null, new In(null,
|
||||
expr("name"),
|
||||
exprs("'net.exe'", "'whoami.exe'", "'hostname.exe'")))
|
||||
);
|
||||
}
|
||||
|
||||
public void testInEmptySet() {
|
||||
expectThrows(ParsingException.class, "Expected syntax error",
|
||||
() -> expr("name in ()"));
|
||||
}
|
||||
}
|
||||
|
||||
@ -6,6 +6,7 @@
|
||||
|
||||
package org.elasticsearch.xpack.eql.planner;
|
||||
|
||||
import org.elasticsearch.xpack.eql.analysis.VerificationException;
|
||||
import org.elasticsearch.xpack.ql.QlIllegalArgumentException;
|
||||
|
||||
public class QueryFolderFailTests extends AbstractQueryFolderTestCase {
|
||||
@ -17,9 +18,10 @@ public class QueryFolderFailTests extends AbstractQueryFolderTestCase {
|
||||
}
|
||||
|
||||
public void testPropertyEquationInClauseFilterUnsupported() {
|
||||
QlIllegalArgumentException e = expectThrows(QlIllegalArgumentException.class,
|
||||
VerificationException e = expectThrows(VerificationException.class,
|
||||
() -> plan("process where opcode in (1,3) and process_name in (parent_process_name, \"SYSTEM\")"));
|
||||
String msg = e.getMessage();
|
||||
assertEquals("Line 1:52: Comparisons against variables are not (currently) supported; offender [parent_process_name] in [==]", msg);
|
||||
assertEquals("Found 1 problem\nline 1:35: Comparisons against variables are not (currently) supported; " +
|
||||
"offender [parent_process_name] in [process_name in (parent_process_name, \"SYSTEM\")]", msg);
|
||||
}
|
||||
}
|
||||
|
||||
@ -20,6 +20,7 @@ import static org.elasticsearch.xpack.ql.type.DataTypes.KEYWORD;
|
||||
import static org.hamcrest.Matchers.containsString;
|
||||
|
||||
public class QueryFolderOkTests extends AbstractQueryFolderTestCase {
|
||||
|
||||
private final String name;
|
||||
private final String query;
|
||||
private final Object expect;
|
||||
|
||||
@ -52,18 +52,13 @@ process where not (exit_code > -1)
|
||||
|
||||
inFilter
|
||||
process where process_name in ("python.exe", "SMSS.exe", "explorer.exe")
|
||||
"term":{"process_name":{"value":"python.exe"
|
||||
"term":{"process_name":{"value":"SMSS.exe"
|
||||
"term":{"process_name":{"value":"explorer.exe"
|
||||
"terms":{"process_name":["python.exe","SMSS.exe","explorer.exe"],
|
||||
|
||||
|
||||
equalsAndInFilter
|
||||
process where process_path == "*\\red_ttp\\wininit.*" and opcode in (0,1,2,3)
|
||||
"wildcard":{"process_path":{"wildcard":"*\\\\red_ttp\\\\wininit.*"
|
||||
"term":{"opcode":{"value":0
|
||||
"term":{"opcode":{"value":1
|
||||
"term":{"opcode":{"value":2
|
||||
"term":{"opcode":{"value":3
|
||||
{"terms":{"opcode":[0,1,2,3]
|
||||
|
||||
|
||||
substringFunction
|
||||
|
||||
@ -32,4 +32,4 @@ public abstract class ExpressionTranslator<E extends Expression> {
|
||||
}
|
||||
return query;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@ -23,6 +23,7 @@ import org.elasticsearch.xpack.ql.expression.predicate.operator.comparison.Binar
|
||||
import org.elasticsearch.xpack.ql.expression.predicate.operator.comparison.Equals;
|
||||
import org.elasticsearch.xpack.ql.expression.predicate.operator.comparison.GreaterThan;
|
||||
import org.elasticsearch.xpack.ql.expression.predicate.operator.comparison.GreaterThanOrEqual;
|
||||
import org.elasticsearch.xpack.ql.expression.predicate.operator.comparison.In;
|
||||
import org.elasticsearch.xpack.ql.expression.predicate.operator.comparison.LessThan;
|
||||
import org.elasticsearch.xpack.ql.expression.predicate.operator.comparison.LessThanOrEqual;
|
||||
import org.elasticsearch.xpack.ql.expression.predicate.operator.comparison.NotEquals;
|
||||
@ -41,29 +42,37 @@ import org.elasticsearch.xpack.ql.querydsl.query.RangeQuery;
|
||||
import org.elasticsearch.xpack.ql.querydsl.query.RegexQuery;
|
||||
import org.elasticsearch.xpack.ql.querydsl.query.ScriptQuery;
|
||||
import org.elasticsearch.xpack.ql.querydsl.query.TermQuery;
|
||||
import org.elasticsearch.xpack.ql.querydsl.query.TermsQuery;
|
||||
import org.elasticsearch.xpack.ql.querydsl.query.WildcardQuery;
|
||||
import org.elasticsearch.xpack.ql.tree.Source;
|
||||
import org.elasticsearch.xpack.ql.type.DataType;
|
||||
import org.elasticsearch.xpack.ql.type.DataTypes;
|
||||
import org.elasticsearch.xpack.ql.util.Check;
|
||||
import org.elasticsearch.xpack.ql.util.CollectionUtils;
|
||||
import org.elasticsearch.xpack.ql.util.Holder;
|
||||
|
||||
import java.time.OffsetTime;
|
||||
import java.time.ZonedDateTime;
|
||||
import java.time.temporal.TemporalAccessor;
|
||||
import java.util.Arrays;
|
||||
import java.util.LinkedHashSet;
|
||||
import java.util.List;
|
||||
import java.util.Set;
|
||||
|
||||
|
||||
public final class ExpressionTranslators {
|
||||
|
||||
public static final String DATE_FORMAT = "strict_date_time";
|
||||
public static final String TIME_FORMAT = "strict_hour_minute_second_millis";
|
||||
|
||||
|
||||
|
||||
public static final List<ExpressionTranslator<?>> QUERY_TRANSLATORS = Arrays.asList(
|
||||
new BinaryComparisons(),
|
||||
new Ranges(),
|
||||
new BinaryLogic(),
|
||||
new Nots(),
|
||||
new Likes(),
|
||||
new InComparisons(),
|
||||
new StringQueries(),
|
||||
new Matches(),
|
||||
new MultiMatches(),
|
||||
@ -86,6 +95,13 @@ public final class ExpressionTranslators {
|
||||
throw new QlIllegalArgumentException("Don't know how to translate {} {}", e.nodeName(), e);
|
||||
}
|
||||
|
||||
public static Object valueOf(Expression e) {
|
||||
if (e.foldable()) {
|
||||
return e.fold();
|
||||
}
|
||||
throw new QlIllegalArgumentException("Cannot determine value for {}", e);
|
||||
}
|
||||
|
||||
// TODO: see whether escaping is needed
|
||||
@SuppressWarnings("rawtypes")
|
||||
public static class Likes extends ExpressionTranslator<RegexMatch> {
|
||||
@ -196,7 +212,7 @@ public final class ExpressionTranslators {
|
||||
protected Query asQuery(BinaryComparison bc, TranslatorHandler handler) {
|
||||
return doTranslate(bc, handler);
|
||||
}
|
||||
|
||||
|
||||
public static void checkBinaryComparison(BinaryComparison bc) {
|
||||
Check.isTrue(bc.right().foldable(),
|
||||
"Line {}:{}: Comparisons against variables are not (currently) supported; offender [{}] in [{}]",
|
||||
@ -315,6 +331,36 @@ public final class ExpressionTranslators {
|
||||
}
|
||||
}
|
||||
|
||||
public static class InComparisons extends ExpressionTranslator<In> {
|
||||
|
||||
protected Query asQuery(In in, TranslatorHandler handler) {
|
||||
return doTranslate(in, handler);
|
||||
}
|
||||
|
||||
public static Query doTranslate(In in, TranslatorHandler handler) {
|
||||
Query q;
|
||||
if (in.value() instanceof FieldAttribute) {
|
||||
// equality should always be against an exact match (which is important for strings)
|
||||
FieldAttribute fa = (FieldAttribute) in.value();
|
||||
List<Expression> list = in.list();
|
||||
|
||||
// TODO: this needs to be handled inside the optimizer
|
||||
list.removeIf(e -> DataTypes.isNull(e.dataType()));
|
||||
DataType dt = list.get(0).dataType();
|
||||
Set<Object> set = new LinkedHashSet<>(CollectionUtils.mapSize(list.size()));
|
||||
|
||||
for (Expression e : list) {
|
||||
set.add(handler.convert(valueOf(e), dt));
|
||||
}
|
||||
|
||||
q = new TermsQuery(in.source(), fa.exactAttribute().name(), set);
|
||||
} else {
|
||||
q = new ScriptQuery(in.source(), in.asScript());
|
||||
}
|
||||
return handler.wrapFunctionQuery(in, in.value(), q);
|
||||
}
|
||||
}
|
||||
|
||||
public static class Scalars extends ExpressionTranslator<ScalarFunction> {
|
||||
|
||||
@Override
|
||||
@ -327,13 +373,6 @@ public final class ExpressionTranslators {
|
||||
}
|
||||
}
|
||||
|
||||
public static Object valueOf(Expression e) {
|
||||
if (e.foldable()) {
|
||||
return e.fold();
|
||||
}
|
||||
throw new QlIllegalArgumentException("Cannot determine value for {}", e);
|
||||
}
|
||||
|
||||
public static Query or(Source source, Query left, Query right) {
|
||||
return boolQuery(source, left, right, false);
|
||||
}
|
||||
|
||||
@ -12,6 +12,8 @@ import org.elasticsearch.xpack.ql.expression.NamedExpression;
|
||||
import org.elasticsearch.xpack.ql.expression.function.scalar.ScalarFunction;
|
||||
import org.elasticsearch.xpack.ql.querydsl.query.Query;
|
||||
import org.elasticsearch.xpack.ql.querydsl.query.ScriptQuery;
|
||||
import org.elasticsearch.xpack.ql.type.DataType;
|
||||
import org.elasticsearch.xpack.ql.type.DataTypeConverter;
|
||||
|
||||
public class QlTranslatorHandler implements TranslatorHandler {
|
||||
|
||||
@ -41,4 +43,9 @@ public class QlTranslatorHandler implements TranslatorHandler {
|
||||
public String dateFormat(Expression e) {
|
||||
return null;
|
||||
}
|
||||
|
||||
@Override
|
||||
public Object convert(Object value, DataType dataType) {
|
||||
return DataTypeConverter.convert(value, dataType);
|
||||
}
|
||||
}
|
||||
|
||||
@ -9,10 +9,11 @@ package org.elasticsearch.xpack.ql.planner;
|
||||
import org.elasticsearch.xpack.ql.expression.Expression;
|
||||
import org.elasticsearch.xpack.ql.expression.function.scalar.ScalarFunction;
|
||||
import org.elasticsearch.xpack.ql.querydsl.query.Query;
|
||||
import org.elasticsearch.xpack.ql.type.DataType;
|
||||
|
||||
/**
|
||||
* Parameterized handler used during query translation.
|
||||
*
|
||||
*
|
||||
* Provides contextual utilities for an individual query to be performed.
|
||||
*/
|
||||
public interface TranslatorHandler {
|
||||
@ -24,4 +25,6 @@ public interface TranslatorHandler {
|
||||
String nameOf(Expression e);
|
||||
|
||||
String dateFormat(Expression e);
|
||||
|
||||
Object convert(Object value, DataType dataType);
|
||||
}
|
||||
|
||||
@ -37,11 +37,8 @@ import org.elasticsearch.xpack.ql.querydsl.query.GeoDistanceQuery;
|
||||
import org.elasticsearch.xpack.ql.querydsl.query.NotQuery;
|
||||
import org.elasticsearch.xpack.ql.querydsl.query.Query;
|
||||
import org.elasticsearch.xpack.ql.querydsl.query.ScriptQuery;
|
||||
import org.elasticsearch.xpack.ql.querydsl.query.TermsQuery;
|
||||
import org.elasticsearch.xpack.ql.tree.Source;
|
||||
import org.elasticsearch.xpack.ql.type.DataType;
|
||||
import org.elasticsearch.xpack.ql.type.DataTypes;
|
||||
import org.elasticsearch.xpack.ql.util.CollectionUtils;
|
||||
import org.elasticsearch.xpack.ql.util.ReflectionUtils;
|
||||
import org.elasticsearch.xpack.sql.SqlIllegalArgumentException;
|
||||
import org.elasticsearch.xpack.sql.expression.function.aggregate.Avg;
|
||||
@ -84,9 +81,7 @@ import org.elasticsearch.xpack.sql.util.Check;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.Arrays;
|
||||
import java.util.LinkedHashSet;
|
||||
import java.util.List;
|
||||
import java.util.Set;
|
||||
|
||||
import static java.util.Collections.singletonList;
|
||||
import static org.elasticsearch.xpack.ql.expression.Expressions.id;
|
||||
@ -461,23 +456,7 @@ final class QueryTranslator {
|
||||
aggFilter = new AggFilter(id(in.value()), in.asScript());
|
||||
}
|
||||
else {
|
||||
Query q = null;
|
||||
if (in.value() instanceof FieldAttribute) {
|
||||
// equality should always be against an exact match (which is important for strings)
|
||||
FieldAttribute fa = (FieldAttribute) in.value();
|
||||
List<Expression> list = in.list();
|
||||
// TODO: this needs to be handled inside the optimizer
|
||||
list.removeIf(e -> DataTypes.isNull(e.dataType()));
|
||||
DataType dt = list.get(0).dataType();
|
||||
Set<Object> set = new LinkedHashSet<>(CollectionUtils.mapSize(list.size()));
|
||||
for (Expression e : list) {
|
||||
set.add(SqlDataTypeConverter.convert(e.fold(), dt));
|
||||
}
|
||||
q = new TermsQuery(in.source(), fa.exactAttribute().name(), set);
|
||||
} else {
|
||||
q = new ScriptQuery(in.source(), in.asScript());
|
||||
}
|
||||
query = handler.wrapFunctionQuery(in, in.value(), q);
|
||||
query = org.elasticsearch.xpack.ql.planner.ExpressionTranslators.InComparisons.doTranslate(in, handler);
|
||||
}
|
||||
return new QueryTranslation(query, aggFilter);
|
||||
}
|
||||
@ -698,7 +677,7 @@ final class QueryTranslator {
|
||||
|
||||
protected abstract LeafAgg toAgg(String id, C f);
|
||||
}
|
||||
|
||||
|
||||
private static List<Double> foldAndConvertToDoubles(List<Expression> list) {
|
||||
List<Double> values = new ArrayList<>(list.size());
|
||||
for (Expression e : list) {
|
||||
@ -706,4 +685,4 @@ final class QueryTranslator {
|
||||
}
|
||||
return values;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@ -14,8 +14,10 @@ import org.elasticsearch.xpack.ql.planner.TranslatorHandler;
|
||||
import org.elasticsearch.xpack.ql.querydsl.query.GeoDistanceQuery;
|
||||
import org.elasticsearch.xpack.ql.querydsl.query.Query;
|
||||
import org.elasticsearch.xpack.ql.querydsl.query.ScriptQuery;
|
||||
import org.elasticsearch.xpack.ql.type.DataType;
|
||||
import org.elasticsearch.xpack.sql.expression.function.scalar.datetime.DateTimeFunction;
|
||||
import org.elasticsearch.xpack.sql.expression.function.scalar.geo.StDistance;
|
||||
import org.elasticsearch.xpack.sql.type.SqlDataTypeConverter;
|
||||
|
||||
public class SqlTranslatorHandler implements TranslatorHandler {
|
||||
|
||||
@ -53,4 +55,9 @@ public class SqlTranslatorHandler implements TranslatorHandler {
|
||||
}
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public Object convert(Object value, DataType dataType) {
|
||||
return SqlDataTypeConverter.convert(value, dataType);
|
||||
}
|
||||
}
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user