Break QueryTranslator into QL and SQL (#52397)

Refactor the code to allow contextual parameterization of dateFormat and
name.
Separate aggs/query implementation though there's room for improvement
in the future

(cherry picked from commit e086f81b688875b33d01e4504ce7377031c8cf28)
This commit is contained in:
Costin Leau 2020-02-17 16:17:14 +02:00 committed by Costin Leau
parent cc628748e1
commit 20862fe64f
8 changed files with 589 additions and 257 deletions

View File

@ -0,0 +1,35 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License;
* you may not use this file except in compliance with the Elastic License.
*/
package org.elasticsearch.xpack.ql.planner;
import org.elasticsearch.xpack.ql.expression.Expression;
import org.elasticsearch.xpack.ql.expression.FieldAttribute;
import org.elasticsearch.xpack.ql.querydsl.query.NestedQuery;
import org.elasticsearch.xpack.ql.querydsl.query.Query;
import org.elasticsearch.xpack.ql.util.ReflectionUtils;
public abstract class ExpressionTranslator<E extends Expression> {
private final Class<E> typeToken = ReflectionUtils.detectSuperTypeForRuleLike(getClass());
@SuppressWarnings("unchecked")
public Query translate(Expression exp, TranslatorHandler handler) {
return (typeToken.isInstance(exp) ? asQuery((E) exp, handler) : null);
}
protected abstract Query asQuery(E e, TranslatorHandler handler);
public static Query wrapIfNested(Query query, Expression exp) {
if (query != null && exp instanceof FieldAttribute) {
FieldAttribute fa = (FieldAttribute) exp;
if (fa.isNested()) {
return new NestedQuery(fa.source(), fa.nestedParent().name(), query);
}
}
return query;
}
}

View File

@ -0,0 +1,351 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License;
* you may not use this file except in compliance with the Elastic License.
*/
package org.elasticsearch.xpack.ql.planner;
import org.elasticsearch.common.time.DateFormatter;
import org.elasticsearch.xpack.ql.QlIllegalArgumentException;
import org.elasticsearch.xpack.ql.expression.Expression;
import org.elasticsearch.xpack.ql.expression.Expressions;
import org.elasticsearch.xpack.ql.expression.FieldAttribute;
import org.elasticsearch.xpack.ql.expression.function.scalar.ScalarFunction;
import org.elasticsearch.xpack.ql.expression.predicate.Range;
import org.elasticsearch.xpack.ql.expression.predicate.fulltext.MatchQueryPredicate;
import org.elasticsearch.xpack.ql.expression.predicate.fulltext.MultiMatchQueryPredicate;
import org.elasticsearch.xpack.ql.expression.predicate.fulltext.StringQueryPredicate;
import org.elasticsearch.xpack.ql.expression.predicate.logical.And;
import org.elasticsearch.xpack.ql.expression.predicate.logical.Not;
import org.elasticsearch.xpack.ql.expression.predicate.logical.Or;
import org.elasticsearch.xpack.ql.expression.predicate.operator.comparison.BinaryComparison;
import org.elasticsearch.xpack.ql.expression.predicate.operator.comparison.Equals;
import org.elasticsearch.xpack.ql.expression.predicate.operator.comparison.GreaterThan;
import org.elasticsearch.xpack.ql.expression.predicate.operator.comparison.GreaterThanOrEqual;
import org.elasticsearch.xpack.ql.expression.predicate.operator.comparison.LessThan;
import org.elasticsearch.xpack.ql.expression.predicate.operator.comparison.LessThanOrEqual;
import org.elasticsearch.xpack.ql.expression.predicate.operator.comparison.NotEquals;
import org.elasticsearch.xpack.ql.expression.predicate.operator.comparison.NullEquals;
import org.elasticsearch.xpack.ql.expression.predicate.regex.Like;
import org.elasticsearch.xpack.ql.expression.predicate.regex.LikePattern;
import org.elasticsearch.xpack.ql.expression.predicate.regex.RLike;
import org.elasticsearch.xpack.ql.expression.predicate.regex.RegexMatch;
import org.elasticsearch.xpack.ql.querydsl.query.BoolQuery;
import org.elasticsearch.xpack.ql.querydsl.query.MatchQuery;
import org.elasticsearch.xpack.ql.querydsl.query.MultiMatchQuery;
import org.elasticsearch.xpack.ql.querydsl.query.NotQuery;
import org.elasticsearch.xpack.ql.querydsl.query.Query;
import org.elasticsearch.xpack.ql.querydsl.query.QueryStringQuery;
import org.elasticsearch.xpack.ql.querydsl.query.RangeQuery;
import org.elasticsearch.xpack.ql.querydsl.query.RegexQuery;
import org.elasticsearch.xpack.ql.querydsl.query.ScriptQuery;
import org.elasticsearch.xpack.ql.querydsl.query.TermQuery;
import org.elasticsearch.xpack.ql.querydsl.query.WildcardQuery;
import org.elasticsearch.xpack.ql.tree.Source;
import org.elasticsearch.xpack.ql.util.Check;
import org.elasticsearch.xpack.ql.util.Holder;
import java.time.OffsetTime;
import java.time.ZonedDateTime;
import java.time.temporal.TemporalAccessor;
import java.util.Arrays;
import java.util.List;
public final class ExpressionTranslators {
public static final String DATE_FORMAT = "strict_date_time";
public static final String TIME_FORMAT = "strict_hour_minute_second_millis";
public static final List<ExpressionTranslator<?>> QUERY_TRANSLATORS = Arrays.asList(
new BinaryComparisons(),
new Ranges(),
new BinaryLogic(),
new Nots(),
new Likes(),
new StringQueries(),
new Matches(),
new MultiMatches(),
new Scalars()
);
public static Query toQuery(Expression e, TranslatorHandler handler) {
Query translation = null;
for (ExpressionTranslator<?> translator : QUERY_TRANSLATORS) {
translation = translator.translate(e, handler);
if (translation != null) {
return translation;
}
}
throw new QlIllegalArgumentException("Don't know how to translate {} {}", e.nodeName(), e);
}
// TODO: see whether escaping is needed
@SuppressWarnings("rawtypes")
public static class Likes extends ExpressionTranslator<RegexMatch> {
@Override
protected Query asQuery(RegexMatch e, TranslatorHandler handler) {
return doTranslate(e, handler);
}
public static Query doTranslate(RegexMatch e, TranslatorHandler handler) {
Query q = null;
String targetFieldName = null;
if (e.field() instanceof FieldAttribute) {
targetFieldName = handler.nameOf(((FieldAttribute) e.field()).exactAttribute());
} else {
throw new QlIllegalArgumentException("Scalar function [{}] not allowed (yet) as argument for " + e.sourceText(),
Expressions.name(e.field()));
}
if (e instanceof Like) {
LikePattern p = ((Like) e).pattern();
q = new WildcardQuery(e.source(), targetFieldName, p.asLuceneWildcard());
}
if (e instanceof RLike) {
String pattern = ((RLike) e).pattern();
q = new RegexQuery(e.source(), targetFieldName, pattern);
}
return wrapIfNested(q, e.field());
}
}
public static class StringQueries extends ExpressionTranslator<StringQueryPredicate> {
@Override
protected Query asQuery(StringQueryPredicate q, TranslatorHandler handler) {
return doTranslate(q, handler);
}
public static Query doTranslate(StringQueryPredicate q, TranslatorHandler handler) {
return new QueryStringQuery(q.source(), q.query(), q.fields(), q);
}
}
public static class Matches extends ExpressionTranslator<MatchQueryPredicate> {
@Override
protected Query asQuery(MatchQueryPredicate q, TranslatorHandler handler) {
return doTranslate(q, handler);
}
public static Query doTranslate(MatchQueryPredicate q, TranslatorHandler handler) {
return new MatchQuery(q.source(), handler.nameOf(q.field()), q.query(), q);
}
}
public static class MultiMatches extends ExpressionTranslator<MultiMatchQueryPredicate> {
@Override
protected Query asQuery(MultiMatchQueryPredicate q, TranslatorHandler handler) {
return doTranslate(q, handler);
}
public static Query doTranslate(MultiMatchQueryPredicate q, TranslatorHandler handler) {
return new MultiMatchQuery(q.source(), q.query(), q.fields(), q);
}
}
public static class BinaryLogic extends ExpressionTranslator<org.elasticsearch.xpack.ql.expression.predicate.logical.BinaryLogic> {
@Override
protected Query asQuery(org.elasticsearch.xpack.ql.expression.predicate.logical.BinaryLogic e, TranslatorHandler handler) {
if (e instanceof And) {
return and(e.source(), toQuery(e.left(), handler), toQuery(e.right(), handler));
}
if (e instanceof Or) {
return or(e.source(), toQuery(e.left(), handler), toQuery(e.right(), handler));
}
return null;
}
}
public static class Nots extends ExpressionTranslator<Not> {
@Override
protected Query asQuery(Not not, TranslatorHandler handler) {
return doTranslate(not, handler);
}
public static Query doTranslate(Not not, TranslatorHandler handler) {
Expression e = not.field();
Query wrappedQuery = handler.asQuery(not.field());
Query q = wrappedQuery instanceof ScriptQuery ?
new ScriptQuery(not.source(), not.asScript()) :
new NotQuery(not.source(), wrappedQuery);
return wrapIfNested(q, e);
}
}
// assume the Optimizer properly orders the predicates to ease the translation
public static class BinaryComparisons extends ExpressionTranslator<BinaryComparison> {
@Override
protected Query asQuery(BinaryComparison bc, TranslatorHandler handler) {
return doTranslate(bc, handler);
}
public static void checkBinaryComparison(BinaryComparison bc) {
Check.isTrue(bc.right().foldable(),
"Line {}:{}: Comparisons against variables are not (currently) supported; offender [{}] in [{}]",
bc.right().sourceLocation().getLineNumber(), bc.right().sourceLocation().getColumnNumber(),
Expressions.name(bc.right()), bc.symbol());
}
public static Query doTranslate(BinaryComparison bc, TranslatorHandler handler) {
checkBinaryComparison(bc);
return handler.wrapFunctionQuery(bc, bc.left(), translate(bc, handler));
}
private static Query translate(BinaryComparison bc, TranslatorHandler handler) {
Source source = bc.source();
String name = handler.nameOf(bc.left());
Object value = valueOf(bc.right());
String format = handler.dateFormat(bc.left());
boolean isDateLiteralComparison = false;
// for a date constant comparison, we need to use a format for the date, to make sure that the format is the same
// no matter the timezone provided by the user
if ((value instanceof ZonedDateTime || value instanceof OffsetTime) && format == null) {
DateFormatter formatter;
if (value instanceof ZonedDateTime) {
formatter = DateFormatter.forPattern(DATE_FORMAT);
// RangeQueryBuilder accepts an Object as its parameter, but it will call .toString() on the ZonedDateTime instance
// which can have a slightly different format depending on the ZoneId used to create the ZonedDateTime
// Since RangeQueryBuilder can handle date as String as well, we'll format it as String and provide the format as well.
value = formatter.format((ZonedDateTime) value);
} else {
formatter = DateFormatter.forPattern(TIME_FORMAT);
value = formatter.format((OffsetTime) value);
}
format = formatter.pattern();
isDateLiteralComparison = true;
}
if (bc instanceof GreaterThan) {
return new RangeQuery(source, name, value, false, null, false, format);
}
if (bc instanceof GreaterThanOrEqual) {
return new RangeQuery(source, name, value, true, null, false, format);
}
if (bc instanceof LessThan) {
return new RangeQuery(source, name, null, false, value, false, format);
}
if (bc instanceof LessThanOrEqual) {
return new RangeQuery(source, name, null, false, value, true, format);
}
if (bc instanceof Equals || bc instanceof NullEquals || bc instanceof NotEquals) {
if (bc.left() instanceof FieldAttribute) {
// equality should always be against an exact match
// (which is important for strings)
name = ((FieldAttribute) bc.left()).exactAttribute().name();
}
Query query;
if (isDateLiteralComparison) {
// dates equality uses a range query because it's the one that has a "format" parameter
query = new RangeQuery(source, name, value, true, value, true, format);
} else {
query = new TermQuery(source, name, value);
}
if (bc instanceof NotEquals) {
query = new NotQuery(source, query);
}
return query;
}
throw new QlIllegalArgumentException("Don't know how to translate binary comparison [{}] in [{}]", bc.right().nodeString(),
bc);
}
}
public static class Ranges extends ExpressionTranslator<Range> {
@Override
protected Query asQuery(Range r, TranslatorHandler handler) {
return doTranslate(r, handler);
}
public static Query doTranslate(Range r, TranslatorHandler handler) {
Expression val = r.value();
Query query = null;
Holder<Object> lower = new Holder<>(valueOf(r.lower()));
Holder<Object> upper = new Holder<>(valueOf(r.upper()));
Holder<String> format = new Holder<>(handler.dateFormat(val));
// for a date constant comparison, we need to use a format for the date, to make sure that the format is the same
// no matter the timezone provided by the user
if (format.get() == null) {
DateFormatter formatter = null;
if (lower.get() instanceof ZonedDateTime || upper.get() instanceof ZonedDateTime) {
formatter = DateFormatter.forPattern(DATE_FORMAT);
} else if (lower.get() instanceof OffsetTime || upper.get() instanceof OffsetTime) {
formatter = DateFormatter.forPattern(TIME_FORMAT);
}
if (formatter != null) {
// RangeQueryBuilder accepts an Object as its parameter, but it will call .toString() on the ZonedDateTime
// instance which can have a slightly different format depending on the ZoneId used to create the ZonedDateTime
// Since RangeQueryBuilder can handle date as String as well, we'll format it as String and provide the format.
if (lower.get() instanceof ZonedDateTime || lower.get() instanceof OffsetTime) {
lower.set(formatter.format((TemporalAccessor) lower.get()));
}
if (upper.get() instanceof ZonedDateTime || upper.get() instanceof OffsetTime) {
upper.set(formatter.format((TemporalAccessor) upper.get()));
}
format.set(formatter.pattern());
}
}
query = handler.wrapFunctionQuery(r, val, new RangeQuery(r.source(), handler.nameOf(val), lower.get(), r.includeLower(),
upper.get(), r.includeUpper(), format.get()));
return query;
}
}
public static class Scalars extends ExpressionTranslator<ScalarFunction> {
@Override
protected Query asQuery(ScalarFunction f, TranslatorHandler handler) {
return doTranslate(f, handler);
}
public static Query doTranslate(ScalarFunction f, TranslatorHandler handler) {
return handler.wrapFunctionQuery(f, f, new ScriptQuery(f.source(), f.asScript()));
}
}
public static Object valueOf(Expression e) {
if (e.foldable()) {
return e.fold();
}
throw new QlIllegalArgumentException("Cannot determine value for {}", e);
}
public static Query or(Source source, Query left, Query right) {
return boolQuery(source, left, right, false);
}
public static Query and(Source source, Query left, Query right) {
return boolQuery(source, left, right, true);
}
private static Query boolQuery(Source source, Query left, Query right, boolean isAnd) {
Check.isTrue(left != null || right != null, "Both expressions are null");
if (left == null) {
return right;
}
if (right == null) {
return left;
}
return new BoolQuery(source, isAnd, left, right);
}
}

View File

@ -0,0 +1,44 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License;
* you may not use this file except in compliance with the Elastic License.
*/
package org.elasticsearch.xpack.ql.planner;
import org.elasticsearch.xpack.ql.expression.Expression;
import org.elasticsearch.xpack.ql.expression.FieldAttribute;
import org.elasticsearch.xpack.ql.expression.NamedExpression;
import org.elasticsearch.xpack.ql.expression.function.scalar.ScalarFunction;
import org.elasticsearch.xpack.ql.querydsl.query.Query;
import org.elasticsearch.xpack.ql.querydsl.query.ScriptQuery;
public class QlTranslatorHandler implements TranslatorHandler {
@Override
public Query asQuery(Expression e) {
return ExpressionTranslators.toQuery(e, this);
}
@Override
public Query wrapFunctionQuery(ScalarFunction sf, Expression field, Query q) {
if (field instanceof FieldAttribute) {
return ExpressionTranslator.wrapIfNested(q, field);
}
return new ScriptQuery(sf.source(), sf.asScript());
}
@Override
public String nameOf(Expression e) {
if (e instanceof NamedExpression) {
return ((NamedExpression) e).name();
} else {
return e.sourceText();
}
}
@Override
public String dateFormat(Expression e) {
return null;
}
}

View File

@ -0,0 +1,27 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License;
* you may not use this file except in compliance with the Elastic License.
*/
package org.elasticsearch.xpack.ql.planner;
import org.elasticsearch.xpack.ql.expression.Expression;
import org.elasticsearch.xpack.ql.expression.function.scalar.ScalarFunction;
import org.elasticsearch.xpack.ql.querydsl.query.Query;
/**
* Parameterized handler used during query translation.
*
* Provides contextual utilities for an individual query to be performed.
*/
public interface TranslatorHandler {
Query asQuery(Expression e);
Query wrapFunctionQuery(ScalarFunction sf, Expression field, Query q);
String nameOf(Expression e);
String dateFormat(Expression e);
}

View File

@ -30,6 +30,7 @@ import org.elasticsearch.xpack.ql.expression.gen.pipeline.Pipe;
import org.elasticsearch.xpack.ql.expression.gen.pipeline.UnaryPipe;
import org.elasticsearch.xpack.ql.expression.gen.processor.Processor;
import org.elasticsearch.xpack.ql.expression.gen.script.ScriptTemplate;
import org.elasticsearch.xpack.ql.planner.ExpressionTranslators;
import org.elasticsearch.xpack.ql.querydsl.query.Query;
import org.elasticsearch.xpack.ql.rule.Rule;
import org.elasticsearch.xpack.ql.rule.RuleExecutor;
@ -90,7 +91,6 @@ import java.util.Map.Entry;
import java.util.concurrent.atomic.AtomicReference;
import static org.elasticsearch.xpack.ql.util.CollectionUtils.combine;
import static org.elasticsearch.xpack.sql.planner.QueryTranslator.and;
import static org.elasticsearch.xpack.sql.planner.QueryTranslator.toAgg;
import static org.elasticsearch.xpack.sql.planner.QueryTranslator.toQuery;
import static org.elasticsearch.xpack.sql.type.SqlDataTypes.DATE;
@ -181,7 +181,7 @@ class QueryFolder extends RuleExecutor<PhysicalPlan> {
Query query = null;
if (qContainer.query() != null || qt.query != null) {
query = and(plan.source(), qContainer.query(), qt.query);
query = ExpressionTranslators.and(plan.source(), qContainer.query(), qt.query);
}
Aggs aggs = addPipelineAggs(qContainer, qt, plan);

View File

@ -5,12 +5,10 @@
*/
package org.elasticsearch.xpack.sql.planner;
import org.elasticsearch.common.time.DateFormatter;
import org.elasticsearch.geometry.Geometry;
import org.elasticsearch.geometry.Point;
import org.elasticsearch.search.sort.SortOrder;
import org.elasticsearch.xpack.ql.expression.Expression;
import org.elasticsearch.xpack.ql.expression.Expressions;
import org.elasticsearch.xpack.ql.expression.FieldAttribute;
import org.elasticsearch.xpack.ql.expression.Literal;
import org.elasticsearch.xpack.ql.expression.NamedExpression;
@ -18,7 +16,6 @@ import org.elasticsearch.xpack.ql.expression.function.Function;
import org.elasticsearch.xpack.ql.expression.function.aggregate.AggregateFunction;
import org.elasticsearch.xpack.ql.expression.function.aggregate.Count;
import org.elasticsearch.xpack.ql.expression.function.scalar.ScalarFunction;
import org.elasticsearch.xpack.ql.expression.gen.script.ScriptTemplate;
import org.elasticsearch.xpack.ql.expression.predicate.Range;
import org.elasticsearch.xpack.ql.expression.predicate.fulltext.MatchQueryPredicate;
import org.elasticsearch.xpack.ql.expression.predicate.fulltext.MultiMatchQueryPredicate;
@ -27,37 +24,21 @@ import org.elasticsearch.xpack.ql.expression.predicate.logical.And;
import org.elasticsearch.xpack.ql.expression.predicate.logical.Not;
import org.elasticsearch.xpack.ql.expression.predicate.logical.Or;
import org.elasticsearch.xpack.ql.expression.predicate.operator.comparison.BinaryComparison;
import org.elasticsearch.xpack.ql.expression.predicate.operator.comparison.Equals;
import org.elasticsearch.xpack.ql.expression.predicate.operator.comparison.GreaterThan;
import org.elasticsearch.xpack.ql.expression.predicate.operator.comparison.GreaterThanOrEqual;
import org.elasticsearch.xpack.ql.expression.predicate.operator.comparison.LessThan;
import org.elasticsearch.xpack.ql.expression.predicate.operator.comparison.LessThanOrEqual;
import org.elasticsearch.xpack.ql.expression.predicate.operator.comparison.NotEquals;
import org.elasticsearch.xpack.ql.expression.predicate.operator.comparison.NullEquals;
import org.elasticsearch.xpack.ql.expression.predicate.regex.Like;
import org.elasticsearch.xpack.ql.expression.predicate.regex.LikePattern;
import org.elasticsearch.xpack.ql.expression.predicate.regex.RLike;
import org.elasticsearch.xpack.ql.expression.predicate.regex.RegexMatch;
import org.elasticsearch.xpack.ql.querydsl.query.BoolQuery;
import org.elasticsearch.xpack.ql.planner.ExpressionTranslators;
import org.elasticsearch.xpack.ql.planner.TranslatorHandler;
import org.elasticsearch.xpack.ql.querydsl.query.ExistsQuery;
import org.elasticsearch.xpack.ql.querydsl.query.GeoDistanceQuery;
import org.elasticsearch.xpack.ql.querydsl.query.MatchQuery;
import org.elasticsearch.xpack.ql.querydsl.query.MultiMatchQuery;
import org.elasticsearch.xpack.ql.querydsl.query.NestedQuery;
import org.elasticsearch.xpack.ql.querydsl.query.NotQuery;
import org.elasticsearch.xpack.ql.querydsl.query.Query;
import org.elasticsearch.xpack.ql.querydsl.query.QueryStringQuery;
import org.elasticsearch.xpack.ql.querydsl.query.RangeQuery;
import org.elasticsearch.xpack.ql.querydsl.query.RegexQuery;
import org.elasticsearch.xpack.ql.querydsl.query.ScriptQuery;
import org.elasticsearch.xpack.ql.querydsl.query.TermQuery;
import org.elasticsearch.xpack.ql.querydsl.query.TermsQuery;
import org.elasticsearch.xpack.ql.querydsl.query.WildcardQuery;
import org.elasticsearch.xpack.ql.tree.Source;
import org.elasticsearch.xpack.ql.type.DataType;
import org.elasticsearch.xpack.ql.type.DataTypes;
import org.elasticsearch.xpack.ql.util.CollectionUtils;
import org.elasticsearch.xpack.ql.util.Holder;
import org.elasticsearch.xpack.ql.util.ReflectionUtils;
import org.elasticsearch.xpack.sql.SqlIllegalArgumentException;
import org.elasticsearch.xpack.sql.expression.function.aggregate.Avg;
@ -100,14 +81,10 @@ import org.elasticsearch.xpack.sql.querydsl.agg.TopHitsAgg;
import org.elasticsearch.xpack.sql.type.SqlDataTypeConverter;
import org.elasticsearch.xpack.sql.util.Check;
import java.time.OffsetTime;
import java.time.ZonedDateTime;
import java.time.temporal.TemporalAccessor;
import java.util.Arrays;
import java.util.LinkedHashSet;
import java.util.List;
import java.util.Set;
import java.util.function.Supplier;
import static java.util.Collections.singletonList;
import static org.elasticsearch.xpack.ql.expression.Expressions.id;
@ -121,7 +98,7 @@ final class QueryTranslator {
private QueryTranslator(){}
private static final List<ExpressionTranslator<?>> QUERY_TRANSLATORS = Arrays.asList(
private static final List<SqlExpressionTranslator<?>> QUERY_TRANSLATORS = Arrays.asList(
new BinaryComparisons(),
new InComparisons(),
new Ranges(),
@ -174,8 +151,9 @@ final class QueryTranslator {
static QueryTranslation toQuery(Expression e, boolean onAggs) {
QueryTranslation translation = null;
for (ExpressionTranslator<?> translator : QUERY_TRANSLATORS) {
translation = translator.translate(e, onAggs);
TranslatorHandler handler = new SqlTranslatorHandler(onAggs);
for (SqlExpressionTranslator<?> translator : QUERY_TRANSLATORS) {
translation = translator.translate(e, onAggs, handler);
if (translation != null) {
return translation;
}
@ -207,7 +185,7 @@ final class QueryTranslator {
Query newQ = null;
if (left.query != null || right.query != null) {
newQ = and(source, left.query, right.query);
newQ = ExpressionTranslators.and(source, left.query, right.query);
}
AggFilter aggFilter;
@ -225,17 +203,6 @@ final class QueryTranslator {
return new QueryTranslation(newQ, aggFilter);
}
static Query and(Source source, Query left, Query right) {
Check.isTrue(left != null || right != null, "Both expressions are null");
if (left == null) {
return right;
}
if (right == null) {
return left;
}
return new BoolQuery(source, true, left, right);
}
static QueryTranslation or(Source source, QueryTranslation left, QueryTranslation right) {
Check.isTrue(left != null || right != null, "Both expressions are null");
if (left == null) {
@ -247,7 +214,7 @@ final class QueryTranslator {
Query newQ = null;
if (left.query != null || right.query != null) {
newQ = or(source, left.query, right.query);
newQ = ExpressionTranslators.or(source, left.query, right.query);
}
AggFilter aggFilter = null;
@ -265,18 +232,6 @@ final class QueryTranslator {
return new QueryTranslation(newQ, aggFilter);
}
static Query or(Source source, Query left, Query right) {
Check.isTrue(left != null || right != null, "Both expressions are null");
if (left == null) {
return right;
}
if (right == null) {
return left;
}
return new BoolQuery(source, false, left, right);
}
static String nameOf(Expression e) {
if (e instanceof DateTimeFunction) {
return nameOf(((DateTimeFunction) e).field());
@ -311,7 +266,7 @@ final class QueryTranslator {
return String.valueOf(((Literal) arg).value());
}
throw new SqlIllegalArgumentException("Does not know how to convert argument {} for function {}", arg.nodeString(),
af.nodeString());
af.nodeString());
}
private static String topAggsField(AggregateFunction af, Expression e) {
@ -321,68 +276,52 @@ final class QueryTranslator {
if (e instanceof FieldAttribute) {
return ((FieldAttribute) e).exactAttribute().name();
}
throw new SqlIllegalArgumentException("Does not know how to convert argument {} for function {}", e.nodeString(),
af.nodeString());
throw new SqlIllegalArgumentException("Does not know how to convert argument {} for function {}", e.nodeString(), af.nodeString());
}
// TODO: see whether escaping is needed
@SuppressWarnings("rawtypes")
static class Likes extends ExpressionTranslator<RegexMatch> {
static class Likes extends SqlExpressionTranslator<RegexMatch> {
@Override
protected QueryTranslation asQuery(RegexMatch e, boolean onAggs) {
Query q = null;
String targetFieldName = null;
if (e.field() instanceof FieldAttribute) {
targetFieldName = nameOf(((FieldAttribute) e.field()).exactAttribute());
} else {
throw new SqlIllegalArgumentException("Scalar function [{}] not allowed (yet) as argument for " + e.sourceText(),
Expressions.name(e.field()));
}
if (e instanceof Like) {
LikePattern p = ((Like) e).pattern();
q = new WildcardQuery(e.source(), targetFieldName, p.asLuceneWildcard());
}
if (e instanceof RLike) {
String pattern = ((RLike) e).pattern();
q = new RegexQuery(e.source(), targetFieldName, pattern);
}
return q != null ? new QueryTranslation(wrapIfNested(q, e.field())) : null;
protected QueryTranslation asQuery(RegexMatch e, boolean onAggs, TranslatorHandler handler) {
Check.isTrue(onAggs == false, "Like not supported within an aggregation context");
return new QueryTranslation(org.elasticsearch.xpack.ql.planner.ExpressionTranslators.Likes.doTranslate(e, handler));
}
}
static class StringQueries extends ExpressionTranslator<StringQueryPredicate> {
static class StringQueries extends SqlExpressionTranslator<StringQueryPredicate> {
@Override
protected QueryTranslation asQuery(StringQueryPredicate q, boolean onAggs) {
return new QueryTranslation(new QueryStringQuery(q.source(), q.query(), q.fields(), q));
protected QueryTranslation asQuery(StringQueryPredicate q, boolean onAggs, TranslatorHandler handler) {
Check.isTrue(onAggs == false, "Like not supported within an aggregation context");
return new QueryTranslation(org.elasticsearch.xpack.ql.planner.ExpressionTranslators.StringQueries.doTranslate(q, handler));
}
}
static class Matches extends ExpressionTranslator<MatchQueryPredicate> {
static class Matches extends SqlExpressionTranslator<MatchQueryPredicate> {
@Override
protected QueryTranslation asQuery(MatchQueryPredicate q, boolean onAggs) {
return new QueryTranslation(wrapIfNested(new MatchQuery(q.source(), nameOf(q.field()), q.query(), q), q.field()));
protected QueryTranslation asQuery(MatchQueryPredicate q, boolean onAggs, TranslatorHandler handler) {
Check.isTrue(onAggs == false, "Like not supported within an aggregation context");
return new QueryTranslation(org.elasticsearch.xpack.ql.planner.ExpressionTranslators.Matches.doTranslate(q, handler));
}
}
static class MultiMatches extends ExpressionTranslator<MultiMatchQueryPredicate> {
static class MultiMatches extends SqlExpressionTranslator<MultiMatchQueryPredicate> {
@Override
protected QueryTranslation asQuery(MultiMatchQueryPredicate q, boolean onAggs) {
return new QueryTranslation(new MultiMatchQuery(q.source(), q.query(), q.fields(), q));
protected QueryTranslation asQuery(MultiMatchQueryPredicate q, boolean onAggs, TranslatorHandler handler) {
Check.isTrue(onAggs == false, "Like not supported within an aggregation context");
return new QueryTranslation(org.elasticsearch.xpack.ql.planner.ExpressionTranslators.MultiMatches.doTranslate(q, handler));
}
}
static class BinaryLogic extends ExpressionTranslator<org.elasticsearch.xpack.ql.expression.predicate.logical.BinaryLogic> {
static class BinaryLogic extends SqlExpressionTranslator<org.elasticsearch.xpack.ql.expression.predicate.logical.BinaryLogic> {
@Override
protected QueryTranslation asQuery(org.elasticsearch.xpack.ql.expression.predicate.logical.BinaryLogic e, boolean onAggs) {
protected QueryTranslation asQuery(org.elasticsearch.xpack.ql.expression.predicate.logical.BinaryLogic e, boolean onAggs,
TranslatorHandler handler) {
if (e instanceof And) {
return and(e.source(), toQuery(e.left(), onAggs), toQuery(e.right(), onAggs));
}
@ -394,36 +333,27 @@ final class QueryTranslator {
}
}
static class Nots extends ExpressionTranslator<Not> {
static class Nots extends SqlExpressionTranslator<Not> {
@Override
protected QueryTranslation asQuery(Not not, boolean onAggs) {
protected QueryTranslation asQuery(Not not, boolean onAggs, TranslatorHandler handler) {
Query query = null;
AggFilter aggFilter = null;
if (onAggs) {
aggFilter = new AggFilter(id(not), not.asScript());
} else {
Expression e = not.field();
Query wrappedQuery = toQuery(not.field(), false).query;
Query q = wrappedQuery instanceof ScriptQuery ? new ScriptQuery(not.source(),
not.asScript()) : new NotQuery(not.source(), wrappedQuery);
if (e instanceof FieldAttribute) {
query = wrapIfNested(q, e);
}
query = q;
query = org.elasticsearch.xpack.ql.planner.ExpressionTranslators.Nots.doTranslate(not, handler);
}
return new QueryTranslation(query, aggFilter);
}
}
static class IsNotNullTranslator extends ExpressionTranslator<IsNotNull> {
static class IsNotNullTranslator extends SqlExpressionTranslator<IsNotNull> {
@Override
protected QueryTranslation asQuery(IsNotNull isNotNull, boolean onAggs) {
protected QueryTranslation asQuery(IsNotNull isNotNull, boolean onAggs, TranslatorHandler handler) {
Query query = null;
AggFilter aggFilter = null;
@ -432,22 +362,21 @@ final class QueryTranslator {
} else {
Query q = null;
if (isNotNull.field() instanceof FieldAttribute) {
q = new ExistsQuery(isNotNull.source(), nameOf(isNotNull.field()));
q = new ExistsQuery(isNotNull.source(), handler.nameOf(isNotNull.field()));
} else {
q = new ScriptQuery(isNotNull.source(), isNotNull.asScript());
}
final Query qu = q;
query = handleQuery(isNotNull, isNotNull.field(), () -> qu);
query = handler.wrapFunctionQuery(isNotNull, isNotNull.field(), q);
}
return new QueryTranslation(query, aggFilter);
}
}
static class IsNullTranslator extends ExpressionTranslator<IsNull> {
static class IsNullTranslator extends SqlExpressionTranslator<IsNull> {
@Override
protected QueryTranslation asQuery(IsNull isNull, boolean onAggs) {
protected QueryTranslation asQuery(IsNull isNull, boolean onAggs, TranslatorHandler handler) {
Query query = null;
AggFilter aggFilter = null;
@ -456,13 +385,11 @@ final class QueryTranslator {
} else {
Query q = null;
if (isNull.field() instanceof FieldAttribute) {
q = new NotQuery(isNull.source(), new ExistsQuery(isNull.source(), nameOf(isNull.field())));
q = new NotQuery(isNull.source(), new ExistsQuery(isNull.source(), handler.nameOf(isNull.field())));
} else {
q = new ScriptQuery(isNull.source(), isNull.asScript());
}
final Query qu = q;
query = handleQuery(isNull, isNull.field(), () -> qu);
query = handler.wrapFunctionQuery(isNull, isNull.field(), q);
}
return new QueryTranslation(query, aggFilter);
@ -470,14 +397,11 @@ final class QueryTranslator {
}
// assume the Optimizer properly orders the predicates to ease the translation
static class BinaryComparisons extends ExpressionTranslator<BinaryComparison> {
static class BinaryComparisons extends SqlExpressionTranslator<BinaryComparison> {
@Override
protected QueryTranslation asQuery(BinaryComparison bc, boolean onAggs) {
Check.isTrue(bc.right().foldable(),
"Line {}:{}: Comparisons against variables are not (currently) supported; offender [{}] in [{}]",
bc.right().sourceLocation().getLineNumber(), bc.right().sourceLocation().getColumnNumber(),
Expressions.name(bc.right()), bc.symbol());
protected QueryTranslation asQuery(BinaryComparison bc, boolean onAggs, TranslatorHandler handler) {
org.elasticsearch.xpack.ql.planner.ExpressionTranslators.BinaryComparisons.checkBinaryComparison(bc);
Query query = null;
AggFilter aggFilter = null;
@ -488,35 +412,14 @@ final class QueryTranslator {
if (onAggs) {
aggFilter = new AggFilter(id(bc.left()), bc.asScript());
} else {
query = handleQuery(bc, bc.left(), () -> translateQuery(bc));
query = translateQuery(bc, handler);
}
return new QueryTranslation(query, aggFilter);
}
private static Query translateQuery(BinaryComparison bc) {
private static Query translateQuery(BinaryComparison bc, TranslatorHandler handler) {
Source source = bc.source();
String name = nameOf(bc.left());
Object value = valueOf(bc.right());
String format = dateFormat(bc.left());
boolean isDateLiteralComparison = false;
// for a date constant comparison, we need to use a format for the date, to make sure that the format is the same
// no matter the timezone provided by the user
if ((value instanceof ZonedDateTime || value instanceof OffsetTime) && format == null) {
DateFormatter formatter;
if (value instanceof ZonedDateTime) {
formatter = DateFormatter.forPattern(DATE_FORMAT);
// RangeQueryBuilder accepts an Object as its parameter, but it will call .toString() on the ZonedDateTime instance
// which can have a slightly different format depending on the ZoneId used to create the ZonedDateTime
// Since RangeQueryBuilder can handle date as String as well, we'll format it as String and provide the format as well.
value = formatter.format((ZonedDateTime) value);
} else {
formatter = DateFormatter.forPattern(TIME_FORMAT);
value = formatter.format((OffsetTime) value);
}
format = formatter.pattern();
isDateLiteralComparison = true;
}
// Possible geo optimization
if (bc.left() instanceof StDistance && value instanceof Number) {
@ -536,47 +439,16 @@ final class QueryTranslator {
}
}
}
if (bc instanceof GreaterThan) {
return new RangeQuery(source, name, value, false, null, false, format);
}
if (bc instanceof GreaterThanOrEqual) {
return new RangeQuery(source, name, value, true, null, false, format);
}
if (bc instanceof LessThan) {
return new RangeQuery(source, name, null, false, value, false, format);
}
if (bc instanceof LessThanOrEqual) {
return new RangeQuery(source, name, null, false, value, true, format);
}
if (bc instanceof Equals || bc instanceof NullEquals || bc instanceof NotEquals) {
if (bc.left() instanceof FieldAttribute) {
// equality should always be against an exact match
// (which is important for strings)
name = ((FieldAttribute) bc.left()).exactAttribute().name();
}
Query query;
if (isDateLiteralComparison) {
// dates equality uses a range query because it's the one that has a "format" parameter
query = new RangeQuery(source, name, value, true, value, true, format);
} else {
query = new TermQuery(source, name, value);
}
if (bc instanceof NotEquals) {
query = new NotQuery(source, query);
}
return query;
}
throw new SqlIllegalArgumentException("Don't know how to translate binary comparison [{}] in [{}]", bc.right().nodeString(),
bc);
// fallback default
return org.elasticsearch.xpack.ql.planner.ExpressionTranslators.BinaryComparisons.doTranslate(bc, handler);
}
}
// assume the Optimizer properly orders the predicates to ease the translation
static class InComparisons extends ExpressionTranslator<In> {
static class InComparisons extends SqlExpressionTranslator<In> {
@Override
protected QueryTranslation asQuery(In in, boolean onAggs) {
protected QueryTranslation asQuery(In in, boolean onAggs, TranslatorHandler handler) {
Query query = null;
AggFilter aggFilter = null;
@ -604,17 +476,16 @@ final class QueryTranslator {
} else {
q = new ScriptQuery(in.source(), in.asScript());
}
Query qu = q;
query = handleQuery(in, in.value(), () -> qu);
query = handler.wrapFunctionQuery(in, in.value(), q);
}
return new QueryTranslation(query, aggFilter);
}
}
static class Ranges extends ExpressionTranslator<Range> {
static class Ranges extends SqlExpressionTranslator<Range> {
@Override
protected QueryTranslation asQuery(Range r, boolean onAggs) {
protected QueryTranslation asQuery(Range r, boolean onAggs, TranslatorHandler handler) {
Expression e = r.value();
Query query = null;
@ -626,55 +497,24 @@ final class QueryTranslator {
if (onAggs) {
aggFilter = new AggFilter(id(e), r.asScript());
} else {
Holder<Object> lower = new Holder<>(valueOf(r.lower()));
Holder<Object> upper = new Holder<>(valueOf(r.upper()));
Holder<String> format = new Holder<>(dateFormat(r.value()));
// for a date constant comparison, we need to use a format for the date, to make sure that the format is the same
// no matter the timezone provided by the user
if (format.get() == null) {
DateFormatter formatter = null;
if (lower.get() instanceof ZonedDateTime || upper.get() instanceof ZonedDateTime) {
formatter = DateFormatter.forPattern(DATE_FORMAT);
} else if (lower.get() instanceof OffsetTime || upper.get() instanceof OffsetTime) {
formatter = DateFormatter.forPattern(TIME_FORMAT);
}
if (formatter != null) {
// RangeQueryBuilder accepts an Object as its parameter, but it will call .toString() on the ZonedDateTime
// instance which can have a slightly different format depending on the ZoneId used to create the ZonedDateTime
// Since RangeQueryBuilder can handle date as String as well, we'll format it as String and provide the format.
if (lower.get() instanceof ZonedDateTime || lower.get() instanceof OffsetTime) {
lower.set(formatter.format((TemporalAccessor) lower.get()));
}
if (upper.get() instanceof ZonedDateTime || upper.get() instanceof OffsetTime) {
upper.set(formatter.format((TemporalAccessor) upper.get()));
}
format.set(formatter.pattern());
}
}
query = handleQuery(r, r.value(),
() -> new RangeQuery(r.source(), nameOf(r.value()), lower.get(), r.includeLower(), upper.get(), r.includeUpper(),
format.get()));
query = org.elasticsearch.xpack.ql.planner.ExpressionTranslators.Ranges.doTranslate(r, handler);
}
return new QueryTranslation(query, aggFilter);
}
}
static class Scalars extends ExpressionTranslator<ScalarFunction> {
static class Scalars extends SqlExpressionTranslator<ScalarFunction> {
@Override
protected QueryTranslation asQuery(ScalarFunction f, boolean onAggs) {
ScriptTemplate script = f.asScript();
protected QueryTranslation asQuery(ScalarFunction f, boolean onAggs, TranslatorHandler handler) {
Query query = null;
AggFilter aggFilter = null;
if (onAggs) {
aggFilter = new AggFilter(id(f), script);
aggFilter = new AggFilter(id(f), f.asScript());
} else {
query = handleQuery(f, f, () -> new ScriptQuery(f.source(), script));
query = org.elasticsearch.xpack.ql.planner.ExpressionTranslators.Scalars.doTranslate(f, handler);
}
return new QueryTranslation(query, aggFilter);
@ -682,6 +522,19 @@ final class QueryTranslator {
}
abstract static class SqlExpressionTranslator<E extends Expression> {
private final Class<E> typeToken = ReflectionUtils.detectSuperTypeForRuleLike(getClass());
@SuppressWarnings("unchecked")
public QueryTranslation translate(Expression exp, boolean onAggs, TranslatorHandler handler) {
return (typeToken.isInstance(exp) ? asQuery((E) exp, onAggs, handler) : null);
}
protected abstract QueryTranslation asQuery(E e, boolean onAggs, TranslatorHandler handler);
}
//
// Agg translators
//
@ -844,38 +697,4 @@ final class QueryTranslator {
protected abstract LeafAgg toAgg(String id, C f);
}
abstract static class ExpressionTranslator<E extends Expression> {
private final Class<E> typeToken = ReflectionUtils.detectSuperTypeForRuleLike(getClass());
@SuppressWarnings("unchecked")
public QueryTranslation translate(Expression exp, boolean onAggs) {
return (typeToken.isInstance(exp) ? asQuery((E) exp, onAggs) : null);
}
protected abstract QueryTranslation asQuery(E e, boolean onAggs);
protected static Query handleQuery(ScalarFunction sf, Expression field, Supplier<Query> query) {
Query q = query.get();
if (field instanceof StDistance && q instanceof GeoDistanceQuery) {
return wrapIfNested(q, ((StDistance) field).left());
}
if (field instanceof FieldAttribute) {
return wrapIfNested(q, field);
}
return new ScriptQuery(sf.source(), sf.asScript());
}
protected static Query wrapIfNested(Query query, Expression exp) {
if (exp instanceof FieldAttribute) {
FieldAttribute fa = (FieldAttribute) exp;
if (fa.isNested()) {
return new NestedQuery(fa.source(), fa.nestedParent().name(), query);
}
}
return query;
}
}
}

View File

@ -0,0 +1,56 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License;
* you may not use this file except in compliance with the Elastic License.
*/
package org.elasticsearch.xpack.sql.planner;
import org.elasticsearch.xpack.ql.expression.Expression;
import org.elasticsearch.xpack.ql.expression.FieldAttribute;
import org.elasticsearch.xpack.ql.expression.function.scalar.ScalarFunction;
import org.elasticsearch.xpack.ql.planner.ExpressionTranslator;
import org.elasticsearch.xpack.ql.planner.TranslatorHandler;
import org.elasticsearch.xpack.ql.querydsl.query.GeoDistanceQuery;
import org.elasticsearch.xpack.ql.querydsl.query.Query;
import org.elasticsearch.xpack.ql.querydsl.query.ScriptQuery;
import org.elasticsearch.xpack.sql.expression.function.scalar.datetime.DateTimeFunction;
import org.elasticsearch.xpack.sql.expression.function.scalar.geo.StDistance;
public class SqlTranslatorHandler implements TranslatorHandler {
private final boolean onAggs;
public SqlTranslatorHandler(boolean onAggs) {
this.onAggs = onAggs;
}
@Override
public Query asQuery(Expression e) {
return QueryTranslator.toQuery(e, onAggs).query;
}
@Override
public Query wrapFunctionQuery(ScalarFunction sf, Expression field, Query q) {
if (field instanceof StDistance && q instanceof GeoDistanceQuery) {
return ExpressionTranslator.wrapIfNested(q, ((StDistance) field).left());
}
if (field instanceof FieldAttribute) {
return ExpressionTranslator.wrapIfNested(q, field);
}
return new ScriptQuery(sf.source(), sf.asScript());
}
@Override
public String nameOf(Expression e) {
return QueryTranslator.nameOf(e);
}
@Override
public String dateFormat(Expression e) {
if (e instanceof DateTimeFunction) {
return ((DateTimeFunction) e).dateTimeFormat();
}
return null;
}
}

View File

@ -12,6 +12,7 @@ import org.elasticsearch.search.aggregations.bucket.filter.FilterAggregationBuil
import org.elasticsearch.search.aggregations.metrics.AvgAggregationBuilder;
import org.elasticsearch.search.aggregations.metrics.CardinalityAggregationBuilder;
import org.elasticsearch.test.ESTestCase;
import org.elasticsearch.xpack.ql.QlIllegalArgumentException;
import org.elasticsearch.xpack.ql.expression.Alias;
import org.elasticsearch.xpack.ql.expression.Expression;
import org.elasticsearch.xpack.ql.expression.FieldAttribute;
@ -37,7 +38,6 @@ import org.elasticsearch.xpack.ql.querydsl.query.TermQuery;
import org.elasticsearch.xpack.ql.querydsl.query.TermsQuery;
import org.elasticsearch.xpack.ql.querydsl.query.WildcardQuery;
import org.elasticsearch.xpack.ql.type.EsField;
import org.elasticsearch.xpack.sql.SqlIllegalArgumentException;
import org.elasticsearch.xpack.sql.SqlTestUtils;
import org.elasticsearch.xpack.sql.analysis.analyzer.Analyzer;
import org.elasticsearch.xpack.sql.analysis.analyzer.Verifier;
@ -230,7 +230,7 @@ public class QueryTranslatorTests extends ESTestCase {
p = ((Project) p).child();
assertTrue(p instanceof Filter);
Expression condition = ((Filter) p).condition();
SqlIllegalArgumentException ex = expectThrows(SqlIllegalArgumentException.class, () -> QueryTranslator.toQuery(condition, false));
QlIllegalArgumentException ex = expectThrows(QlIllegalArgumentException.class, () -> QueryTranslator.toQuery(condition, false));
assertEquals("Line 1:43: Comparisons against variables are not (currently) supported; offender [int] in [>]", ex.getMessage());
}
@ -475,7 +475,7 @@ public class QueryTranslatorTests extends ESTestCase {
p = ((Project) p).child();
assertTrue(p instanceof Filter);
Expression condition = ((Filter) p).condition();
SqlIllegalArgumentException ex = expectThrows(SqlIllegalArgumentException.class, () -> QueryTranslator.toQuery(condition, false));
QlIllegalArgumentException ex = expectThrows(QlIllegalArgumentException.class, () -> QueryTranslator.toQuery(condition, false));
assertEquals("Scalar function [LTRIM(keyword)] not allowed (yet) as argument for LTRIM(keyword) like '%a%'", ex.getMessage());
}
@ -485,7 +485,7 @@ public class QueryTranslatorTests extends ESTestCase {
p = ((Project) p).child();
assertTrue(p instanceof Filter);
Expression condition = ((Filter) p).condition();
SqlIllegalArgumentException ex = expectThrows(SqlIllegalArgumentException.class, () -> QueryTranslator.toQuery(condition, false));
QlIllegalArgumentException ex = expectThrows(QlIllegalArgumentException.class, () -> QueryTranslator.toQuery(condition, false));
assertEquals("Scalar function [LTRIM(keyword)] not allowed (yet) as argument for LTRIM(keyword) RLIKE '.*a.*'", ex.getMessage());
}