Expressions work better with strings. (#4394)

* Expressions work better with strings.

- ExpressionObjectSelector able to read from string columns, and able to
  return strings.
- ExpressionVirtualColumn able to offer string (and long for that matter)
  as its native type.
- ExpressionPostAggregator able to return strings.
- groupBy, topN: Allow post-aggregators to accept dimensions as inputs,
  making ExpressionPostAggregator more useful.
- topN: Use DimExtractionTopNAlgorithm for STRING columns that do not
  have dictionaries, allowing it to work with STRING-type expression
  virtual columns.
- Adjusts null handling to better match the rest of Druid: null and
  empty string treated the same; nulls implicitly treated as zeroes in
  numeric context.

* Code review comments.

* More code review.

* Fix test.

* Adjust annotations.
This commit is contained in:
Gian Merlino 2017-06-14 14:50:18 -07:00 committed by GitHub
parent 976492c186
commit 6edee7f434
26 changed files with 672 additions and 261 deletions

View File

@ -19,14 +19,18 @@
package io.druid.math.expr; package io.druid.math.expr;
import com.google.common.base.Preconditions;
import com.google.common.base.Strings;
import com.google.common.math.LongMath; import com.google.common.math.LongMath;
import com.google.common.primitives.Ints; import com.google.common.primitives.Ints;
import io.druid.java.util.common.IAE; import io.druid.java.util.common.IAE;
import io.druid.java.util.common.ISE; import io.druid.java.util.common.ISE;
import io.druid.java.util.common.guava.Comparators;
import javax.annotation.Nonnull; import javax.annotation.Nonnull;
import javax.annotation.Nullable; import javax.annotation.Nullable;
import java.util.List; import java.util.List;
import java.util.Objects;
/** /**
*/ */
@ -57,7 +61,8 @@ public interface Expr
interface ObjectBinding interface ObjectBinding
{ {
Number get(String name); @Nullable
Object get(String name);
} }
void visit(Visitor visitor); void visit(Visitor visitor);
@ -89,10 +94,10 @@ class LongExpr extends ConstantExpr
public LongExpr(Long value) public LongExpr(Long value)
{ {
this.value = value; this.value = Preconditions.checkNotNull(value, "value");
} }
@Nullable @Nonnull
@Override @Override
public Object getLiteralValue() public Object getLiteralValue()
{ {
@ -119,7 +124,7 @@ class StringExpr extends ConstantExpr
public StringExpr(String value) public StringExpr(String value)
{ {
this.value = value; this.value = Strings.emptyToNull(value);
} }
@Nullable @Nullable
@ -149,10 +154,10 @@ class DoubleExpr extends ConstantExpr
public DoubleExpr(Double value) public DoubleExpr(Double value)
{ {
this.value = value; this.value = Preconditions.checkNotNull(value, "value");
} }
@Nullable @Nonnull
@Override @Override
public Object getLiteralValue() public Object getLiteralValue()
{ {
@ -357,19 +362,16 @@ abstract class BinaryEvalOpExprBase extends BinaryOpExprBase
{ {
ExprEval leftVal = left.eval(bindings); ExprEval leftVal = left.eval(bindings);
ExprEval rightVal = right.eval(bindings); ExprEval rightVal = right.eval(bindings);
if (leftVal.isNull() || rightVal.isNull()) { if (leftVal.type() == ExprType.STRING && rightVal.type() == ExprType.STRING) {
return ExprEval.of(null);
}
if (leftVal.type() == ExprType.STRING || rightVal.type() == ExprType.STRING) {
return evalString(leftVal.asString(), rightVal.asString()); return evalString(leftVal.asString(), rightVal.asString());
} } else if (leftVal.type() == ExprType.LONG && rightVal.type() == ExprType.LONG) {
if (leftVal.type() == ExprType.LONG && rightVal.type() == ExprType.LONG) {
return ExprEval.of(evalLong(leftVal.asLong(), rightVal.asLong())); return ExprEval.of(evalLong(leftVal.asLong(), rightVal.asLong()));
} else {
return ExprEval.of(evalDouble(leftVal.asDouble(), rightVal.asDouble()));
} }
return ExprEval.of(evalDouble(leftVal.asDouble(), rightVal.asDouble()));
} }
protected ExprEval evalString(String left, String right) protected ExprEval evalString(@Nullable String left, @Nullable String right)
{ {
throw new IllegalArgumentException("unsupported type " + ExprType.STRING); throw new IllegalArgumentException("unsupported type " + ExprType.STRING);
} }
@ -487,9 +489,9 @@ class BinPlusExpr extends BinaryEvalOpExprBase
} }
@Override @Override
protected ExprEval evalString(String left, String right) protected ExprEval evalString(@Nullable String left, @Nullable String right)
{ {
return ExprEval.of(left + right); return ExprEval.of(Strings.nullToEmpty(left) + Strings.nullToEmpty(right));
} }
@Override @Override
@ -513,9 +515,9 @@ class BinLtExpr extends BinaryEvalOpExprBase
} }
@Override @Override
protected ExprEval evalString(String left, String right) protected ExprEval evalString(@Nullable String left, @Nullable String right)
{ {
return ExprEval.of(left.compareTo(right) < 0, ExprType.LONG); return ExprEval.of(Comparators.<String>naturalNullsFirst().compare(left, right) < 0, ExprType.LONG);
} }
@Override @Override
@ -527,7 +529,8 @@ class BinLtExpr extends BinaryEvalOpExprBase
@Override @Override
protected final double evalDouble(double left, double right) protected final double evalDouble(double left, double right)
{ {
return Evals.asDouble(left < right); // Use Double.compare for more consistent NaN handling.
return Evals.asDouble(Double.compare(left, right) < 0);
} }
} }
@ -539,9 +542,9 @@ class BinLeqExpr extends BinaryEvalOpExprBase
} }
@Override @Override
protected ExprEval evalString(String left, String right) protected ExprEval evalString(@Nullable String left, @Nullable String right)
{ {
return ExprEval.of(left.compareTo(right) <= 0, ExprType.LONG); return ExprEval.of(Comparators.<String>naturalNullsFirst().compare(left, right) <= 0, ExprType.LONG);
} }
@Override @Override
@ -553,7 +556,8 @@ class BinLeqExpr extends BinaryEvalOpExprBase
@Override @Override
protected final double evalDouble(double left, double right) protected final double evalDouble(double left, double right)
{ {
return Evals.asDouble(left <= right); // Use Double.compare for more consistent NaN handling.
return Evals.asDouble(Double.compare(left, right) <= 0);
} }
} }
@ -565,9 +569,9 @@ class BinGtExpr extends BinaryEvalOpExprBase
} }
@Override @Override
protected ExprEval evalString(String left, String right) protected ExprEval evalString(@Nullable String left, @Nullable String right)
{ {
return ExprEval.of(left.compareTo(right) > 0, ExprType.LONG); return ExprEval.of(Comparators.<String>naturalNullsFirst().compare(left, right) > 0, ExprType.LONG);
} }
@Override @Override
@ -579,7 +583,8 @@ class BinGtExpr extends BinaryEvalOpExprBase
@Override @Override
protected final double evalDouble(double left, double right) protected final double evalDouble(double left, double right)
{ {
return Evals.asDouble(left > right); // Use Double.compare for more consistent NaN handling.
return Evals.asDouble(Double.compare(left, right) > 0);
} }
} }
@ -591,9 +596,9 @@ class BinGeqExpr extends BinaryEvalOpExprBase
} }
@Override @Override
protected ExprEval evalString(String left, String right) protected ExprEval evalString(@Nullable String left, @Nullable String right)
{ {
return ExprEval.of(left.compareTo(right) >= 0, ExprType.LONG); return ExprEval.of(Comparators.<String>naturalNullsFirst().compare(left, right) >= 0, ExprType.LONG);
} }
@Override @Override
@ -605,7 +610,8 @@ class BinGeqExpr extends BinaryEvalOpExprBase
@Override @Override
protected final double evalDouble(double left, double right) protected final double evalDouble(double left, double right)
{ {
return Evals.asDouble(left >= right); // Use Double.compare for more consistent NaN handling.
return Evals.asDouble(Double.compare(left, right) >= 0);
} }
} }
@ -617,9 +623,9 @@ class BinEqExpr extends BinaryEvalOpExprBase
} }
@Override @Override
protected ExprEval evalString(String left, String right) protected ExprEval evalString(@Nullable String left, @Nullable String right)
{ {
return ExprEval.of(left.equals(right), ExprType.LONG); return ExprEval.of(Objects.equals(left, right), ExprType.LONG);
} }
@Override @Override
@ -643,9 +649,9 @@ class BinNeqExpr extends BinaryEvalOpExprBase
} }
@Override @Override
protected ExprEval evalString(String left, String right) protected ExprEval evalString(@Nullable String left, @Nullable String right)
{ {
return ExprEval.of(!left.equals(right), ExprType.LONG); return ExprEval.of(!Objects.equals(left, right), ExprType.LONG);
} }
@Override @Override

View File

@ -19,7 +19,11 @@
package io.druid.math.expr; package io.druid.math.expr;
import com.google.common.base.Preconditions;
import com.google.common.base.Strings; import com.google.common.base.Strings;
import com.google.common.primitives.Doubles;
import com.google.common.primitives.Ints;
import io.druid.common.guava.GuavaUtils;
import io.druid.java.util.common.IAE; import io.druid.java.util.common.IAE;
/** /**
@ -72,9 +76,9 @@ public abstract class ExprEval<T>
} }
if (val instanceof Number) { if (val instanceof Number) {
if (val instanceof Float || val instanceof Double) { if (val instanceof Float || val instanceof Double) {
return new DoubleExprEval((Number)val); return new DoubleExprEval((Number) val);
} }
return new LongExprEval((Number)val); return new LongExprEval((Number) val);
} }
return new StringExprEval(val == null ? null : String.valueOf(val)); return new StringExprEval(val == null ? null : String.valueOf(val));
} }
@ -98,11 +102,6 @@ public abstract class ExprEval<T>
return value == null; return value == null;
} }
public Number numericValue()
{
return (Number) value;
}
public abstract int asInt(); public abstract int asInt();
public abstract long asLong(); public abstract long asLong();
@ -120,7 +119,8 @@ public abstract class ExprEval<T>
public abstract Expr toExpr(); public abstract Expr toExpr();
private static abstract class NumericExprEval extends ExprEval<Number> { private static abstract class NumericExprEval extends ExprEval<Number>
{
private NumericExprEval(Number value) private NumericExprEval(Number value)
{ {
@ -150,7 +150,7 @@ public abstract class ExprEval<T>
{ {
private DoubleExprEval(Number value) private DoubleExprEval(Number value)
{ {
super(value); super(Preconditions.checkNotNull(value, "value"));
} }
@Override @Override
@ -182,7 +182,7 @@ public abstract class ExprEval<T>
@Override @Override
public Expr toExpr() public Expr toExpr()
{ {
return new DoubleExpr(value == null ? null : value.doubleValue()); return new DoubleExpr(value.doubleValue());
} }
} }
@ -190,7 +190,7 @@ public abstract class ExprEval<T>
{ {
private LongExprEval(Number value) private LongExprEval(Number value)
{ {
super(value); super(Preconditions.checkNotNull(value, "value"));
} }
@Override @Override
@ -222,7 +222,7 @@ public abstract class ExprEval<T>
@Override @Override
public Expr toExpr() public Expr toExpr()
{ {
return new LongExpr(value == null ? null : value.longValue()); return new LongExpr(value.longValue());
} }
} }
@ -230,7 +230,7 @@ public abstract class ExprEval<T>
{ {
private StringExprEval(String value) private StringExprEval(String value)
{ {
super(value); super(Strings.emptyToNull(value));
} }
@Override @Override
@ -239,28 +239,34 @@ public abstract class ExprEval<T>
return ExprType.STRING; return ExprType.STRING;
} }
@Override
public final boolean isNull()
{
return Strings.isNullOrEmpty(value);
}
@Override @Override
public final int asInt() public final int asInt()
{ {
return Integer.parseInt(value); if (value == null) {
return 0;
}
final Integer theInt = Ints.tryParse(value);
return theInt == null ? 0 : theInt;
} }
@Override @Override
public final long asLong() public final long asLong()
{ {
return Long.parseLong(value); // GuavaUtils.tryParseLong handles nulls, no need for special null handling here.
final Long theLong = GuavaUtils.tryParseLong(value);
return theLong == null ? 0L : theLong;
} }
@Override @Override
public final double asDouble() public final double asDouble()
{ {
return Double.parseDouble(value); if (value == null) {
return 0.0;
}
final Double theDouble = Doubles.tryParse(value);
return theDouble == null ? 0.0 : theDouble;
} }
@Override @Override

View File

@ -148,30 +148,14 @@ public class Parser
public static Expr.ObjectBinding withMap(final Map<String, ?> bindings) public static Expr.ObjectBinding withMap(final Map<String, ?> bindings)
{ {
return new Expr.ObjectBinding() return bindings::get;
{
@Override
public Number get(String name)
{
Number number = (Number)bindings.get(name);
if (number == null && !bindings.containsKey(name)) {
throw new RuntimeException("No binding found for " + name);
}
return number;
}
};
} }
public static Expr.ObjectBinding withSuppliers(final Map<String, Supplier<Number>> bindings) public static Expr.ObjectBinding withSuppliers(final Map<String, Supplier<Object>> bindings)
{ {
return new Expr.ObjectBinding() return (String name) -> {
{ Supplier<Object> supplier = bindings.get(name);
@Override return supplier == null ? null : supplier.get();
public Number get(String name)
{
Supplier<Number> supplier = bindings.get(name);
return supplier == null ? null : supplier.get();
}
}; };
} }
} }

View File

@ -21,11 +21,12 @@ package io.druid.query;
import com.google.common.base.Preconditions; import com.google.common.base.Preconditions;
import com.google.common.collect.Lists; import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
import com.google.common.collect.Sets; import com.google.common.collect.Sets;
import io.druid.query.aggregation.AggregatorFactory; import io.druid.query.aggregation.AggregatorFactory;
import io.druid.query.aggregation.PostAggregator; import io.druid.query.aggregation.PostAggregator;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List; import java.util.List;
import java.util.Map; import java.util.Map;
import java.util.Set; import java.util.Set;
@ -34,8 +35,10 @@ import java.util.Set;
*/ */
public class Queries public class Queries
{ {
public static List<PostAggregator> decoratePostAggregators(List<PostAggregator> postAggs, public static List<PostAggregator> decoratePostAggregators(
Map<String, AggregatorFactory> aggFactories) List<PostAggregator> postAggs,
Map<String, AggregatorFactory> aggFactories
)
{ {
List<PostAggregator> decorated = Lists.newArrayListWithExpectedSize(postAggs.size()); List<PostAggregator> decorated = Lists.newArrayListWithExpectedSize(postAggs.size());
for (PostAggregator aggregator : postAggs) { for (PostAggregator aggregator : postAggs) {
@ -44,34 +47,57 @@ public class Queries
return decorated; return decorated;
} }
/**
* Returns decorated post-aggregators, based on original un-decorated post-aggregators. In addition, this method
* also verifies that there are no output name collisions, and that all of the post-aggregators' required input
* fields are present.
*
* @param otherOutputNames names of fields that will appear in the same output namespace as aggregators and
* post-aggregators. For most built-in query types, this is either empty, or the list of
* dimension output names.
* @param aggFactories aggregator factories for this query
* @param postAggs post-aggregators for this query
*
* @return decorated post-aggregators
*
* @throws NullPointerException if otherOutputNames or aggFactories is null
* @throws IllegalArgumentException if there are any output name collisions or missing post-aggregator inputs
*/
public static List<PostAggregator> prepareAggregations( public static List<PostAggregator> prepareAggregations(
List<String> otherOutputNames,
List<AggregatorFactory> aggFactories, List<AggregatorFactory> aggFactories,
List<PostAggregator> postAggs List<PostAggregator> postAggs
) )
{ {
Preconditions.checkNotNull(otherOutputNames, "otherOutputNames cannot be null");
Preconditions.checkNotNull(aggFactories, "aggregations cannot be null"); Preconditions.checkNotNull(aggFactories, "aggregations cannot be null");
final Map<String, AggregatorFactory> aggsFactoryMap = Maps.newHashMap(); final Set<String> combinedOutputNames = new HashSet<>();
combinedOutputNames.addAll(otherOutputNames);
final Map<String, AggregatorFactory> aggsFactoryMap = new HashMap<>();
for (AggregatorFactory aggFactory : aggFactories) { for (AggregatorFactory aggFactory : aggFactories) {
Preconditions.checkArgument(!aggsFactoryMap.containsKey(aggFactory.getName()), Preconditions.checkArgument(
"[%s] already defined", aggFactory.getName()); combinedOutputNames.add(aggFactory.getName()),
"[%s] already defined", aggFactory.getName()
);
aggsFactoryMap.put(aggFactory.getName(), aggFactory); aggsFactoryMap.put(aggFactory.getName(), aggFactory);
} }
if (postAggs != null && !postAggs.isEmpty()) { if (postAggs != null && !postAggs.isEmpty()) {
final Set<String> combinedAggNames = Sets.newHashSet(aggsFactoryMap.keySet());
List<PostAggregator> decorated = Lists.newArrayListWithExpectedSize(postAggs.size()); List<PostAggregator> decorated = Lists.newArrayListWithExpectedSize(postAggs.size());
for (final PostAggregator postAgg : postAggs) { for (final PostAggregator postAgg : postAggs) {
final Set<String> dependencies = postAgg.getDependentFields(); final Set<String> dependencies = postAgg.getDependentFields();
final Set<String> missing = Sets.difference(dependencies, combinedAggNames); final Set<String> missing = Sets.difference(dependencies, combinedOutputNames);
Preconditions.checkArgument( Preconditions.checkArgument(
missing.isEmpty(), missing.isEmpty(),
"Missing fields [%s] for postAggregator [%s]", missing, postAgg.getName() "Missing fields [%s] for postAggregator [%s]", missing, postAgg.getName()
); );
Preconditions.checkArgument(combinedAggNames.add(postAgg.getName()), Preconditions.checkArgument(
"[%s] already defined", postAgg.getName()); combinedOutputNames.add(postAgg.getName()),
"[%s] already defined", postAgg.getName()
);
decorated.add(postAgg.decorate(aggsFactoryMap)); decorated.add(postAgg.decorate(aggsFactoryMap));
} }

View File

@ -271,6 +271,7 @@ public class ArithmeticPostAggregator implements PostAggregator
public static enum Ordering implements Comparator<Double> { public static enum Ordering implements Comparator<Double> {
// ensures the following order: numeric > NaN > Infinite // ensures the following order: numeric > NaN > Infinite
// The name may be referenced via Ordering.valueOf(ordering) in the constructor.
numericFirst { numericFirst {
@Override @Override
public int compare(Double lhs, Double rhs) { public int compare(Double lhs, Double rhs) {

View File

@ -24,6 +24,7 @@ import com.fasterxml.jackson.annotation.JsonCreator;
import com.fasterxml.jackson.annotation.JsonProperty; import com.fasterxml.jackson.annotation.JsonProperty;
import com.google.common.base.Preconditions; import com.google.common.base.Preconditions;
import com.google.common.collect.ImmutableSet; import com.google.common.collect.ImmutableSet;
import io.druid.java.util.common.guava.Comparators;
import io.druid.math.expr.Expr; import io.druid.math.expr.Expr;
import io.druid.math.expr.ExprMacroTable; import io.druid.math.expr.ExprMacroTable;
import io.druid.math.expr.Parser; import io.druid.math.expr.Parser;
@ -36,25 +37,22 @@ import java.util.Map;
import java.util.Objects; import java.util.Objects;
import java.util.Set; import java.util.Set;
/**
*/
public class ExpressionPostAggregator implements PostAggregator public class ExpressionPostAggregator implements PostAggregator
{ {
private static final Comparator<Number> DEFAULT_COMPARATOR = new Comparator<Number>() private static final Comparator<Comparable> DEFAULT_COMPARATOR = Comparator.nullsFirst(
{ (Comparable o1, Comparable o2) -> {
@Override if (o1 instanceof Long && o2 instanceof Long) {
public int compare(Number o1, Number o2) return Long.compare((long) o1, (long) o2);
{ } else if (o1 instanceof Number && o2 instanceof Number) {
if (o1 instanceof Long && o2 instanceof Long) { return Double.compare(((Number) o1).doubleValue(), ((Number) o2).doubleValue());
return Long.compare(o1.longValue(), o2.longValue()); } else {
} return o1.compareTo(o2);
return Double.compare(o1.doubleValue(), o2.doubleValue()); }
} });
};
private final String name; private final String name;
private final String expression; private final String expression;
private final Comparator comparator; private final Comparator<Comparable> comparator;
private final String ordering; private final String ordering;
private final ExprMacroTable macroTable; private final ExprMacroTable macroTable;
@ -150,31 +148,29 @@ public class ExpressionPostAggregator implements PostAggregator
.build(); .build();
} }
public static enum Ordering implements Comparator<Number> public enum Ordering implements Comparator<Comparable>
{ {
// ensures the following order: numeric > NaN > Infinite // ensures the following order: numeric > NaN > Infinite
// The name may be referenced via Ordering.valueOf(ordering) in the constructor.
numericFirst { numericFirst {
@Override @Override
public int compare(Number lhs, Number rhs) public int compare(Comparable lhs, Comparable rhs)
{ {
if (lhs instanceof Long && rhs instanceof Long) { if (lhs instanceof Long && rhs instanceof Long) {
return Long.compare(lhs.longValue(), rhs.longValue()); return Long.compare(((Number) lhs).longValue(), ((Number) rhs).longValue());
} else if (lhs instanceof Number && rhs instanceof Number) {
double d1 = ((Number) lhs).doubleValue();
double d2 = ((Number) rhs).doubleValue();
if (Double.isFinite(d1) && !Double.isFinite(d2)) {
return 1;
}
if (!Double.isFinite(d1) && Double.isFinite(d2)) {
return -1;
}
return Double.compare(d1, d2);
} else {
return Comparators.<Comparable>naturalNullsFirst().compare(lhs, rhs);
} }
double d1 = lhs.doubleValue();
double d2 = rhs.doubleValue();
if (isFinite(d1) && !isFinite(d2)) {
return 1;
}
if (!isFinite(d1) && isFinite(d2)) {
return -1;
}
return Double.compare(d1, d2);
}
// Double.isFinite only exist in JDK8
private boolean isFinite(double value)
{
return !Double.isInfinite(value) && !Double.isNaN(value);
} }
} }
} }

View File

@ -75,6 +75,7 @@ import java.util.List;
import java.util.Map; import java.util.Map;
import java.util.Objects; import java.util.Objects;
import java.util.Set; import java.util.Set;
import java.util.stream.Collectors;
/** /**
*/ */
@ -182,6 +183,7 @@ public class GroupByQuery extends BaseQuery<Row>
} }
this.aggregatorSpecs = aggregatorSpecs == null ? ImmutableList.<AggregatorFactory>of() : aggregatorSpecs; this.aggregatorSpecs = aggregatorSpecs == null ? ImmutableList.<AggregatorFactory>of() : aggregatorSpecs;
this.postAggregatorSpecs = Queries.prepareAggregations( this.postAggregatorSpecs = Queries.prepareAggregations(
this.dimensions.stream().map(DimensionSpec::getOutputName).collect(Collectors.toList()),
this.aggregatorSpecs, this.aggregatorSpecs,
postAggregatorSpecs == null ? ImmutableList.<PostAggregator>of() : postAggregatorSpecs postAggregatorSpecs == null ? ImmutableList.<PostAggregator>of() : postAggregatorSpecs
); );

View File

@ -71,6 +71,7 @@ public class TimeseriesQuery extends BaseQuery<Result<TimeseriesResultValue>>
this.granularity = granularity; this.granularity = granularity;
this.aggregatorSpecs = aggregatorSpecs == null ? ImmutableList.of() : aggregatorSpecs; this.aggregatorSpecs = aggregatorSpecs == null ? ImmutableList.of() : aggregatorSpecs;
this.postAggregatorSpecs = Queries.prepareAggregations( this.postAggregatorSpecs = Queries.prepareAggregations(
ImmutableList.of(),
this.aggregatorSpecs, this.aggregatorSpecs,
postAggregatorSpecs == null ? ImmutableList.of() : postAggregatorSpecs postAggregatorSpecs == null ? ImmutableList.of() : postAggregatorSpecs
); );

View File

@ -55,7 +55,10 @@ public class NumericTopNMetricSpec implements TopNMetricSpec
{ {
Preconditions.checkNotNull(metric, "metric can't be null"); Preconditions.checkNotNull(metric, "metric can't be null");
Preconditions.checkNotNull(aggregatorSpecs, "aggregations cannot be null"); Preconditions.checkNotNull(aggregatorSpecs, "aggregations cannot be null");
Preconditions.checkArgument(aggregatorSpecs.size() > 0, "Must have at least one AggregatorFactory"); Preconditions.checkArgument(
aggregatorSpecs.size() > 0 || postAggregatorSpecs.size() > 0,
"Must have at least one AggregatorFactory or PostAggregator"
);
final AggregatorFactory aggregator = Iterables.tryFind( final AggregatorFactory aggregator = Iterables.tryFind(
aggregatorSpecs, aggregatorSpecs,

View File

@ -81,6 +81,7 @@ public class TopNQuery extends BaseQuery<Result<TopNResultValue>>
this.granularity = granularity; this.granularity = granularity;
this.aggregatorSpecs = aggregatorSpecs == null ? ImmutableList.<AggregatorFactory>of() : aggregatorSpecs; this.aggregatorSpecs = aggregatorSpecs == null ? ImmutableList.<AggregatorFactory>of() : aggregatorSpecs;
this.postAggregatorSpecs = Queries.prepareAggregations( this.postAggregatorSpecs = Queries.prepareAggregations(
ImmutableList.of(dimensionSpec.getOutputName()),
this.aggregatorSpecs, this.aggregatorSpecs,
postAggregatorSpecs == null postAggregatorSpecs == null
? ImmutableList.<PostAggregator>of() ? ImmutableList.<PostAggregator>of()

View File

@ -143,8 +143,9 @@ public class TopNQueryEngine
topNAlgorithm = new TimeExtractionTopNAlgorithm(capabilities, query); topNAlgorithm = new TimeExtractionTopNAlgorithm(capabilities, query);
} else if (selector.isHasExtractionFn()) { } else if (selector.isHasExtractionFn()) {
topNAlgorithm = new DimExtractionTopNAlgorithm(capabilities, query); topNAlgorithm = new DimExtractionTopNAlgorithm(capabilities, query);
} else if (columnCapabilities != null && columnCapabilities.getType() != ValueType.STRING) { } else if (columnCapabilities != null && !(columnCapabilities.getType() == ValueType.STRING
// force non-Strings to use DimExtraction for now, do a typed PooledTopN later && columnCapabilities.isDictionaryEncoded())) {
// Use DimExtraction for non-Strings and for non-dictionary-encoded Strings.
topNAlgorithm = new DimExtractionTopNAlgorithm(capabilities, query); topNAlgorithm = new DimExtractionTopNAlgorithm(capabilities, query);
} else if (selector.isAggregateAllMetrics()) { } else if (selector.isAggregateAllMetrics()) {
topNAlgorithm = new PooledTopNAlgorithm(capabilities, query, bufferPool); topNAlgorithm = new PooledTopNAlgorithm(capabilities, query, bufferPool);

View File

@ -249,11 +249,16 @@ public class TopNQueryQueryToolChest extends QueryToolChest<Result<TopNResultVal
+ 1 + 1
); );
// Put non-finalized aggregators before post-aggregators.
for (int i = 0; i < aggFactoryNames.length; ++i) { for (int i = 0; i < aggFactoryNames.length; ++i) {
final String name = aggFactoryNames[i]; final String name = aggFactoryNames[i];
values.put(name, input.getMetric(name)); values.put(name, input.getMetric(name));
} }
// Put dimension, post-aggregators might depend on it.
values.put(dimension, input.getDimensionValue(dimension));
// Put post-aggregators.
for (PostAggregator postAgg : postAggregators) { for (PostAggregator postAgg : postAggregators) {
Object calculatedPostAgg = input.getMetric(postAgg.getName()); Object calculatedPostAgg = input.getMetric(postAgg.getName());
if (calculatedPostAgg != null) { if (calculatedPostAgg != null) {
@ -262,13 +267,13 @@ public class TopNQueryQueryToolChest extends QueryToolChest<Result<TopNResultVal
values.put(postAgg.getName(), postAgg.compute(values)); values.put(postAgg.getName(), postAgg.compute(values));
} }
} }
// Put finalized aggregators now that post-aggregators are done.
for (int i = 0; i < aggFactoryNames.length; ++i) { for (int i = 0; i < aggFactoryNames.length; ++i) {
final String name = aggFactoryNames[i]; final String name = aggFactoryNames[i];
values.put(name, fn.manipulate(aggregatorFactories[i], input.getMetric(name))); values.put(name, fn.manipulate(aggregatorFactories[i], input.getMetric(name)));
} }
values.put(dimension, input.getDimensionValue(dimension));
return values; return values;
} }
} }

View File

@ -23,22 +23,22 @@ import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Preconditions; import com.google.common.base.Preconditions;
import com.google.common.base.Supplier; import com.google.common.base.Supplier;
import com.google.common.collect.Maps; import com.google.common.collect.Maps;
import com.google.common.primitives.Doubles;
import io.druid.common.guava.GuavaUtils;
import io.druid.math.expr.Expr; import io.druid.math.expr.Expr;
import io.druid.math.expr.ExprEval;
import io.druid.math.expr.Parser; import io.druid.math.expr.Parser;
import io.druid.query.dimension.DefaultDimensionSpec;
import io.druid.segment.ColumnSelectorFactory; import io.druid.segment.ColumnSelectorFactory;
import io.druid.segment.FloatColumnSelector; import io.druid.segment.DimensionSelector;
import io.druid.segment.LongColumnSelector;
import io.druid.segment.ObjectColumnSelector; import io.druid.segment.ObjectColumnSelector;
import io.druid.segment.column.ColumnCapabilities; import io.druid.segment.column.ColumnCapabilities;
import io.druid.segment.column.ValueType; import io.druid.segment.column.ValueType;
import io.druid.segment.data.IndexedInts;
import javax.annotation.Nonnull; import javax.annotation.Nonnull;
import javax.annotation.Nullable; import javax.annotation.Nullable;
import java.util.Map; import java.util.Map;
public class ExpressionObjectSelector implements ObjectColumnSelector<Number> public class ExpressionObjectSelector implements ObjectColumnSelector<ExprEval>
{ {
private final Expr expression; private final Expr expression;
private final Expr.ObjectBinding bindings; private final Expr.ObjectBinding bindings;
@ -49,28 +49,32 @@ public class ExpressionObjectSelector implements ObjectColumnSelector<Number>
this.expression = Preconditions.checkNotNull(expression, "expression"); this.expression = Preconditions.checkNotNull(expression, "expression");
} }
public static ExpressionObjectSelector from(ColumnSelectorFactory columnSelectorFactory, Expr expression) static ExpressionObjectSelector from(ColumnSelectorFactory columnSelectorFactory, Expr expression)
{ {
return new ExpressionObjectSelector(createBindings(columnSelectorFactory, expression), expression); return new ExpressionObjectSelector(createBindings(columnSelectorFactory, expression), expression);
} }
private static Expr.ObjectBinding createBindings(ColumnSelectorFactory columnSelectorFactory, Expr expression) private static Expr.ObjectBinding createBindings(ColumnSelectorFactory columnSelectorFactory, Expr expression)
{ {
final Map<String, Supplier<Number>> suppliers = Maps.newHashMap(); final Map<String, Supplier<Object>> suppliers = Maps.newHashMap();
for (String columnName : Parser.findRequiredBindings(expression)) { for (String columnName : Parser.findRequiredBindings(expression)) {
final ColumnCapabilities columnCapabilities = columnSelectorFactory.getColumnCapabilities(columnName); final ColumnCapabilities columnCapabilities = columnSelectorFactory.getColumnCapabilities(columnName);
final ValueType nativeType = columnCapabilities != null ? columnCapabilities.getType() : null; final ValueType nativeType = columnCapabilities != null ? columnCapabilities.getType() : null;
final Supplier<Number> supplier; final Supplier<Object> supplier;
if (nativeType == ValueType.FLOAT) { if (nativeType == ValueType.FLOAT) {
supplier = supplierFromFloatSelector(columnSelectorFactory.makeFloatColumnSelector(columnName)); supplier = columnSelectorFactory.makeFloatColumnSelector(columnName)::get;
} else if (nativeType == ValueType.LONG) { } else if (nativeType == ValueType.LONG) {
supplier = supplierFromLongSelector(columnSelectorFactory.makeLongColumnSelector(columnName)); supplier = columnSelectorFactory.makeLongColumnSelector(columnName)::get;
} else if (nativeType == ValueType.STRING) {
supplier = supplierFromDimensionSelector(
columnSelectorFactory.makeDimensionSelector(new DefaultDimensionSpec(columnName, columnName))
);
} else if (nativeType == null) { } else if (nativeType == null) {
// Unknown ValueType. Try making an Object selector and see if that gives us anything useful. // Unknown ValueType. Try making an Object selector and see if that gives us anything useful.
supplier = supplierFromObjectSelector(columnSelectorFactory.makeObjectColumnSelector(columnName)); supplier = supplierFromObjectSelector(columnSelectorFactory.makeObjectColumnSelector(columnName));
} else { } else {
// Unhandleable ValueType (possibly STRING or COMPLEX). // Unhandleable ValueType (COMPLEX).
supplier = null; supplier = null;
} }
@ -84,91 +88,61 @@ public class ExpressionObjectSelector implements ObjectColumnSelector<Number>
@VisibleForTesting @VisibleForTesting
@Nonnull @Nonnull
static Supplier<Number> supplierFromFloatSelector(final FloatColumnSelector selector) static Supplier<Object> supplierFromDimensionSelector(final DimensionSelector selector)
{ {
Preconditions.checkNotNull(selector, "selector"); Preconditions.checkNotNull(selector, "selector");
return new Supplier<Number>() return () -> {
{ final IndexedInts row = selector.getRow();
@Override if (row.size() == 0) {
public Number get() // Treat empty multi-value rows as nulls.
{ return null;
return selector.get(); } else if (row.size() == 1) {
} return selector.lookupName(row.get(0));
}; } else {
} // Can't handle multi-value rows in expressions.
// Treat them as nulls until we think of something better to do.
@VisibleForTesting return null;
@Nonnull
static Supplier<Number> supplierFromLongSelector(final LongColumnSelector selector)
{
Preconditions.checkNotNull(selector, "selector");
return new Supplier<Number>()
{
@Override
public Number get()
{
return selector.get();
} }
}; };
} }
@VisibleForTesting @VisibleForTesting
@Nullable @Nullable
static Supplier<Number> supplierFromObjectSelector(final ObjectColumnSelector selector) static Supplier<Object> supplierFromObjectSelector(final ObjectColumnSelector selector)
{ {
final Class<?> clazz = selector == null ? null : selector.classOfObject(); if (selector == null) {
if (selector != null && (clazz.isAssignableFrom(Number.class) return null;
|| clazz.isAssignableFrom(String.class) }
|| Number.class.isAssignableFrom(clazz))) {
// There may be numbers here. final Class<?> clazz = selector.classOfObject();
return new Supplier<Number>() if (Number.class.isAssignableFrom(clazz) || String.class.isAssignableFrom(clazz)) {
{ // Number, String supported as-is.
@Override return selector::get;
public Number get() } else if (clazz.isAssignableFrom(Number.class) || clazz.isAssignableFrom(String.class)) {
{ // Might be Numbers and Strings. Use a selector that double-checks.
return tryParse(selector.get()); return () -> {
final Object val = selector.get();
if (val instanceof Number || val instanceof String) {
return val;
} else {
return null;
} }
}; };
} else { } else {
// We know there are no numbers here. Use a null supplier. // No numbers or strings.
return null; return null;
} }
} }
@Nullable @Override
private static Number tryParse(final Object value) public Class<ExprEval> classOfObject()
{ {
if (value == null) { return ExprEval.class;
return null;
}
if (value instanceof Number) {
return (Number) value;
}
final String stringValue = String.valueOf(value);
final Long longValue = GuavaUtils.tryParseLong(stringValue);
if (longValue != null) {
return longValue;
}
final Double doubleValue = Doubles.tryParse(stringValue);
if (doubleValue != null) {
return doubleValue;
}
return null;
} }
@Override @Override
public Class<Number> classOfObject() public ExprEval get()
{ {
return Number.class; return expression.eval(bindings);
}
@Override
public Number get()
{
return expression.eval(bindings).numericValue();
} }
} }

View File

@ -19,7 +19,9 @@
package io.druid.segment.virtual; package io.druid.segment.virtual;
import com.google.common.base.Strings;
import io.druid.math.expr.Expr; import io.druid.math.expr.Expr;
import io.druid.math.expr.ExprEval;
import io.druid.query.extraction.ExtractionFn; import io.druid.query.extraction.ExtractionFn;
import io.druid.query.monomorphicprocessing.RuntimeShapeInspector; import io.druid.query.monomorphicprocessing.RuntimeShapeInspector;
import io.druid.segment.ColumnSelectorFactory; import io.druid.segment.ColumnSelectorFactory;
@ -54,8 +56,8 @@ public class ExpressionSelectors
@Override @Override
public long get() public long get()
{ {
final Number number = baseSelector.get(); final ExprEval exprEval = baseSelector.get();
return number != null ? number.longValue() : nullValue; return exprEval.isNull() ? nullValue : exprEval.asLong();
} }
@Override @Override
@ -79,8 +81,8 @@ public class ExpressionSelectors
@Override @Override
public float get() public float get()
{ {
final Number number = baseSelector.get(); final ExprEval exprEval = baseSelector.get();
return number != null ? number.floatValue() : nullValue; return exprEval.isNull() ? nullValue : (float) exprEval.asDouble();
} }
@Override @Override
@ -106,8 +108,7 @@ public class ExpressionSelectors
@Override @Override
protected String getValue() protected String getValue()
{ {
final Number number = baseSelector.get(); return Strings.emptyToNull(baseSelector.get().asString());
return number == null ? null : String.valueOf(number);
} }
@Override @Override
@ -123,7 +124,7 @@ public class ExpressionSelectors
@Override @Override
protected String getValue() protected String getValue()
{ {
return extractionFn.apply(baseSelector.get()); return extractionFn.apply(Strings.emptyToNull(baseSelector.get().asString()));
} }
@Override @Override

View File

@ -43,21 +43,22 @@ import java.util.Objects;
public class ExpressionVirtualColumn implements VirtualColumn public class ExpressionVirtualColumn implements VirtualColumn
{ {
private static final ColumnCapabilities CAPABILITIES = new ColumnCapabilitiesImpl().setType(ValueType.FLOAT);
private final String name; private final String name;
private final String expression; private final String expression;
private final ValueType outputType;
private final Expr parsedExpression; private final Expr parsedExpression;
@JsonCreator @JsonCreator
public ExpressionVirtualColumn( public ExpressionVirtualColumn(
@JsonProperty("name") String name, @JsonProperty("name") String name,
@JsonProperty("expression") String expression, @JsonProperty("expression") String expression,
@JsonProperty("outputType") ValueType outputType,
@JacksonInject ExprMacroTable macroTable @JacksonInject ExprMacroTable macroTable
) )
{ {
this.name = Preconditions.checkNotNull(name, "name"); this.name = Preconditions.checkNotNull(name, "name");
this.expression = Preconditions.checkNotNull(expression, "expression"); this.expression = Preconditions.checkNotNull(expression, "expression");
this.outputType = outputType != null ? outputType : ValueType.FLOAT;
this.parsedExpression = Parser.parse(expression, macroTable); this.parsedExpression = Parser.parse(expression, macroTable);
} }
@ -74,6 +75,12 @@ public class ExpressionVirtualColumn implements VirtualColumn
return expression; return expression;
} }
@JsonProperty
public ValueType getOutputType()
{
return outputType;
}
@Override @Override
public ObjectColumnSelector makeObjectColumnSelector( public ObjectColumnSelector makeObjectColumnSelector(
final String columnName, final String columnName,
@ -95,7 +102,7 @@ public class ExpressionVirtualColumn implements VirtualColumn
@Override @Override
public Object get() public Object get()
{ {
return baseSelector.get(); return baseSelector.get().value();
} }
}; };
} }
@ -136,7 +143,7 @@ public class ExpressionVirtualColumn implements VirtualColumn
@Override @Override
public ColumnCapabilities capabilities(String columnName) public ColumnCapabilities capabilities(String columnName)
{ {
return CAPABILITIES; return new ColumnCapabilitiesImpl().setType(outputType);
} }
@Override @Override
@ -157,6 +164,7 @@ public class ExpressionVirtualColumn implements VirtualColumn
return new CacheKeyBuilder(VirtualColumnCacheHelper.CACHE_TYPE_ID_EXPRESSION) return new CacheKeyBuilder(VirtualColumnCacheHelper.CACHE_TYPE_ID_EXPRESSION)
.appendString(name) .appendString(name)
.appendString(expression) .appendString(expression)
.appendString(outputType.toString())
.build(); .build();
} }
@ -171,13 +179,14 @@ public class ExpressionVirtualColumn implements VirtualColumn
} }
final ExpressionVirtualColumn that = (ExpressionVirtualColumn) o; final ExpressionVirtualColumn that = (ExpressionVirtualColumn) o;
return Objects.equals(name, that.name) && return Objects.equals(name, that.name) &&
Objects.equals(expression, that.expression); Objects.equals(expression, that.expression) &&
outputType == that.outputType;
} }
@Override @Override
public int hashCode() public int hashCode()
{ {
return Objects.hash(name, expression); return Objects.hash(name, expression, outputType);
} }
@Override @Override
@ -186,7 +195,7 @@ public class ExpressionVirtualColumn implements VirtualColumn
return "ExpressionVirtualColumn{" + return "ExpressionVirtualColumn{" +
"name='" + name + '\'' + "name='" + name + '\'' +
", expression='" + expression + '\'' + ", expression='" + expression + '\'' +
", parsedExpression=" + parsedExpression + ", outputType=" + outputType +
'}'; '}';
} }
} }

View File

@ -19,6 +19,7 @@
package io.druid.query; package io.druid.query;
import com.google.common.collect.ImmutableList;
import io.druid.query.aggregation.AggregatorFactory; import io.druid.query.aggregation.AggregatorFactory;
import io.druid.query.aggregation.CountAggregatorFactory; import io.druid.query.aggregation.CountAggregatorFactory;
import io.druid.query.aggregation.DoubleSumAggregatorFactory; import io.druid.query.aggregation.DoubleSumAggregatorFactory;
@ -59,7 +60,7 @@ public class QueriesTest
boolean exceptionOccured = false; boolean exceptionOccured = false;
try { try {
Queries.prepareAggregations(aggFactories, postAggs); Queries.prepareAggregations(ImmutableList.of(), aggFactories, postAggs);
} }
catch (IllegalArgumentException e) { catch (IllegalArgumentException e) {
exceptionOccured = true; exceptionOccured = true;
@ -91,7 +92,7 @@ public class QueriesTest
boolean exceptionOccured = false; boolean exceptionOccured = false;
try { try {
Queries.prepareAggregations(aggFactories, postAggs); Queries.prepareAggregations(ImmutableList.of(), aggFactories, postAggs);
} }
catch (IllegalArgumentException e) { catch (IllegalArgumentException e) {
exceptionOccured = true; exceptionOccured = true;
@ -145,7 +146,7 @@ public class QueriesTest
boolean exceptionOccured = false; boolean exceptionOccured = false;
try { try {
Queries.prepareAggregations(aggFactories, postAggs); Queries.prepareAggregations(ImmutableList.of(), aggFactories, postAggs);
} }
catch (IllegalArgumentException e) { catch (IllegalArgumentException e) {
exceptionOccured = true; exceptionOccured = true;
@ -199,7 +200,7 @@ public class QueriesTest
boolean exceptionOccured = false; boolean exceptionOccured = false;
try { try {
Queries.prepareAggregations(aggFactories, postAggs); Queries.prepareAggregations(ImmutableList.of(), aggFactories, postAggs);
} }
catch (IllegalArgumentException e) { catch (IllegalArgumentException e) {
exceptionOccured = true; exceptionOccured = true;

View File

@ -269,8 +269,9 @@ public class SchemaEvolutionTest
.build(); .build();
// Only string(1) // Only string(1)
// Note: Expressions implicitly cast strings to numbers, leading to the a/b vs c/d difference.
Assert.assertEquals( Assert.assertEquals(
timeseriesResult(ImmutableMap.of("a", 0L, "b", 0.0, "c", 0L, "d", 0.0)), timeseriesResult(ImmutableMap.of("a", 0L, "b", 0.0, "c", 31L, "d", THIRTY_ONE_POINT_ONE)),
runQuery(query, factory, ImmutableList.of(index1)) runQuery(query, factory, ImmutableList.of(index1))
); );
@ -293,12 +294,13 @@ public class SchemaEvolutionTest
); );
// string(1) + long(2) + float(3) + nonexistent(4) // string(1) + long(2) + float(3) + nonexistent(4)
// Note: Expressions implicitly cast strings to numbers, leading to the a/b vs c/d difference.
Assert.assertEquals( Assert.assertEquals(
timeseriesResult(ImmutableMap.of( timeseriesResult(ImmutableMap.of(
"a", 31L * 2, "a", 31L * 2,
"b", THIRTY_ONE_POINT_ONE + 31, "b", THIRTY_ONE_POINT_ONE + 31,
"c", 31L * 2, "c", 31L * 3,
"d", THIRTY_ONE_POINT_ONE + 31 "d", THIRTY_ONE_POINT_ONE * 2 + 31
)), )),
runQuery(query, factory, ImmutableList.of(index1, index2, index3, index4)) runQuery(query, factory, ImmutableList.of(index1, index2, index3, index4))
); );

View File

@ -446,6 +446,134 @@ public class GroupByQueryRunnerTest
TestHelper.assertExpectedObjects(expectedResults, results, ""); TestHelper.assertExpectedObjects(expectedResults, results, "");
} }
@Test
public void testGroupByWithStringPostAggregator()
{
GroupByQuery query = GroupByQuery
.builder()
.setDataSource(QueryRunnerTestHelper.dataSource)
.setQuerySegmentSpec(QueryRunnerTestHelper.firstToThird)
.setDimensions(Lists.<DimensionSpec>newArrayList(new DefaultDimensionSpec("quality", "alias")))
.setAggregatorSpecs(
Arrays.asList(
QueryRunnerTestHelper.rowsCount,
new LongSumAggregatorFactory("idx", "index")
)
)
.setPostAggregatorSpecs(
ImmutableList.of(
new ExpressionPostAggregator("post", "alias + 'x'", null, TestExprMacroTable.INSTANCE)
)
)
.setGranularity(QueryRunnerTestHelper.dayGran)
.setLimitSpec(
new DefaultLimitSpec(
ImmutableList.of(
new OrderByColumnSpec("post", OrderByColumnSpec.Direction.DESCENDING)
),
Integer.MAX_VALUE
)
)
.build();
List<Row> expectedResults = Arrays.asList(
GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-01", "alias", "travel", "post", "travelx", "rows", 1L, "idx", 119L),
GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-01", "alias", "technology", "post", "technologyx", "rows", 1L, "idx", 78L),
GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-01", "alias", "premium", "post", "premiumx", "rows", 3L, "idx", 2900L),
GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-01", "alias", "news", "post", "newsx", "rows", 1L, "idx", 121L),
GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-01", "alias", "mezzanine", "post", "mezzaninex", "rows", 3L, "idx", 2870L),
GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-01", "alias", "health", "post", "healthx", "rows", 1L, "idx", 120L),
GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-01", "alias", "entertainment", "post", "entertainmentx", "rows", 1L, "idx", 158L),
GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-01", "alias", "business", "post", "businessx", "rows", 1L, "idx", 118L),
GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-01", "alias", "automotive", "post", "automotivex", "rows", 1L, "idx", 135L),
GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-02", "alias", "travel", "post", "travelx", "rows", 1L, "idx", 126L),
GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-02", "alias", "technology", "post", "technologyx", "rows", 1L, "idx", 97L),
GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-02", "alias", "premium", "post", "premiumx", "rows", 3L, "idx", 2505L),
GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-02", "alias", "news", "post", "newsx", "rows", 1L, "idx", 114L),
GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-02", "alias", "mezzanine", "post", "mezzaninex", "rows", 3L, "idx", 2447L),
GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-02", "alias", "health", "post", "healthx", "rows", 1L, "idx", 113L),
GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-02", "alias", "entertainment", "post", "entertainmentx", "rows", 1L, "idx", 166L),
GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-02", "alias", "business", "post", "businessx", "rows", 1L, "idx", 112L),
GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-02", "alias", "automotive", "post", "automotivex", "rows", 1L, "idx", 147L)
);
Iterable<Row> results = GroupByQueryRunnerTestHelper.runQuery(factory, runner, query);
TestHelper.assertExpectedObjects(expectedResults, results, "");
}
@Test
public void testGroupByWithStringVirtualColumn()
{
GroupByQuery query = GroupByQuery
.builder()
.setDataSource(QueryRunnerTestHelper.dataSource)
.setQuerySegmentSpec(QueryRunnerTestHelper.firstToThird)
.setVirtualColumns(
new ExpressionVirtualColumn(
"vc",
"quality + 'x'",
ValueType.STRING,
TestExprMacroTable.INSTANCE
)
)
.setDimensions(Lists.<DimensionSpec>newArrayList(new DefaultDimensionSpec("vc", "alias")))
.setAggregatorSpecs(
Arrays.asList(
QueryRunnerTestHelper.rowsCount,
new LongSumAggregatorFactory("idx", "index")
)
)
.setGranularity(QueryRunnerTestHelper.dayGran)
.build();
if (config.getDefaultStrategy().equals(GroupByStrategySelector.STRATEGY_V1)) {
expectedException.expect(UnsupportedOperationException.class);
expectedException.expectMessage("GroupBy v1 does not support dimension selectors with unknown cardinality.");
}
List<Row> expectedResults = Arrays.asList(
GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-01", "alias", "automotivex", "rows", 1L, "idx", 135L),
GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-01", "alias", "businessx", "rows", 1L, "idx", 118L),
GroupByQueryRunnerTestHelper.createExpectedRow(
"2011-04-01",
"alias",
"entertainmentx",
"rows",
1L,
"idx",
158L
),
GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-01", "alias", "healthx", "rows", 1L, "idx", 120L),
GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-01", "alias", "mezzaninex", "rows", 3L, "idx", 2870L),
GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-01", "alias", "newsx", "rows", 1L, "idx", 121L),
GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-01", "alias", "premiumx", "rows", 3L, "idx", 2900L),
GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-01", "alias", "technologyx", "rows", 1L, "idx", 78L),
GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-01", "alias", "travelx", "rows", 1L, "idx", 119L),
GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-02", "alias", "automotivex", "rows", 1L, "idx", 147L),
GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-02", "alias", "businessx", "rows", 1L, "idx", 112L),
GroupByQueryRunnerTestHelper.createExpectedRow(
"2011-04-02",
"alias",
"entertainmentx",
"rows",
1L,
"idx",
166L
),
GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-02", "alias", "healthx", "rows", 1L, "idx", 113L),
GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-02", "alias", "mezzaninex", "rows", 3L, "idx", 2447L),
GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-02", "alias", "newsx", "rows", 1L, "idx", 114L),
GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-02", "alias", "premiumx", "rows", 3L, "idx", 2505L),
GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-02", "alias", "technologyx", "rows", 1L, "idx", 97L),
GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-02", "alias", "travelx", "rows", 1L, "idx", 126L)
);
Iterable<Row> results = GroupByQueryRunnerTestHelper.runQuery(factory, runner, query);
TestHelper.assertExpectedObjects(expectedResults, results, "");
}
@Test @Test
public void testGroupByWithDurationGranularity() public void testGroupByWithDurationGranularity()
{ {
@ -493,7 +621,7 @@ public class GroupByQueryRunnerTest
public void testGroupByWithOutputNameCollisions() public void testGroupByWithOutputNameCollisions()
{ {
expectedException.expect(IllegalArgumentException.class); expectedException.expect(IllegalArgumentException.class);
expectedException.expectMessage("Duplicate output name[alias]"); expectedException.expectMessage("[alias] already defined");
GroupByQuery query = GroupByQuery GroupByQuery query = GroupByQuery
.builder() .builder()
@ -2508,6 +2636,7 @@ public class GroupByQueryRunnerTest
new ExpressionVirtualColumn( new ExpressionVirtualColumn(
"expr", "expr",
"index * 2 + indexMin / 10", "index * 2 + indexMin / 10",
ValueType.FLOAT,
TestExprMacroTable.INSTANCE TestExprMacroTable.INSTANCE
) )
) )
@ -2747,7 +2876,7 @@ public class GroupByQueryRunnerTest
// Now try it with an expression virtual column. // Now try it with an expression virtual column.
builder.setLimit(Integer.MAX_VALUE) builder.setLimit(Integer.MAX_VALUE)
.setVirtualColumns( .setVirtualColumns(
new ExpressionVirtualColumn("expr", "index / 2 + indexMin", TestExprMacroTable.INSTANCE) new ExpressionVirtualColumn("expr", "index / 2 + indexMin", ValueType.FLOAT, TestExprMacroTable.INSTANCE)
) )
.setAggregatorSpecs( .setAggregatorSpecs(
Arrays.asList( Arrays.asList(
@ -4337,7 +4466,7 @@ public class GroupByQueryRunnerTest
subquery = new GroupByQuery.Builder(subquery) subquery = new GroupByQuery.Builder(subquery)
.setVirtualColumns( .setVirtualColumns(
new ExpressionVirtualColumn("expr", "-index + 100", TestExprMacroTable.INSTANCE) new ExpressionVirtualColumn("expr", "-index + 100", ValueType.FLOAT, TestExprMacroTable.INSTANCE)
) )
.setAggregatorSpecs( .setAggregatorSpecs(
Arrays.asList( Arrays.asList(
@ -5439,7 +5568,7 @@ public class GroupByQueryRunnerTest
.builder() .builder()
.setDataSource(subquery) .setDataSource(subquery)
.setQuerySegmentSpec(QueryRunnerTestHelper.firstToThird) .setQuerySegmentSpec(QueryRunnerTestHelper.firstToThird)
.setVirtualColumns(new ExpressionVirtualColumn("expr", "1", TestExprMacroTable.INSTANCE)) .setVirtualColumns(new ExpressionVirtualColumn("expr", "1", ValueType.FLOAT, TestExprMacroTable.INSTANCE))
.setDimensions(Lists.<DimensionSpec>newArrayList()) .setDimensions(Lists.<DimensionSpec>newArrayList())
.setAggregatorSpecs(ImmutableList.<AggregatorFactory>of(new LongSumAggregatorFactory("count", "expr"))) .setAggregatorSpecs(ImmutableList.<AggregatorFactory>of(new LongSumAggregatorFactory("count", "expr")))
.setGranularity(QueryRunnerTestHelper.allGran) .setGranularity(QueryRunnerTestHelper.allGran)
@ -8612,7 +8741,7 @@ public class GroupByQueryRunnerTest
GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-02", "alias", "travel", "rows", 2L, "idx", 243L) GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-02", "alias", "travel", "rows", 2L, "idx", 243L)
); );
Iterable<Row> results = Sequences.toList(mergedRunner.run(allGranQuery, context), Lists.newArrayList()); Iterable<Row> results = Sequences.toList(mergedRunner.run(allGranQuery, context), Lists.<Row>newArrayList());
TestHelper.assertExpectedObjects(allGranExpectedResults, results, "merged"); TestHelper.assertExpectedObjects(allGranExpectedResults, results, "merged");
} }
@ -8706,7 +8835,7 @@ public class GroupByQueryRunnerTest
GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-02", "alias", "premium", "market", "spot", "rows", 2L, "idx", 257L) GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-02", "alias", "premium", "market", "spot", "rows", 2L, "idx", 257L)
); );
Iterable<Row> results = Sequences.toList(mergedRunner.run(allGranQuery, context), Lists.newArrayList()); Iterable<Row> results = Sequences.toList(mergedRunner.run(allGranQuery, context), Lists.<Row>newArrayList());
TestHelper.assertExpectedObjects(allGranExpectedResults, results, "merged"); TestHelper.assertExpectedObjects(allGranExpectedResults, results, "merged");
} }
@ -8804,7 +8933,7 @@ public class GroupByQueryRunnerTest
GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-02", "alias", "premium", "market", "spot", "rows", 2L, "idx", 257L) GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-02", "alias", "premium", "market", "spot", "rows", 2L, "idx", 257L)
); );
Iterable<Row> results = Sequences.toList(mergedRunner.run(allGranQuery, context), Lists.newArrayList()); Iterable<Row> results = Sequences.toList(mergedRunner.run(allGranQuery, context), Lists.<Row>newArrayList());
TestHelper.assertExpectedObjects(allGranExpectedResults, results, "merged"); TestHelper.assertExpectedObjects(allGranExpectedResults, results, "merged");
} }

View File

@ -510,7 +510,7 @@ public class SelectQueryRunnerTest
.metrics(Lists.<String>newArrayList(QueryRunnerTestHelper.indexMetric)) .metrics(Lists.<String>newArrayList(QueryRunnerTestHelper.indexMetric))
.pagingSpec(new PagingSpec(null, 10, true)) .pagingSpec(new PagingSpec(null, 10, true))
.virtualColumns( .virtualColumns(
new ExpressionVirtualColumn("expr", "index / 10.0", TestExprMacroTable.INSTANCE) new ExpressionVirtualColumn("expr", "index / 10.0", ValueType.FLOAT, TestExprMacroTable.INSTANCE)
) )
.build(); .build();

View File

@ -55,6 +55,7 @@ import io.druid.query.lookup.LookupExtractionFn;
import io.druid.query.ordering.StringComparators; import io.druid.query.ordering.StringComparators;
import io.druid.query.spec.MultipleIntervalSegmentSpec; import io.druid.query.spec.MultipleIntervalSegmentSpec;
import io.druid.segment.TestHelper; import io.druid.segment.TestHelper;
import io.druid.segment.column.ValueType;
import io.druid.segment.virtual.ExpressionVirtualColumn; import io.druid.segment.virtual.ExpressionVirtualColumn;
import org.joda.time.DateTime; import org.joda.time.DateTime;
import org.joda.time.DateTimeZone; import org.joda.time.DateTimeZone;
@ -421,6 +422,7 @@ public class TimeseriesQueryRunnerTest
new ExpressionVirtualColumn( new ExpressionVirtualColumn(
"expr", "expr",
"index", "index",
ValueType.FLOAT,
TestExprMacroTable.INSTANCE TestExprMacroTable.INSTANCE
) )
) )

View File

@ -62,10 +62,12 @@ import io.druid.query.aggregation.first.LongFirstAggregatorFactory;
import io.druid.query.aggregation.hyperloglog.HyperUniqueFinalizingPostAggregator; import io.druid.query.aggregation.hyperloglog.HyperUniqueFinalizingPostAggregator;
import io.druid.query.aggregation.hyperloglog.HyperUniquesAggregatorFactory; import io.druid.query.aggregation.hyperloglog.HyperUniquesAggregatorFactory;
import io.druid.query.aggregation.last.LongLastAggregatorFactory; import io.druid.query.aggregation.last.LongLastAggregatorFactory;
import io.druid.query.aggregation.post.ExpressionPostAggregator;
import io.druid.query.dimension.DefaultDimensionSpec; import io.druid.query.dimension.DefaultDimensionSpec;
import io.druid.query.dimension.DimensionSpec; import io.druid.query.dimension.DimensionSpec;
import io.druid.query.dimension.ExtractionDimensionSpec; import io.druid.query.dimension.ExtractionDimensionSpec;
import io.druid.query.dimension.ListFilteredDimensionSpec; import io.druid.query.dimension.ListFilteredDimensionSpec;
import io.druid.query.expression.TestExprMacroTable;
import io.druid.query.extraction.DimExtractionFn; import io.druid.query.extraction.DimExtractionFn;
import io.druid.query.extraction.ExtractionFn; import io.druid.query.extraction.ExtractionFn;
import io.druid.query.extraction.JavaScriptExtractionFn; import io.druid.query.extraction.JavaScriptExtractionFn;
@ -416,6 +418,78 @@ public class TopNQueryRunnerTest
assertExpectedResults(expectedResults, query); assertExpectedResults(expectedResults, query);
} }
@Test
public void testFullOnTopNOverPostAggsOnDimension()
{
TopNQuery query = new TopNQueryBuilder()
.dataSource(QueryRunnerTestHelper.dataSource)
.granularity(QueryRunnerTestHelper.allGran)
.dimension(QueryRunnerTestHelper.marketDimension)
.metric("dimPostAgg")
.threshold(4)
.intervals(QueryRunnerTestHelper.fullOnInterval)
.aggregators(
Lists.<AggregatorFactory>newArrayList(
Iterables.concat(
QueryRunnerTestHelper.commonAggregators,
Lists.newArrayList(
new DoubleMaxAggregatorFactory("maxIndex", "index"),
new DoubleMinAggregatorFactory("minIndex", "index")
)
)
)
)
.postAggregators(
ImmutableList.of(
new ExpressionPostAggregator(
"dimPostAgg",
"market + 'x'",
null,
TestExprMacroTable.INSTANCE
)
)
)
.build();
List<Result<TopNResultValue>> expectedResults = Arrays.asList(
new Result<TopNResultValue>(
new DateTime("2011-01-12T00:00:00.000Z"),
new TopNResultValue(
Arrays.<Map<String, Object>>asList(
ImmutableMap.<String, Object>builder()
.put(QueryRunnerTestHelper.marketDimension, "upfront")
.put("dimPostAgg", "upfrontx")
.put("rows", 186L)
.put("index", 192046.1060180664D)
.put("uniques", QueryRunnerTestHelper.UNIQUES_2)
.put("maxIndex", 1870.06103515625D)
.put("minIndex", 545.9906005859375D)
.build(),
ImmutableMap.<String, Object>builder()
.put(QueryRunnerTestHelper.marketDimension, "total_market")
.put("dimPostAgg", "total_marketx")
.put("rows", 186L)
.put("index", 215679.82879638672D)
.put("uniques", QueryRunnerTestHelper.UNIQUES_2)
.put("maxIndex", 1743.9217529296875D)
.put("minIndex", 792.3260498046875D)
.build(),
ImmutableMap.<String, Object>builder()
.put(QueryRunnerTestHelper.marketDimension, "spot")
.put("dimPostAgg", "spotx")
.put("rows", 837L)
.put("index", 95606.57232284546D)
.put("uniques", QueryRunnerTestHelper.UNIQUES_9)
.put("maxIndex", 277.2735290527344D)
.put("minIndex", 59.02102279663086D)
.build()
)
)
)
);
assertExpectedResults(expectedResults, query);
}
@Test @Test
public void testFullOnTopNOverUniques() public void testFullOnTopNOverUniques()
{ {
@ -4211,7 +4285,7 @@ public class TopNQueryRunnerTest
) )
) )
.postAggregators(Arrays.<PostAggregator>asList(QueryRunnerTestHelper.addRowsIndexConstant)) .postAggregators(Arrays.<PostAggregator>asList(QueryRunnerTestHelper.addRowsIndexConstant))
.virtualColumns(new ExpressionVirtualColumn("ql_expr", "qualityLong", ExprMacroTable.nil())) .virtualColumns(new ExpressionVirtualColumn("ql_expr", "qualityLong", ValueType.LONG, ExprMacroTable.nil()))
.build(); .build();
List<Result<TopNResultValue>> expectedResults = Arrays.asList( List<Result<TopNResultValue>> expectedResults = Arrays.asList(
@ -4262,6 +4336,61 @@ public class TopNQueryRunnerTest
assertExpectedResults(expectedResults, query); assertExpectedResults(expectedResults, query);
} }
@Test
public void testTopNStringVirtualColumn()
{
TopNQuery query = new TopNQueryBuilder()
.dataSource(QueryRunnerTestHelper.dataSource)
.granularity(QueryRunnerTestHelper.allGran)
.virtualColumns(
new ExpressionVirtualColumn(
"vc",
"market + ' ' + market",
ValueType.STRING,
TestExprMacroTable.INSTANCE
)
)
.dimension("vc")
.metric("rows")
.threshold(4)
.intervals(QueryRunnerTestHelper.firstToThird)
.aggregators(QueryRunnerTestHelper.commonAggregators)
.postAggregators(Arrays.<PostAggregator>asList(QueryRunnerTestHelper.addRowsIndexConstant))
.build();
List<Result<TopNResultValue>> expectedResults = Arrays.asList(
new Result<>(
new DateTime("2011-04-01T00:00:00.000Z"),
new TopNResultValue(
Arrays.<Map<String, Object>>asList(
ImmutableMap.<String, Object>of(
"vc", "spot spot",
"rows", 18L,
"index", 2231.8768157958984D,
"addRowsIndexConstant", 2250.8768157958984D,
"uniques", QueryRunnerTestHelper.UNIQUES_9
),
ImmutableMap.<String, Object>of(
"vc", "total_market total_market",
"rows", 4L,
"index", 5351.814697265625D,
"addRowsIndexConstant", 5356.814697265625D,
"uniques", QueryRunnerTestHelper.UNIQUES_2
),
ImmutableMap.<String, Object>of(
"vc", "upfront upfront",
"rows", 4L,
"index", 4875.669677734375D,
"addRowsIndexConstant", 4880.669677734375D,
"uniques", QueryRunnerTestHelper.UNIQUES_2
)
)
)
)
);
assertExpectedResults(expectedResults, query);
}
@Test @Test
public void testFullOnTopNLongColumnWithExFn() public void testFullOnTopNLongColumnWithExFn()
{ {

View File

@ -42,6 +42,7 @@ import io.druid.query.aggregation.DoubleSumAggregatorFactory;
import io.druid.query.aggregation.hyperloglog.HyperUniquesAggregatorFactory; import io.druid.query.aggregation.hyperloglog.HyperUniquesAggregatorFactory;
import io.druid.query.aggregation.hyperloglog.HyperUniquesSerde; import io.druid.query.aggregation.hyperloglog.HyperUniquesSerde;
import io.druid.query.expression.TestExprMacroTable; import io.druid.query.expression.TestExprMacroTable;
import io.druid.segment.column.ValueType;
import io.druid.segment.incremental.IncrementalIndex; import io.druid.segment.incremental.IncrementalIndex;
import io.druid.segment.incremental.IncrementalIndexSchema; import io.druid.segment.incremental.IncrementalIndexSchema;
import io.druid.segment.incremental.OnheapIncrementalIndex; import io.druid.segment.incremental.OnheapIncrementalIndex;
@ -113,7 +114,7 @@ public class TestIndex
private static final Interval DATA_INTERVAL = new Interval("2011-01-12T00:00:00.000Z/2011-05-01T00:00:00.000Z"); private static final Interval DATA_INTERVAL = new Interval("2011-01-12T00:00:00.000Z/2011-05-01T00:00:00.000Z");
private static final VirtualColumns VIRTUAL_COLUMNS = VirtualColumns.create( private static final VirtualColumns VIRTUAL_COLUMNS = VirtualColumns.create(
Collections.<VirtualColumn>singletonList( Collections.<VirtualColumn>singletonList(
new ExpressionVirtualColumn("expr", "index + 10", TestExprMacroTable.INSTANCE) new ExpressionVirtualColumn("expr", "index + 10", ValueType.FLOAT, TestExprMacroTable.INSTANCE)
) )
); );
public static final AggregatorFactory[] METRIC_AGGS = new AggregatorFactory[]{ public static final AggregatorFactory[] METRIC_AGGS = new AggregatorFactory[]{

View File

@ -84,7 +84,7 @@ public abstract class BaseFilterTest
{ {
private static final VirtualColumns VIRTUAL_COLUMNS = VirtualColumns.create( private static final VirtualColumns VIRTUAL_COLUMNS = VirtualColumns.create(
ImmutableList.<VirtualColumn>of( ImmutableList.<VirtualColumn>of(
new ExpressionVirtualColumn("expr", "1.0 + 0.1", TestExprMacroTable.INSTANCE) new ExpressionVirtualColumn("expr", "1.0 + 0.1", ValueType.FLOAT, TestExprMacroTable.INSTANCE)
) )
); );

View File

@ -31,11 +31,29 @@ import java.util.List;
public class ExpressionObjectSelectorTest public class ExpressionObjectSelectorTest
{ {
@Test
public void testSupplierFromDimensionSelector()
{
final SettableSupplier<String> settableSupplier = new SettableSupplier<>();
final Supplier<Object> supplier = ExpressionObjectSelector.supplierFromDimensionSelector(
dimensionSelectorFromSupplier(settableSupplier)
);
Assert.assertNotNull(supplier);
Assert.assertEquals(null, supplier.get());
settableSupplier.set(null);
Assert.assertEquals(null, supplier.get());
settableSupplier.set("1234");
Assert.assertEquals("1234", supplier.get());
}
@Test @Test
public void testSupplierFromObjectSelectorObject() public void testSupplierFromObjectSelectorObject()
{ {
final SettableSupplier<Object> settableSupplier = new SettableSupplier<>(); final SettableSupplier<Object> settableSupplier = new SettableSupplier<>();
final Supplier<Number> supplier = ExpressionObjectSelector.supplierFromObjectSelector( final Supplier<Object> supplier = ExpressionObjectSelector.supplierFromObjectSelector(
objectSelectorFromSupplier(settableSupplier, Object.class) objectSelectorFromSupplier(settableSupplier, Object.class)
); );
@ -49,17 +67,17 @@ public class ExpressionObjectSelectorTest
Assert.assertEquals(1L, supplier.get()); Assert.assertEquals(1L, supplier.get());
settableSupplier.set("1234"); settableSupplier.set("1234");
Assert.assertEquals(1234L, supplier.get()); Assert.assertEquals("1234", supplier.get());
settableSupplier.set("1.234"); settableSupplier.set("1.234");
Assert.assertEquals(1.234d, supplier.get()); Assert.assertEquals("1.234", supplier.get());
} }
@Test @Test
public void testSupplierFromObjectSelectorNumber() public void testSupplierFromObjectSelectorNumber()
{ {
final SettableSupplier<Number> settableSupplier = new SettableSupplier<>(); final SettableSupplier<Number> settableSupplier = new SettableSupplier<>();
final Supplier<Number> supplier = ExpressionObjectSelector.supplierFromObjectSelector( final Supplier<Object> supplier = ExpressionObjectSelector.supplierFromObjectSelector(
objectSelectorFromSupplier(settableSupplier, Number.class) objectSelectorFromSupplier(settableSupplier, Number.class)
); );
@ -78,7 +96,7 @@ public class ExpressionObjectSelectorTest
public void testSupplierFromObjectSelectorString() public void testSupplierFromObjectSelectorString()
{ {
final SettableSupplier<String> settableSupplier = new SettableSupplier<>(); final SettableSupplier<String> settableSupplier = new SettableSupplier<>();
final Supplier<Number> supplier = ExpressionObjectSelector.supplierFromObjectSelector( final Supplier<Object> supplier = ExpressionObjectSelector.supplierFromObjectSelector(
objectSelectorFromSupplier(settableSupplier, String.class) objectSelectorFromSupplier(settableSupplier, String.class)
); );
@ -86,17 +104,17 @@ public class ExpressionObjectSelectorTest
Assert.assertEquals(null, supplier.get()); Assert.assertEquals(null, supplier.get());
settableSupplier.set("1.1"); settableSupplier.set("1.1");
Assert.assertEquals(1.1d, supplier.get()); Assert.assertEquals("1.1", supplier.get());
settableSupplier.set("1"); settableSupplier.set("1");
Assert.assertEquals(1L, supplier.get()); Assert.assertEquals("1", supplier.get());
} }
@Test @Test
public void testSupplierFromObjectSelectorList() public void testSupplierFromObjectSelectorList()
{ {
final SettableSupplier<List> settableSupplier = new SettableSupplier<>(); final SettableSupplier<List> settableSupplier = new SettableSupplier<>();
final Supplier<Number> supplier = ExpressionObjectSelector.supplierFromObjectSelector( final Supplier<Object> supplier = ExpressionObjectSelector.supplierFromObjectSelector(
objectSelectorFromSupplier(settableSupplier, List.class) objectSelectorFromSupplier(settableSupplier, List.class)
); );

View File

@ -34,6 +34,7 @@ import io.druid.segment.DimensionSelector;
import io.druid.segment.FloatColumnSelector; import io.druid.segment.FloatColumnSelector;
import io.druid.segment.LongColumnSelector; import io.druid.segment.LongColumnSelector;
import io.druid.segment.ObjectColumnSelector; import io.druid.segment.ObjectColumnSelector;
import io.druid.segment.column.ValueType;
import org.junit.Assert; import org.junit.Assert;
import org.junit.Test; import org.junit.Test;
@ -41,30 +42,49 @@ public class ExpressionVirtualColumnTest
{ {
private static final InputRow ROW0 = new MapBasedInputRow( private static final InputRow ROW0 = new MapBasedInputRow(
0, 0,
ImmutableList.<String>of(), ImmutableList.of(),
ImmutableMap.<String, Object>of() ImmutableMap.of()
); );
private static final InputRow ROW1 = new MapBasedInputRow( private static final InputRow ROW1 = new MapBasedInputRow(
0, 0,
ImmutableList.<String>of(), ImmutableList.of(),
ImmutableMap.<String, Object>of("x", 4) ImmutableMap.of("x", 4)
); );
private static final InputRow ROW2 = new MapBasedInputRow( private static final InputRow ROW2 = new MapBasedInputRow(
0, 0,
ImmutableList.<String>of(), ImmutableList.of(),
ImmutableMap.<String, Object>of("x", 2.1, "y", 3L) ImmutableMap.of("x", 2.1, "y", 3L, "z", "foobar")
);
private static final InputRow ROW3 = new MapBasedInputRow(
0,
ImmutableList.of(),
ImmutableMap.of("x", 2L, "y", 3L, "z", "foobar")
); );
private static final ExpressionVirtualColumn XPLUSY = new ExpressionVirtualColumn( private static final ExpressionVirtualColumn XPLUSY = new ExpressionVirtualColumn(
"expr", "expr",
"x + y", "x + y",
ValueType.FLOAT,
TestExprMacroTable.INSTANCE TestExprMacroTable.INSTANCE
); );
private static final ExpressionVirtualColumn CONSTANT_LIKE = new ExpressionVirtualColumn( private static final ExpressionVirtualColumn CONSTANT_LIKE = new ExpressionVirtualColumn(
"expr", "expr",
"like('foo', 'f%')", "like('foo', 'f%')",
ValueType.FLOAT,
TestExprMacroTable.INSTANCE
);
private static final ExpressionVirtualColumn ZLIKE = new ExpressionVirtualColumn(
"expr",
"like(z, 'f%')",
ValueType.FLOAT,
TestExprMacroTable.INSTANCE
);
private static final ExpressionVirtualColumn ZCONCATX = new ExpressionVirtualColumn(
"expr",
"z + cast(x, 'string')",
ValueType.STRING,
TestExprMacroTable.INSTANCE TestExprMacroTable.INSTANCE
); );
private static final TestColumnSelectorFactory COLUMN_SELECTOR_FACTORY = new TestColumnSelectorFactory(); private static final TestColumnSelectorFactory COLUMN_SELECTOR_FACTORY = new TestColumnSelectorFactory();
@ -78,10 +98,13 @@ public class ExpressionVirtualColumnTest
Assert.assertEquals(null, selector.get()); Assert.assertEquals(null, selector.get());
COLUMN_SELECTOR_FACTORY.setRow(ROW1); COLUMN_SELECTOR_FACTORY.setRow(ROW1);
Assert.assertEquals(null, selector.get()); Assert.assertEquals(4.0d, selector.get());
COLUMN_SELECTOR_FACTORY.setRow(ROW2); COLUMN_SELECTOR_FACTORY.setRow(ROW2);
Assert.assertEquals(5.1d, selector.get()); Assert.assertEquals(5.1d, selector.get());
COLUMN_SELECTOR_FACTORY.setRow(ROW3);
Assert.assertEquals(5L, selector.get());
} }
@Test @Test
@ -93,10 +116,31 @@ public class ExpressionVirtualColumnTest
Assert.assertEquals(0L, selector.get()); Assert.assertEquals(0L, selector.get());
COLUMN_SELECTOR_FACTORY.setRow(ROW1); COLUMN_SELECTOR_FACTORY.setRow(ROW1);
Assert.assertEquals(0L, selector.get()); Assert.assertEquals(4L, selector.get());
COLUMN_SELECTOR_FACTORY.setRow(ROW2); COLUMN_SELECTOR_FACTORY.setRow(ROW2);
Assert.assertEquals(5L, selector.get()); Assert.assertEquals(5L, selector.get());
COLUMN_SELECTOR_FACTORY.setRow(ROW3);
Assert.assertEquals(5L, selector.get());
}
@Test
public void testLongSelectorUsingStringFunction()
{
final LongColumnSelector selector = ZCONCATX.makeLongColumnSelector("expr", COLUMN_SELECTOR_FACTORY);
COLUMN_SELECTOR_FACTORY.setRow(ROW0);
Assert.assertEquals(0L, selector.get());
COLUMN_SELECTOR_FACTORY.setRow(ROW1);
Assert.assertEquals(4L, selector.get());
COLUMN_SELECTOR_FACTORY.setRow(ROW2);
Assert.assertEquals(0L, selector.get());
COLUMN_SELECTOR_FACTORY.setRow(ROW3);
Assert.assertEquals(0L, selector.get());
} }
@Test @Test
@ -108,17 +152,20 @@ public class ExpressionVirtualColumnTest
Assert.assertEquals(0.0f, selector.get(), 0.0f); Assert.assertEquals(0.0f, selector.get(), 0.0f);
COLUMN_SELECTOR_FACTORY.setRow(ROW1); COLUMN_SELECTOR_FACTORY.setRow(ROW1);
Assert.assertEquals(0.0f, selector.get(), 0.0f); Assert.assertEquals(4.0f, selector.get(), 0.0f);
COLUMN_SELECTOR_FACTORY.setRow(ROW2); COLUMN_SELECTOR_FACTORY.setRow(ROW2);
Assert.assertEquals(5.1f, selector.get(), 0.0f); Assert.assertEquals(5.1f, selector.get(), 0.0f);
COLUMN_SELECTOR_FACTORY.setRow(ROW3);
Assert.assertEquals(5.0f, selector.get(), 0.0f);
} }
@Test @Test
public void testDimensionSelector() public void testDimensionSelector()
{ {
final DimensionSelector selector = XPLUSY.makeDimensionSelector( final DimensionSelector selector = XPLUSY.makeDimensionSelector(
new DefaultDimensionSpec("expr", "x"), new DefaultDimensionSpec("expr", "expr"),
COLUMN_SELECTOR_FACTORY COLUMN_SELECTOR_FACTORY
); );
@ -133,16 +180,49 @@ public class ExpressionVirtualColumnTest
Assert.assertEquals(null, selector.lookupName(selector.getRow().get(0))); Assert.assertEquals(null, selector.lookupName(selector.getRow().get(0)));
COLUMN_SELECTOR_FACTORY.setRow(ROW1); COLUMN_SELECTOR_FACTORY.setRow(ROW1);
Assert.assertEquals(true, nullMatcher.matches()); Assert.assertEquals(false, nullMatcher.matches());
Assert.assertEquals(false, fiveMatcher.matches()); Assert.assertEquals(false, fiveMatcher.matches());
Assert.assertEquals(false, nonNullMatcher.matches()); Assert.assertEquals(true, nonNullMatcher.matches());
Assert.assertEquals(null, selector.lookupName(selector.getRow().get(0))); Assert.assertEquals("4.0", selector.lookupName(selector.getRow().get(0)));
COLUMN_SELECTOR_FACTORY.setRow(ROW2); COLUMN_SELECTOR_FACTORY.setRow(ROW2);
Assert.assertEquals(false, nullMatcher.matches()); Assert.assertEquals(false, nullMatcher.matches());
Assert.assertEquals(false, fiveMatcher.matches()); Assert.assertEquals(false, fiveMatcher.matches());
Assert.assertEquals(true, nonNullMatcher.matches()); Assert.assertEquals(true, nonNullMatcher.matches());
Assert.assertEquals("5.1", selector.lookupName(selector.getRow().get(0))); Assert.assertEquals("5.1", selector.lookupName(selector.getRow().get(0)));
COLUMN_SELECTOR_FACTORY.setRow(ROW3);
Assert.assertEquals(false, nullMatcher.matches());
Assert.assertEquals(true, fiveMatcher.matches());
Assert.assertEquals(true, nonNullMatcher.matches());
Assert.assertEquals("5", selector.lookupName(selector.getRow().get(0)));
}
@Test
public void testDimensionSelectorUsingStringFunction()
{
final DimensionSelector selector = ZCONCATX.makeDimensionSelector(
new DefaultDimensionSpec("expr", "expr"),
COLUMN_SELECTOR_FACTORY
);
Assert.assertNotNull(selector);
COLUMN_SELECTOR_FACTORY.setRow(ROW0);
Assert.assertEquals(1, selector.getRow().size());
Assert.assertEquals(null, selector.lookupName(selector.getRow().get(0)));
COLUMN_SELECTOR_FACTORY.setRow(ROW1);
Assert.assertEquals(1, selector.getRow().size());
Assert.assertEquals("4", selector.lookupName(selector.getRow().get(0)));
COLUMN_SELECTOR_FACTORY.setRow(ROW2);
Assert.assertEquals(1, selector.getRow().size());
Assert.assertEquals("foobar2.1", selector.lookupName(selector.getRow().get(0)));
COLUMN_SELECTOR_FACTORY.setRow(ROW3);
Assert.assertEquals(1, selector.getRow().size());
Assert.assertEquals("foobar2", selector.lookupName(selector.getRow().get(0)));
} }
@Test @Test
@ -164,16 +244,22 @@ public class ExpressionVirtualColumnTest
Assert.assertEquals(null, selector.lookupName(selector.getRow().get(0))); Assert.assertEquals(null, selector.lookupName(selector.getRow().get(0)));
COLUMN_SELECTOR_FACTORY.setRow(ROW1); COLUMN_SELECTOR_FACTORY.setRow(ROW1);
Assert.assertEquals(true, nullMatcher.matches()); Assert.assertEquals(false, nullMatcher.matches());
Assert.assertEquals(false, fiveMatcher.matches()); Assert.assertEquals(false, fiveMatcher.matches());
Assert.assertEquals(false, nonNullMatcher.matches()); Assert.assertEquals(true, nonNullMatcher.matches());
Assert.assertEquals(null, selector.lookupName(selector.getRow().get(0))); Assert.assertEquals("4", selector.lookupName(selector.getRow().get(0)));
COLUMN_SELECTOR_FACTORY.setRow(ROW2); COLUMN_SELECTOR_FACTORY.setRow(ROW2);
Assert.assertEquals(false, nullMatcher.matches()); Assert.assertEquals(false, nullMatcher.matches());
Assert.assertEquals(true, fiveMatcher.matches()); Assert.assertEquals(true, fiveMatcher.matches());
Assert.assertEquals(true, nonNullMatcher.matches()); Assert.assertEquals(true, nonNullMatcher.matches());
Assert.assertEquals("5", selector.lookupName(selector.getRow().get(0))); Assert.assertEquals("5", selector.lookupName(selector.getRow().get(0)));
COLUMN_SELECTOR_FACTORY.setRow(ROW3);
Assert.assertEquals(false, nullMatcher.matches());
Assert.assertEquals(true, fiveMatcher.matches());
Assert.assertEquals(true, nonNullMatcher.matches());
Assert.assertEquals("5", selector.lookupName(selector.getRow().get(0)));
} }
@Test @Test
@ -185,9 +271,30 @@ public class ExpressionVirtualColumnTest
Assert.assertEquals(1L, selector.get()); Assert.assertEquals(1L, selector.get());
} }
@Test
public void testLongSelectorWithZLikeExprMacro()
{
final LongColumnSelector selector = ZLIKE.makeLongColumnSelector("expr", COLUMN_SELECTOR_FACTORY);
COLUMN_SELECTOR_FACTORY.setRow(ROW0);
Assert.assertEquals(0L, selector.get());
COLUMN_SELECTOR_FACTORY.setRow(ROW1);
Assert.assertEquals(0L, selector.get());
COLUMN_SELECTOR_FACTORY.setRow(ROW2);
Assert.assertEquals(1L, selector.get());
COLUMN_SELECTOR_FACTORY.setRow(ROW3);
Assert.assertEquals(1L, selector.get());
}
@Test @Test
public void testRequiredColumns() public void testRequiredColumns()
{ {
Assert.assertEquals(ImmutableList.of("x", "y"), XPLUSY.requiredColumns()); Assert.assertEquals(ImmutableList.of("x", "y"), XPLUSY.requiredColumns());
Assert.assertEquals(ImmutableList.of(), CONSTANT_LIKE.requiredColumns());
Assert.assertEquals(ImmutableList.of("z"), ZLIKE.requiredColumns());
Assert.assertEquals(ImmutableList.of("z", "x"), ZCONCATX.requiredColumns());
} }
} }

View File

@ -127,6 +127,7 @@ public class VirtualColumnsTest
final ExpressionVirtualColumn expr = new ExpressionVirtualColumn( final ExpressionVirtualColumn expr = new ExpressionVirtualColumn(
"__time", "__time",
"x + y", "x + y",
ValueType.FLOAT,
TestExprMacroTable.INSTANCE TestExprMacroTable.INSTANCE
); );
@ -142,12 +143,14 @@ public class VirtualColumnsTest
final ExpressionVirtualColumn expr = new ExpressionVirtualColumn( final ExpressionVirtualColumn expr = new ExpressionVirtualColumn(
"expr", "expr",
"x + y", "x + y",
ValueType.FLOAT,
TestExprMacroTable.INSTANCE TestExprMacroTable.INSTANCE
); );
final ExpressionVirtualColumn expr2 = new ExpressionVirtualColumn( final ExpressionVirtualColumn expr2 = new ExpressionVirtualColumn(
"expr", "expr",
"x * 2", "x * 2",
ValueType.FLOAT,
TestExprMacroTable.INSTANCE TestExprMacroTable.INSTANCE
); );
@ -163,12 +166,14 @@ public class VirtualColumnsTest
final ExpressionVirtualColumn expr = new ExpressionVirtualColumn( final ExpressionVirtualColumn expr = new ExpressionVirtualColumn(
"expr", "expr",
"x + expr2", "x + expr2",
ValueType.FLOAT,
TestExprMacroTable.INSTANCE TestExprMacroTable.INSTANCE
); );
final ExpressionVirtualColumn expr2 = new ExpressionVirtualColumn( final ExpressionVirtualColumn expr2 = new ExpressionVirtualColumn(
"expr2", "expr2",
"expr * 2", "expr * 2",
ValueType.FLOAT,
TestExprMacroTable.INSTANCE TestExprMacroTable.INSTANCE
); );
@ -183,13 +188,13 @@ public class VirtualColumnsTest
{ {
final VirtualColumns virtualColumns = VirtualColumns.create( final VirtualColumns virtualColumns = VirtualColumns.create(
ImmutableList.<VirtualColumn>of( ImmutableList.<VirtualColumn>of(
new ExpressionVirtualColumn("expr", "x + y", TestExprMacroTable.INSTANCE) new ExpressionVirtualColumn("expr", "x + y", ValueType.FLOAT, TestExprMacroTable.INSTANCE)
) )
); );
final VirtualColumns virtualColumns2 = VirtualColumns.create( final VirtualColumns virtualColumns2 = VirtualColumns.create(
ImmutableList.<VirtualColumn>of( ImmutableList.<VirtualColumn>of(
new ExpressionVirtualColumn("expr", "x + y", TestExprMacroTable.INSTANCE) new ExpressionVirtualColumn("expr", "x + y", ValueType.FLOAT, TestExprMacroTable.INSTANCE)
) )
); );
@ -202,13 +207,13 @@ public class VirtualColumnsTest
{ {
final VirtualColumns virtualColumns = VirtualColumns.create( final VirtualColumns virtualColumns = VirtualColumns.create(
ImmutableList.<VirtualColumn>of( ImmutableList.<VirtualColumn>of(
new ExpressionVirtualColumn("expr", "x + y", TestExprMacroTable.INSTANCE) new ExpressionVirtualColumn("expr", "x + y", ValueType.FLOAT, TestExprMacroTable.INSTANCE)
) )
); );
final VirtualColumns virtualColumns2 = VirtualColumns.create( final VirtualColumns virtualColumns2 = VirtualColumns.create(
ImmutableList.<VirtualColumn>of( ImmutableList.<VirtualColumn>of(
new ExpressionVirtualColumn("expr", "x + y", TestExprMacroTable.INSTANCE) new ExpressionVirtualColumn("expr", "x + y", ValueType.FLOAT, TestExprMacroTable.INSTANCE)
) )
); );
@ -227,8 +232,8 @@ public class VirtualColumnsTest
{ {
final ObjectMapper mapper = TestHelper.getJsonMapper(); final ObjectMapper mapper = TestHelper.getJsonMapper();
final ImmutableList<VirtualColumn> theColumns = ImmutableList.<VirtualColumn>of( final ImmutableList<VirtualColumn> theColumns = ImmutableList.<VirtualColumn>of(
new ExpressionVirtualColumn("expr", "x + y", TestExprMacroTable.INSTANCE), new ExpressionVirtualColumn("expr", "x + y", ValueType.FLOAT, TestExprMacroTable.INSTANCE),
new ExpressionVirtualColumn("expr2", "x + z", TestExprMacroTable.INSTANCE) new ExpressionVirtualColumn("expr2", "x + z", ValueType.FLOAT, TestExprMacroTable.INSTANCE)
); );
final VirtualColumns virtualColumns = VirtualColumns.create(theColumns); final VirtualColumns virtualColumns = VirtualColumns.create(theColumns);
@ -254,6 +259,7 @@ public class VirtualColumnsTest
final ExpressionVirtualColumn expr = new ExpressionVirtualColumn( final ExpressionVirtualColumn expr = new ExpressionVirtualColumn(
"expr", "expr",
"1", "1",
ValueType.FLOAT,
TestExprMacroTable.INSTANCE TestExprMacroTable.INSTANCE
); );
final DottyVirtualColumn dotty = new DottyVirtualColumn("foo"); final DottyVirtualColumn dotty = new DottyVirtualColumn("foo");