diff --git a/core/src/main/java/org/apache/druid/math/expr/IdentifierExpr.java b/core/src/main/java/org/apache/druid/math/expr/IdentifierExpr.java index a41d7a99575..01622a5b268 100644 --- a/core/src/main/java/org/apache/druid/math/expr/IdentifierExpr.java +++ b/core/src/main/java/org/apache/druid/math/expr/IdentifierExpr.java @@ -23,12 +23,11 @@ import org.apache.commons.lang.StringEscapeUtils; import org.apache.druid.java.util.common.StringUtils; import org.apache.druid.math.expr.vector.ExprEvalDoubleVector; import org.apache.druid.math.expr.vector.ExprEvalLongVector; -import org.apache.druid.math.expr.vector.ExprEvalStringVector; +import org.apache.druid.math.expr.vector.ExprEvalObjectVector; import org.apache.druid.math.expr.vector.ExprEvalVector; import org.apache.druid.math.expr.vector.ExprVectorProcessor; import javax.annotation.Nullable; -import java.util.Arrays; import java.util.Objects; /** @@ -158,15 +157,12 @@ class IdentifierExpr implements Expr if (inputType == null) { // nil column, we can be anything, so be a string because it's the most flexible // (numbers will be populated with default values in default mode and non-null) - return new IdentifierVectorProcessor(ExpressionType.STRING) + return new IdentifierVectorProcessor(ExpressionType.STRING) { @Override - public ExprEvalVector evalVector(VectorInputBinding bindings) + public ExprEvalVector evalVector(VectorInputBinding bindings) { - // need to cast to string[] because null columns come out as object[] - return new ExprEvalStringVector( - Arrays.stream(bindings.getObjectVector(binding)).map(x -> (String) x).toArray(String[]::new) - ); + return new ExprEvalObjectVector(bindings.getObjectVector(binding)); } }; } @@ -190,12 +186,12 @@ class IdentifierExpr implements Expr } }; case STRING: - return new IdentifierVectorProcessor(inputType) + return new IdentifierVectorProcessor(inputType) { @Override - public ExprEvalVector evalVector(VectorInputBinding bindings) + public ExprEvalVector evalVector(VectorInputBinding bindings) { - return new ExprEvalStringVector(bindings.getObjectVector(binding)); + return new ExprEvalObjectVector(bindings.getObjectVector(binding)); } }; default: diff --git a/core/src/main/java/org/apache/druid/math/expr/vector/CastToStringVectorProcessor.java b/core/src/main/java/org/apache/druid/math/expr/vector/CastToStringVectorProcessor.java index 27e0d1b9cf7..b02afd4f88f 100644 --- a/core/src/main/java/org/apache/druid/math/expr/vector/CastToStringVectorProcessor.java +++ b/core/src/main/java/org/apache/druid/math/expr/vector/CastToStringVectorProcessor.java @@ -23,7 +23,7 @@ import org.apache.druid.math.expr.Evals; import org.apache.druid.math.expr.Expr; import org.apache.druid.math.expr.ExpressionType; -public final class CastToStringVectorProcessor extends CastToTypeVectorProcessor +public final class CastToStringVectorProcessor extends CastToTypeVectorProcessor { public CastToStringVectorProcessor(ExprVectorProcessor delegate) { @@ -31,15 +31,15 @@ public final class CastToStringVectorProcessor extends CastToTypeVectorProcessor } @Override - public ExprEvalVector evalVector(Expr.VectorInputBinding bindings) + public ExprEvalVector evalVector(Expr.VectorInputBinding bindings) { ExprEvalVector result = delegate.evalVector(bindings); final Object[] objects = result.getObjectVector(); - final String[] output = new String[objects.length]; + final Object[] output = new String[objects.length]; for (int i = 0; i < objects.length; i++) { output[i] = Evals.asString(objects[i]); } - return new ExprEvalStringVector(output); + return new ExprEvalObjectVector(output); } @Override diff --git a/core/src/main/java/org/apache/druid/math/expr/vector/ExprEvalStringVector.java b/core/src/main/java/org/apache/druid/math/expr/vector/ExprEvalObjectVector.java similarity index 90% rename from core/src/main/java/org/apache/druid/math/expr/vector/ExprEvalStringVector.java rename to core/src/main/java/org/apache/druid/math/expr/vector/ExprEvalObjectVector.java index 257e5ecdf80..eaeb0f04f1e 100644 --- a/core/src/main/java/org/apache/druid/math/expr/vector/ExprEvalStringVector.java +++ b/core/src/main/java/org/apache/druid/math/expr/vector/ExprEvalObjectVector.java @@ -20,12 +20,13 @@ package org.apache.druid.math.expr.vector; import org.apache.druid.common.config.NullHandling; +import org.apache.druid.math.expr.Evals; import org.apache.druid.math.expr.ExprEval; import org.apache.druid.math.expr.ExpressionType; import javax.annotation.Nullable; -public final class ExprEvalStringVector extends ExprEvalVector +public final class ExprEvalObjectVector extends ExprEvalVector { @Nullable private long[] longs; @@ -35,7 +36,7 @@ public final class ExprEvalStringVector extends ExprEvalVector @Nullable private boolean[] numericNulls; - public ExprEvalStringVector(String[] values) + public ExprEvalObjectVector(Object[] values) { super(values, null); } @@ -47,7 +48,7 @@ public final class ExprEvalStringVector extends ExprEvalVector doubles = new double[values.length]; numericNulls = new boolean[values.length]; for (int i = 0; i < values.length; i++) { - Number n = ExprEval.computeNumber(values[i]); + Number n = ExprEval.computeNumber(Evals.asString(values[i])); if (n != null) { longs[i] = n.longValue(); doubles[i] = n.doubleValue(); diff --git a/core/src/main/java/org/apache/druid/math/expr/vector/ExprEvalVector.java b/core/src/main/java/org/apache/druid/math/expr/vector/ExprEvalVector.java index fdb6605989e..723b34f12c2 100644 --- a/core/src/main/java/org/apache/druid/math/expr/vector/ExprEvalVector.java +++ b/core/src/main/java/org/apache/druid/math/expr/vector/ExprEvalVector.java @@ -29,7 +29,7 @@ import java.lang.reflect.Array; * Result of {@link ExprVectorProcessor#evalVector} which wraps the actual evaluated results of the operation over the * input vector(s). Methods to get actual results mirror vectorized value and object selectors. * - * The generic parameter T should be the native java array type of the vector result (long[], String[], etc.) + * The generic parameter T should be the native java array type of the vector result (long[], Object[], etc.) */ public abstract class ExprEvalVector { diff --git a/core/src/main/java/org/apache/druid/math/expr/vector/LongOutStringInFunctionVectorProcessor.java b/core/src/main/java/org/apache/druid/math/expr/vector/LongOutObjectInFunctionVectorProcessor.java similarity index 71% rename from core/src/main/java/org/apache/druid/math/expr/vector/LongOutStringInFunctionVectorProcessor.java rename to core/src/main/java/org/apache/druid/math/expr/vector/LongOutObjectInFunctionVectorProcessor.java index 2ebc5190861..8ae0c4d7519 100644 --- a/core/src/main/java/org/apache/druid/math/expr/vector/LongOutStringInFunctionVectorProcessor.java +++ b/core/src/main/java/org/apache/druid/math/expr/vector/LongOutObjectInFunctionVectorProcessor.java @@ -22,14 +22,21 @@ package org.apache.druid.math.expr.vector; import org.apache.druid.math.expr.ExpressionType; /** - * specialized {@link UnivariateFunctionVectorObjectProcessor} for processing (String[]) -> long[] + * specialized {@link UnivariateFunctionVectorObjectProcessor} for processing (Object[]) -> long[] */ -public abstract class LongOutStringInFunctionVectorProcessor - extends UnivariateFunctionVectorObjectProcessor +public abstract class LongOutObjectInFunctionVectorProcessor + extends UnivariateFunctionVectorObjectProcessor { - public LongOutStringInFunctionVectorProcessor(ExprVectorProcessor processor, int maxVectorSize) + final ExpressionType inputType; + + public LongOutObjectInFunctionVectorProcessor( + ExprVectorProcessor processor, + int maxVectorSize, + ExpressionType inputType + ) { - super(CastToTypeVectorProcessor.cast(processor, ExpressionType.STRING), maxVectorSize, new long[maxVectorSize]); + super(CastToTypeVectorProcessor.cast(processor, inputType), maxVectorSize, new long[maxVectorSize]); + this.inputType = inputType; } @Override diff --git a/core/src/main/java/org/apache/druid/math/expr/vector/LongOutStringsInFunctionVectorProcessor.java b/core/src/main/java/org/apache/druid/math/expr/vector/LongOutObjectsInFunctionVectorProcessor.java similarity index 71% rename from core/src/main/java/org/apache/druid/math/expr/vector/LongOutStringsInFunctionVectorProcessor.java rename to core/src/main/java/org/apache/druid/math/expr/vector/LongOutObjectsInFunctionVectorProcessor.java index d450b863304..88f2f0ff54b 100644 --- a/core/src/main/java/org/apache/druid/math/expr/vector/LongOutStringsInFunctionVectorProcessor.java +++ b/core/src/main/java/org/apache/druid/math/expr/vector/LongOutObjectsInFunctionVectorProcessor.java @@ -23,19 +23,21 @@ import org.apache.druid.math.expr.ExpressionType; import javax.annotation.Nullable; -public abstract class LongOutStringsInFunctionVectorProcessor extends BivariateFunctionVectorObjectProcessor +public abstract class LongOutObjectsInFunctionVectorProcessor + extends BivariateFunctionVectorObjectProcessor { private final boolean[] outNulls; - protected LongOutStringsInFunctionVectorProcessor( - ExprVectorProcessor left, - ExprVectorProcessor right, - int maxVectorSize + protected LongOutObjectsInFunctionVectorProcessor( + ExprVectorProcessor left, + ExprVectorProcessor right, + int maxVectorSize, + ExpressionType inputType ) { super( - CastToTypeVectorProcessor.cast(left, ExpressionType.STRING), - CastToTypeVectorProcessor.cast(right, ExpressionType.STRING), + CastToTypeVectorProcessor.cast(left, inputType), + CastToTypeVectorProcessor.cast(right, inputType), maxVectorSize, new long[maxVectorSize] ); @@ -43,12 +45,12 @@ public abstract class LongOutStringsInFunctionVectorProcessor extends BivariateF } @Nullable - abstract Long processValue(@Nullable String leftVal, @Nullable String rightVal); + abstract Long processValue(@Nullable Object leftVal, @Nullable Object rightVal); @Override - void processIndex(String[] strings, String[] strings2, int i) + void processIndex(Object[] in1, Object[] in2, int i) { - final Long outVal = processValue(strings[i], strings2[i]); + final Long outVal = processValue(in1[i], in2[i]); if (outVal == null) { outValues[i] = 0L; outNulls[i] = true; diff --git a/core/src/main/java/org/apache/druid/math/expr/vector/StringOutMultiStringInVectorProcessor.java b/core/src/main/java/org/apache/druid/math/expr/vector/ObjectOutMultiObjectInVectorProcessor.java similarity index 67% rename from core/src/main/java/org/apache/druid/math/expr/vector/StringOutMultiStringInVectorProcessor.java rename to core/src/main/java/org/apache/druid/math/expr/vector/ObjectOutMultiObjectInVectorProcessor.java index 1008098d605..2a58e3ea826 100644 --- a/core/src/main/java/org/apache/druid/math/expr/vector/StringOutMultiStringInVectorProcessor.java +++ b/core/src/main/java/org/apache/druid/math/expr/vector/ObjectOutMultiObjectInVectorProcessor.java @@ -19,41 +19,43 @@ package org.apache.druid.math.expr.vector; -import org.apache.druid.common.config.NullHandling; import org.apache.druid.math.expr.Expr; import org.apache.druid.math.expr.ExpressionType; /** * many strings enter, one string leaves... */ -public abstract class StringOutMultiStringInVectorProcessor implements ExprVectorProcessor +public abstract class ObjectOutMultiObjectInVectorProcessor implements ExprVectorProcessor { - final ExprVectorProcessor[] inputs; + final ExprVectorProcessor[] inputs; final int maxVectorSize; - final String[] outValues; - final boolean sqlCompatible = NullHandling.sqlCompatible(); + final Object[] outValues; - protected StringOutMultiStringInVectorProcessor( - ExprVectorProcessor[] inputs, - int maxVectorSize + final ExpressionType expressionType; + + protected ObjectOutMultiObjectInVectorProcessor( + ExprVectorProcessor[] inputs, + int maxVectorSize, + ExpressionType objectType ) { this.inputs = inputs; this.maxVectorSize = maxVectorSize; - this.outValues = new String[maxVectorSize]; + this.outValues = new Object[maxVectorSize]; + this.expressionType = objectType; } @Override public ExpressionType getOutputType() { - return ExpressionType.STRING; + return expressionType; } @Override - public ExprEvalVector evalVector(Expr.VectorInputBinding bindings) + public ExprEvalVector evalVector(Expr.VectorInputBinding bindings) { final int currentSize = bindings.getCurrentVectorSize(); - final String[][] in = new String[inputs.length][]; + final Object[][] in = new Object[inputs.length][]; for (int i = 0; i < inputs.length; i++) { in[i] = inputs[i].evalVector(bindings).values(); } @@ -61,8 +63,8 @@ public abstract class StringOutMultiStringInVectorProcessor implements ExprVecto for (int i = 0; i < currentSize; i++) { processIndex(in, i); } - return new ExprEvalStringVector(outValues); + return new ExprEvalObjectVector(outValues); } - abstract void processIndex(String[][] in, int i); + abstract void processIndex(Object[][] in, int i); } diff --git a/core/src/main/java/org/apache/druid/math/expr/vector/StringOutStringsInFunctionVectorProcessor.java b/core/src/main/java/org/apache/druid/math/expr/vector/ObjectOutObjectsInFunctionVectorProcessor.java similarity index 56% rename from core/src/main/java/org/apache/druid/math/expr/vector/StringOutStringsInFunctionVectorProcessor.java rename to core/src/main/java/org/apache/druid/math/expr/vector/ObjectOutObjectsInFunctionVectorProcessor.java index 7bd7ed12772..93089f5fc5e 100644 --- a/core/src/main/java/org/apache/druid/math/expr/vector/StringOutStringsInFunctionVectorProcessor.java +++ b/core/src/main/java/org/apache/druid/math/expr/vector/ObjectOutObjectsInFunctionVectorProcessor.java @@ -23,30 +23,34 @@ import org.apache.druid.math.expr.ExpressionType; import javax.annotation.Nullable; -public abstract class StringOutStringsInFunctionVectorProcessor - extends BivariateFunctionVectorObjectProcessor +public abstract class ObjectOutObjectsInFunctionVectorProcessor + extends BivariateFunctionVectorObjectProcessor { - protected StringOutStringsInFunctionVectorProcessor( - ExprVectorProcessor left, - ExprVectorProcessor right, - int maxVectorSize + final ExpressionType expressionType; + + protected ObjectOutObjectsInFunctionVectorProcessor( + ExprVectorProcessor left, + ExprVectorProcessor right, + int maxVectorSize, + ExpressionType expressionType ) { super( - CastToTypeVectorProcessor.cast(left, ExpressionType.STRING), - CastToTypeVectorProcessor.cast(right, ExpressionType.STRING), + CastToTypeVectorProcessor.cast(left, expressionType), + CastToTypeVectorProcessor.cast(right, expressionType), maxVectorSize, - new String[maxVectorSize] + new Object[maxVectorSize] ); + this.expressionType = expressionType; } @Nullable - protected abstract String processValue(@Nullable String leftVal, @Nullable String rightVal); + protected abstract Object processValue(@Nullable Object leftVal, @Nullable Object rightVal); @Override - void processIndex(String[] strings, String[] strings2, int i) + void processIndex(Object[] in1, Object[] in2, int i) { - outValues[i] = processValue(strings[i], strings2[i]); + outValues[i] = processValue(in1[i], in2[i]); } @Override @@ -56,14 +60,14 @@ public abstract class StringOutStringsInFunctionVectorProcessor } @Override - ExprEvalVector asEval() + ExprEvalVector asEval() { - return new ExprEvalStringVector(outValues); + return new ExprEvalObjectVector(outValues); } @Override public ExpressionType getOutputType() { - return ExpressionType.STRING; + return expressionType; } } diff --git a/core/src/main/java/org/apache/druid/math/expr/vector/VectorComparisonProcessors.java b/core/src/main/java/org/apache/druid/math/expr/vector/VectorComparisonProcessors.java index bf4e665925e..decfd9b3d49 100644 --- a/core/src/main/java/org/apache/druid/math/expr/vector/VectorComparisonProcessors.java +++ b/core/src/main/java/org/apache/druid/math/expr/vector/VectorComparisonProcessors.java @@ -38,7 +38,7 @@ public class VectorComparisonProcessors Expr.VectorInputBindingInspector inspector, Expr left, Expr right, - Supplier longOutStringsInFunctionVectorProcessor, + Supplier longOutStringsInFunctionVectorProcessor, Supplier longOutLongsInProcessor, Supplier doubleOutLongDoubleInProcessor, Supplier doubleOutDoubleLongInProcessor, @@ -81,7 +81,7 @@ public class VectorComparisonProcessors Expr.VectorInputBindingInspector inspector, Expr left, Expr right, - Supplier longOutStringsInFunctionVectorProcessor, + Supplier longOutStringsInFunctionVectorProcessor, Supplier longOutLongsInProcessor, Supplier longOutLongDoubleInProcessor, Supplier longOutDoubleLongInProcessor, @@ -136,15 +136,16 @@ public class VectorComparisonProcessors inspector, left, right, - () -> new LongOutStringsInFunctionVectorProcessor( + () -> new LongOutObjectsInFunctionVectorProcessor( left.buildVectorized(inspector), right.buildVectorized(inspector), - inspector.getMaxVectorSize() + inspector.getMaxVectorSize(), + ExpressionType.STRING ) { @Nullable @Override - Long processValue(@Nullable String leftVal, @Nullable String rightVal) + Long processValue(@Nullable Object leftVal, @Nullable Object rightVal) { return Evals.asLong(Objects.equals(leftVal, rightVal)); } @@ -203,15 +204,16 @@ public class VectorComparisonProcessors inspector, left, right, - () -> new LongOutStringsInFunctionVectorProcessor( + () -> new LongOutObjectsInFunctionVectorProcessor( left.buildVectorized(inspector), right.buildVectorized(inspector), - inspector.getMaxVectorSize() + inspector.getMaxVectorSize(), + ExpressionType.STRING ) { @Nullable @Override - Long processValue(@Nullable String leftVal, @Nullable String rightVal) + Long processValue(@Nullable Object leftVal, @Nullable Object rightVal) { return Evals.asLong(Objects.equals(leftVal, rightVal)); } @@ -278,15 +280,16 @@ public class VectorComparisonProcessors inspector, left, right, - () -> new LongOutStringsInFunctionVectorProcessor( + () -> new LongOutObjectsInFunctionVectorProcessor( left.buildVectorized(inspector), right.buildVectorized(inspector), - inspector.getMaxVectorSize() + inspector.getMaxVectorSize(), + ExpressionType.STRING ) { @Nullable @Override - Long processValue(@Nullable String leftVal, @Nullable String rightVal) + Long processValue(@Nullable Object leftVal, @Nullable Object rightVal) { return Evals.asLong(!Objects.equals(leftVal, rightVal)); } @@ -345,15 +348,16 @@ public class VectorComparisonProcessors inspector, left, right, - () -> new LongOutStringsInFunctionVectorProcessor( + () -> new LongOutObjectsInFunctionVectorProcessor( left.buildVectorized(inspector), right.buildVectorized(inspector), - inspector.getMaxVectorSize() + inspector.getMaxVectorSize(), + ExpressionType.STRING ) { @Nullable @Override - Long processValue(@Nullable String leftVal, @Nullable String rightVal) + Long processValue(@Nullable Object leftVal, @Nullable Object rightVal) { return Evals.asLong(!Objects.equals(leftVal, rightVal)); } @@ -420,17 +424,20 @@ public class VectorComparisonProcessors inspector, left, right, - () -> new LongOutStringsInFunctionVectorProcessor( + () -> new LongOutObjectsInFunctionVectorProcessor( left.buildVectorized(inspector), right.buildVectorized(inspector), - inspector.getMaxVectorSize() + inspector.getMaxVectorSize(), + ExpressionType.STRING ) { @Nullable @Override - Long processValue(@Nullable String leftVal, @Nullable String rightVal) + Long processValue(@Nullable Object leftVal, @Nullable Object rightVal) { - return Evals.asLong(Comparators.naturalNullsFirst().compare(leftVal, rightVal) >= 0); + return Evals.asLong( + Comparators.naturalNullsFirst().compare((String) leftVal, (String) rightVal) >= 0 + ); } }, () -> new LongOutLongsInFunctionVectorValueProcessor( @@ -487,17 +494,20 @@ public class VectorComparisonProcessors inspector, left, right, - () -> new LongOutStringsInFunctionVectorProcessor( + () -> new LongOutObjectsInFunctionVectorProcessor( left.buildVectorized(inspector), right.buildVectorized(inspector), - inspector.getMaxVectorSize() + inspector.getMaxVectorSize(), + ExpressionType.STRING ) { @Nullable @Override - Long processValue(@Nullable String leftVal, @Nullable String rightVal) + Long processValue(@Nullable Object leftVal, @Nullable Object rightVal) { - return Evals.asLong(Comparators.naturalNullsFirst().compare(leftVal, rightVal) >= 0); + return Evals.asLong( + Comparators.naturalNullsFirst().compare((String) leftVal, (String) rightVal) >= 0 + ); } }, () -> new LongOutLongsInFunctionVectorValueProcessor( @@ -562,17 +572,20 @@ public class VectorComparisonProcessors inspector, left, right, - () -> new LongOutStringsInFunctionVectorProcessor( + () -> new LongOutObjectsInFunctionVectorProcessor( left.buildVectorized(inspector), right.buildVectorized(inspector), - inspector.getMaxVectorSize() + inspector.getMaxVectorSize(), + ExpressionType.STRING ) { @Nullable @Override - Long processValue(@Nullable String leftVal, @Nullable String rightVal) + Long processValue(@Nullable Object leftVal, @Nullable Object rightVal) { - return Evals.asLong(Comparators.naturalNullsFirst().compare(leftVal, rightVal) > 0); + return Evals.asLong( + Comparators.naturalNullsFirst().compare((String) leftVal, (String) rightVal) > 0 + ); } }, () -> new LongOutLongsInFunctionVectorValueProcessor( @@ -629,17 +642,20 @@ public class VectorComparisonProcessors inspector, left, right, - () -> new LongOutStringsInFunctionVectorProcessor( + () -> new LongOutObjectsInFunctionVectorProcessor( left.buildVectorized(inspector), right.buildVectorized(inspector), - inspector.getMaxVectorSize() + inspector.getMaxVectorSize(), + ExpressionType.STRING ) { @Nullable @Override - Long processValue(@Nullable String leftVal, @Nullable String rightVal) + Long processValue(@Nullable Object leftVal, @Nullable Object rightVal) { - return Evals.asLong(Comparators.naturalNullsFirst().compare(leftVal, rightVal) > 0); + return Evals.asLong( + Comparators.naturalNullsFirst().compare((String) leftVal, (String) rightVal) > 0 + ); } }, () -> new LongOutLongsInFunctionVectorValueProcessor( @@ -704,17 +720,20 @@ public class VectorComparisonProcessors inspector, left, right, - () -> new LongOutStringsInFunctionVectorProcessor( + () -> new LongOutObjectsInFunctionVectorProcessor( left.buildVectorized(inspector), right.buildVectorized(inspector), - inspector.getMaxVectorSize() + inspector.getMaxVectorSize(), + ExpressionType.STRING ) { @Nullable @Override - Long processValue(@Nullable String leftVal, @Nullable String rightVal) + Long processValue(@Nullable Object leftVal, @Nullable Object rightVal) { - return Evals.asLong(Comparators.naturalNullsFirst().compare(leftVal, rightVal) <= 0); + return Evals.asLong( + Comparators.naturalNullsFirst().compare((String) leftVal, (String) rightVal) <= 0 + ); } }, () -> new LongOutLongsInFunctionVectorValueProcessor( @@ -771,17 +790,20 @@ public class VectorComparisonProcessors inspector, left, right, - () -> new LongOutStringsInFunctionVectorProcessor( + () -> new LongOutObjectsInFunctionVectorProcessor( left.buildVectorized(inspector), right.buildVectorized(inspector), - inspector.getMaxVectorSize() + inspector.getMaxVectorSize(), + ExpressionType.STRING ) { @Nullable @Override - Long processValue(@Nullable String leftVal, @Nullable String rightVal) + Long processValue(@Nullable Object leftVal, @Nullable Object rightVal) { - return Evals.asLong(Comparators.naturalNullsFirst().compare(leftVal, rightVal) <= 0); + return Evals.asLong( + Comparators.naturalNullsFirst().compare((String) leftVal, (String) rightVal) <= 0 + ); } }, () -> new LongOutLongsInFunctionVectorValueProcessor( @@ -846,17 +868,20 @@ public class VectorComparisonProcessors inspector, left, right, - () -> new LongOutStringsInFunctionVectorProcessor( + () -> new LongOutObjectsInFunctionVectorProcessor( left.buildVectorized(inspector), right.buildVectorized(inspector), - inspector.getMaxVectorSize() + inspector.getMaxVectorSize(), + ExpressionType.STRING ) { @Nullable @Override - Long processValue(@Nullable String leftVal, @Nullable String rightVal) + Long processValue(@Nullable Object leftVal, @Nullable Object rightVal) { - return Evals.asLong(Comparators.naturalNullsFirst().compare(leftVal, rightVal) < 0); + return Evals.asLong( + Comparators.naturalNullsFirst().compare((String) leftVal, (String) rightVal) < 0 + ); } }, () -> new LongOutLongsInFunctionVectorValueProcessor( @@ -913,17 +938,20 @@ public class VectorComparisonProcessors inspector, left, right, - () -> new LongOutStringsInFunctionVectorProcessor( + () -> new LongOutObjectsInFunctionVectorProcessor( left.buildVectorized(inspector), right.buildVectorized(inspector), - inspector.getMaxVectorSize() + inspector.getMaxVectorSize(), + ExpressionType.STRING ) { @Nullable @Override - Long processValue(@Nullable String leftVal, @Nullable String rightVal) + Long processValue(@Nullable Object leftVal, @Nullable Object rightVal) { - return Evals.asLong(Comparators.naturalNullsFirst().compare(leftVal, rightVal) < 0); + return Evals.asLong( + Comparators.naturalNullsFirst().compare((String) leftVal, (String) rightVal) < 0 + ); } }, () -> new LongOutLongsInFunctionVectorValueProcessor( diff --git a/core/src/main/java/org/apache/druid/math/expr/vector/VectorProcessors.java b/core/src/main/java/org/apache/druid/math/expr/vector/VectorProcessors.java index 4a548ca10d1..ee066065ad1 100644 --- a/core/src/main/java/org/apache/druid/math/expr/vector/VectorProcessors.java +++ b/core/src/main/java/org/apache/druid/math/expr/vector/VectorProcessors.java @@ -82,9 +82,9 @@ public class VectorProcessors public static ExprVectorProcessor constant(@Nullable String constant, int maxVectorSize) { - final String[] strings = new String[maxVectorSize]; + final Object[] strings = new Object[maxVectorSize]; Arrays.fill(strings, constant); - final ExprEvalStringVector eval = new ExprEvalStringVector(strings); + final ExprEvalObjectVector eval = new ExprEvalObjectVector(strings); return new ExprVectorProcessor() { @Override @@ -159,23 +159,29 @@ public class VectorProcessors public static ExprVectorProcessor parseLong(Expr.VectorInputBindingInspector inspector, Expr arg, int radix) { - final ExprVectorProcessor processor = new LongOutStringInFunctionVectorProcessor( - CastToTypeVectorProcessor.cast(arg.buildVectorized(inspector), ExpressionType.STRING), - inspector.getMaxVectorSize() + final ExprVectorProcessor processor = new LongOutObjectInFunctionVectorProcessor( + arg.buildVectorized(inspector), + inspector.getMaxVectorSize(), + ExpressionType.STRING ) { @Override - public void processIndex(String[] strings, long[] longs, boolean[] outputNulls, int i) + public void processIndex(Object[] strings, long[] longs, boolean[] outputNulls, int i) { try { - final String input = strings[i]; - if (radix == 16 && (input.startsWith("0x") || input.startsWith("0X"))) { - // Strip leading 0x from hex strings. - longs[i] = Long.parseLong(input.substring(2), radix); + final String input = (String) strings[i]; + if (input == null) { + longs[i] = 0L; + outputNulls[i] = NullHandling.sqlCompatible(); } else { - longs[i] = Long.parseLong(input, radix); + if (radix == 16 && (input.startsWith("0x") || input.startsWith("0X"))) { + // Strip leading 0x from hex strings. + longs[i] = Long.parseLong(input.substring(2), radix); + } else { + longs[i] = Long.parseLong(input, radix); + } + outputNulls[i] = false; } - outputNulls[i] = false; } catch (NumberFormatException e) { longs[i] = 0L; @@ -199,16 +205,16 @@ public class VectorProcessors ExprVectorProcessor processor = null; if (Types.is(type, ExprType.STRING)) { - final ExprVectorProcessor input = expr.buildVectorized(inspector); + final ExprVectorProcessor input = expr.buildVectorized(inspector); processor = new ExprVectorProcessor() { @Override public ExprEvalVector evalVector(Expr.VectorInputBinding bindings) { - final ExprEvalVector inputEval = input.evalVector(bindings); + final ExprEvalVector inputEval = input.evalVector(bindings); final int currentSize = bindings.getCurrentVectorSize(); - final String[] values = inputEval.values(); + final Object[] values = inputEval.values(); for (int i = 0; i < currentSize; i++) { if (values[i] == null) { outputValues[i] = 1L; @@ -307,16 +313,16 @@ public class VectorProcessors ExprVectorProcessor processor = null; if (Types.is(type, ExprType.STRING)) { - final ExprVectorProcessor input = expr.buildVectorized(inspector); + final ExprVectorProcessor input = expr.buildVectorized(inspector); processor = new ExprVectorProcessor() { @Override public ExprEvalVector evalVector(Expr.VectorInputBinding bindings) { - final ExprEvalVector inputEval = input.evalVector(bindings); + final ExprEvalVector inputEval = input.evalVector(bindings); final int currentSize = bindings.getCurrentVectorSize(); - final String[] values = inputEval.values(); + final Object[] values = inputEval.values(); for (int i = 0; i < currentSize; i++) { if (values[i] == null) { outputValues[i] = 0L; @@ -483,19 +489,19 @@ public class VectorProcessors return new ExprEvalDoubleVector(output, outputNulls); } }, - () -> new SymmetricalBivariateFunctionVectorProcessor( + () -> new SymmetricalBivariateFunctionVectorProcessor( ExpressionType.STRING, left.buildVectorized(inspector), right.buildVectorized(inspector) ) { - final String[] output = new String[maxVectorSize]; + final Object[] output = new Object[maxVectorSize]; @Override public void processIndex( - String[] leftInput, + Object[] leftInput, @Nullable boolean[] leftNulls, - String[] rightInput, + Object[] rightInput, @Nullable boolean[] rightNulls, int i ) @@ -504,9 +510,9 @@ public class VectorProcessors } @Override - public ExprEvalVector asEval() + public ExprEvalVector asEval() { - return new ExprEvalStringVector(output); + return new ExprEvalObjectVector(output); } } ); @@ -518,14 +524,18 @@ public class VectorProcessors final int maxVectorSize = inspector.getMaxVectorSize(); ExprVectorProcessor processor = null; if (Types.is(inputType, ExprType.STRING)) { - processor = new LongOutStringInFunctionVectorProcessor(expr.buildVectorized(inspector), maxVectorSize) + processor = new LongOutObjectInFunctionVectorProcessor( + expr.buildVectorized(inspector), + maxVectorSize, + ExpressionType.STRING + ) { @Override - public void processIndex(String[] strings, long[] longs, boolean[] outputNulls, int i) + public void processIndex(Object[] strings, long[] longs, boolean[] outputNulls, int i) { outputNulls[i] = strings[i] == null; if (!outputNulls[i]) { - longs[i] = Evals.asLong(!Evals.asBoolean(strings[i])); + longs[i] = Evals.asLong(!Evals.asBoolean((String) strings[i])); } } }; @@ -670,7 +680,7 @@ public class VectorProcessors return new ExprEvalLongVector(output, outputNulls); } }, - () -> new BivariateFunctionVectorProcessor( + () -> new BivariateFunctionVectorProcessor( ExpressionType.LONG, left.buildVectorized(inspector), right.buildVectorized(inspector) @@ -681,9 +691,9 @@ public class VectorProcessors @Override public void processIndex( - String[] leftInput, + Object[] leftInput, @Nullable boolean[] leftNulls, - String[] rightInput, + Object[] rightInput, @Nullable boolean[] rightNulls, int i ) @@ -697,17 +707,17 @@ public class VectorProcessors outputNulls[i] = true; return; } - final boolean bool = Evals.asBoolean(rightInput[i]); + final boolean bool = Evals.asBoolean((String) rightInput[i]); output[i] = Evals.asLong(bool); outputNulls[i] = !bool; return; } else if (rightNull) { - final boolean bool = Evals.asBoolean(leftInput[i]); + final boolean bool = Evals.asBoolean((String) leftInput[i]); output[i] = Evals.asLong(bool); outputNulls[i] = !bool; return; } - output[i] = Evals.asLong(Evals.asBoolean(leftInput[i]) || Evals.asBoolean(rightInput[i])); + output[i] = Evals.asLong(Evals.asBoolean((String) leftInput[i]) || Evals.asBoolean((String) rightInput[i])); } @Override @@ -824,7 +834,7 @@ public class VectorProcessors return new ExprEvalLongVector(output, outputNulls); } }, - () -> new BivariateFunctionVectorProcessor( + () -> new BivariateFunctionVectorProcessor( ExpressionType.STRING, left.buildVectorized(inputTypes), right.buildVectorized(inputTypes) @@ -835,9 +845,9 @@ public class VectorProcessors @Override public void processIndex( - String[] leftInput, + Object[] leftInput, @Nullable boolean[] leftNulls, - String[] rightInput, + Object[] rightInput, @Nullable boolean[] rightNulls, int i ) @@ -851,17 +861,19 @@ public class VectorProcessors outputNulls[i] = true; return; } - final boolean bool = Evals.asBoolean(rightInput[i]); + final boolean bool = Evals.asBoolean((String) rightInput[i]); output[i] = Evals.asLong(bool); outputNulls[i] = bool; return; } else if (rightNull) { - final boolean bool = Evals.asBoolean(leftInput[i]); + final boolean bool = Evals.asBoolean((String) leftInput[i]); output[i] = Evals.asLong(bool); outputNulls[i] = bool; return; } - output[i] = Evals.asLong(Evals.asBoolean(leftInput[i]) && Evals.asBoolean(rightInput[i])); + output[i] = Evals.asLong( + Evals.asBoolean((String) leftInput[i]) && Evals.asBoolean((String) rightInput[i]) + ); } @Override diff --git a/core/src/main/java/org/apache/druid/math/expr/vector/VectorStringProcessors.java b/core/src/main/java/org/apache/druid/math/expr/vector/VectorStringProcessors.java index 35c966a7546..1ffaa2518d3 100644 --- a/core/src/main/java/org/apache/druid/math/expr/vector/VectorStringProcessors.java +++ b/core/src/main/java/org/apache/druid/math/expr/vector/VectorStringProcessors.java @@ -20,6 +20,7 @@ package org.apache.druid.math.expr.vector; import org.apache.druid.common.config.NullHandling; +import org.apache.druid.math.expr.Evals; import org.apache.druid.math.expr.Expr; import org.apache.druid.math.expr.ExpressionType; @@ -32,32 +33,35 @@ public class VectorStringProcessors { final ExprVectorProcessor processor; if (NullHandling.sqlCompatible()) { - processor = new StringOutStringsInFunctionVectorProcessor( + processor = new ObjectOutObjectsInFunctionVectorProcessor( left.buildVectorized(inspector), right.buildVectorized(inspector), - inspector.getMaxVectorSize() + inspector.getMaxVectorSize(), + ExpressionType.STRING ) { @Nullable @Override - protected String processValue(@Nullable String leftVal, @Nullable String rightVal) + protected String processValue(@Nullable Object leftVal, @Nullable Object rightVal) { // in sql compatible mode, nulls are handled by super class and never make it here... - return leftVal + rightVal; + return leftVal + (String) rightVal; } }; } else { - processor = new StringOutStringsInFunctionVectorProcessor( + processor = new ObjectOutObjectsInFunctionVectorProcessor( left.buildVectorized(inspector), right.buildVectorized(inspector), - inspector.getMaxVectorSize() + inspector.getMaxVectorSize(), + ExpressionType.STRING ) { @Nullable @Override - protected String processValue(@Nullable String leftVal, @Nullable String rightVal) + protected Object processValue(@Nullable Object leftVal, @Nullable Object rightVal) { - return NullHandling.nullToEmptyIfNeeded(leftVal) + NullHandling.nullToEmptyIfNeeded(rightVal); + return NullHandling.nullToEmptyIfNeeded((String) leftVal) + + NullHandling.nullToEmptyIfNeeded((String) rightVal); } }; } @@ -66,28 +70,32 @@ public class VectorStringProcessors public static ExprVectorProcessor concat(Expr.VectorInputBindingInspector inspector, List inputs) { - final ExprVectorProcessor[] inputProcessors = new ExprVectorProcessor[inputs.size()]; + final ExprVectorProcessor[] inputProcessors = new ExprVectorProcessor[inputs.size()]; for (int i = 0; i < inputs.size(); i++) { - inputProcessors[i] = CastToTypeVectorProcessor.cast(inputs.get(i).buildVectorized(inspector), ExpressionType.STRING); + inputProcessors[i] = CastToTypeVectorProcessor.cast( + inputs.get(i).buildVectorized(inspector), + ExpressionType.STRING + ); } - final ExprVectorProcessor processor = new StringOutMultiStringInVectorProcessor( + final ExprVectorProcessor processor = new ObjectOutMultiObjectInVectorProcessor( inputProcessors, - inspector.getMaxVectorSize() + inspector.getMaxVectorSize(), + ExpressionType.STRING ) { @Override - void processIndex(String[][] in, int i) + void processIndex(Object[][] in, int i) { // Result of concatenation is null if any of the Values is null. // e.g. 'select CONCAT(null, "abc") as c;' will return null as per Standard SQL spec. - String first = NullHandling.nullToEmptyIfNeeded(in[0][i]); + String first = NullHandling.nullToEmptyIfNeeded(Evals.asString(in[0][i])); if (first == null) { outValues[i] = null; return; } final StringBuilder builder = new StringBuilder(first); for (int inputNumber = 1; inputNumber < in.length; inputNumber++) { - final String s = NullHandling.nullToEmptyIfNeeded(in[inputNumber][i]); + final String s = NullHandling.nullToEmptyIfNeeded(Evals.asString(in[inputNumber][i])); if (s == null) { outValues[i] = null; return; diff --git a/processing/src/main/java/org/apache/druid/segment/column/StringDictionaryEncodedColumn.java b/processing/src/main/java/org/apache/druid/segment/column/StringDictionaryEncodedColumn.java index e24273f036c..ba50991215b 100644 --- a/processing/src/main/java/org/apache/druid/segment/column/StringDictionaryEncodedColumn.java +++ b/processing/src/main/java/org/apache/druid/segment/column/StringDictionaryEncodedColumn.java @@ -561,7 +561,7 @@ public class StringDictionaryEncodedColumn implements DictionaryEncodedColumn T[] getObjectVector(String name) + public Object[] getObjectVector(String name) { - return (T[]) objects.getOrDefault(name, nilSelector).getObjectVector(); + return objects.getOrDefault(name, nilSelector).getObjectVector(); } @Override diff --git a/processing/src/main/java/org/apache/druid/segment/virtual/SingleStringInputDeferredEvaluationExpressionDimensionVectorSelector.java b/processing/src/main/java/org/apache/druid/segment/virtual/SingleStringInputDeferredEvaluationExpressionDimensionVectorSelector.java index 1cee307adb7..4b884972946 100644 --- a/processing/src/main/java/org/apache/druid/segment/virtual/SingleStringInputDeferredEvaluationExpressionDimensionVectorSelector.java +++ b/processing/src/main/java/org/apache/druid/segment/virtual/SingleStringInputDeferredEvaluationExpressionDimensionVectorSelector.java @@ -20,6 +20,7 @@ package org.apache.druid.segment.virtual; import org.apache.druid.java.util.common.ISE; +import org.apache.druid.math.expr.Evals; import org.apache.druid.math.expr.Expr; import org.apache.druid.math.expr.ExpressionType; import org.apache.druid.math.expr.vector.ExprVectorProcessor; @@ -43,7 +44,7 @@ public class SingleStringInputDeferredEvaluationExpressionDimensionVectorSelecto implements SingleValueDimensionVectorSelector { private final SingleValueDimensionVectorSelector selector; - private final ExprVectorProcessor stringProcessor; + private final ExprVectorProcessor stringProcessor; private final StringLookupVectorInputBindings inputBinding; public SingleStringInputDeferredEvaluationExpressionDimensionVectorSelector( @@ -75,7 +76,7 @@ public class SingleStringInputDeferredEvaluationExpressionDimensionVectorSelecto public String lookupName(int id) { inputBinding.currentValue[0] = selector.lookupName(id); - return stringProcessor.evalVector(inputBinding).values()[0]; + return Evals.asString(stringProcessor.evalVector(inputBinding).values()[0]); } @Override @@ -118,7 +119,7 @@ public class SingleStringInputDeferredEvaluationExpressionDimensionVectorSelecto */ private static final class StringLookupVectorInputBindings implements Expr.VectorInputBinding { - private final String[] currentValue = new String[1]; + private final Object[] currentValue = new Object[1]; @Nullable @Override diff --git a/processing/src/test/java/org/apache/druid/query/groupby/NestedDataGroupByQueryTest.java b/processing/src/test/java/org/apache/druid/query/groupby/NestedDataGroupByQueryTest.java index 30372012966..dac3295e3ca 100644 --- a/processing/src/test/java/org/apache/druid/query/groupby/NestedDataGroupByQueryTest.java +++ b/processing/src/test/java/org/apache/druid/query/groupby/NestedDataGroupByQueryTest.java @@ -35,12 +35,14 @@ import org.apache.druid.query.aggregation.AggregationTestHelper; import org.apache.druid.query.aggregation.CountAggregatorFactory; import org.apache.druid.query.aggregation.LongSumAggregatorFactory; import org.apache.druid.query.dimension.DefaultDimensionSpec; +import org.apache.druid.query.expression.TestExprMacroTable; import org.apache.druid.query.filter.InDimFilter; import org.apache.druid.query.groupby.strategy.GroupByStrategySelector; import org.apache.druid.segment.Segment; import org.apache.druid.segment.column.ColumnType; import org.apache.druid.segment.column.RowSignature; import org.apache.druid.segment.column.ValueType; +import org.apache.druid.segment.virtual.ExpressionVirtualColumn; import org.apache.druid.segment.virtual.NestedFieldVirtualColumn; import org.apache.druid.testing.InitializedNullHandlingTest; import org.junit.After; @@ -76,6 +78,8 @@ public class NestedDataGroupByQueryTest extends InitializedNullHandlingTest private final TrinaryFn> segmentsGenerator; private final String segmentsName; + private boolean cannotVectorize = false; + public NestedDataGroupByQueryTest( GroupByQueryConfig config, TrinaryFn> segmentGenerator, @@ -273,6 +277,46 @@ public class NestedDataGroupByQueryTest extends InitializedNullHandlingTest ); } + @Test + public void testGroupByNonExistentVirtualColumn() + { + if (GroupByStrategySelector.STRATEGY_V1.equals(config.getDefaultStrategy())) { + expectedException.expect(RuntimeException.class); + expectedException.expectMessage( + "GroupBy v1 does not support dimension selectors with unknown cardinality." + ); + } + GroupByQuery groupQuery = GroupByQuery.builder() + .setDataSource("test_datasource") + .setGranularity(Granularities.ALL) + .setInterval(Intervals.ETERNITY) + .setDimensions(DefaultDimensionSpec.of("v1")) + .setVirtualColumns( + new NestedFieldVirtualColumn("fake", "$.fake", "v0", ColumnType.STRING), + new ExpressionVirtualColumn( + "v1", + "concat(v0, 'foo')", + ColumnType.STRING, + TestExprMacroTable.INSTANCE + ) + ) + .setAggregatorSpecs(new CountAggregatorFactory("count")) + .setContext(getContext()) + .build(); + + + Sequence seq = helper.runQueryOnSegmentsObjs(segmentsGenerator.apply(helper, tempFolder, closer), groupQuery); + + List results = seq.toList(); + verifyResults( + groupQuery.getResultRowSignature(), + results, + NullHandling.sqlCompatible() + ? ImmutableList.of(new Object[]{null, 16L}) + : ImmutableList.of(new Object[]{"foo", 16L}) + ); + } + private static void verifyResults(RowSignature rowSignature, List results, List expected) { LOG.info("results:\n%s", results); diff --git a/processing/src/test/java/org/apache/druid/segment/virtual/ExpressionVectorObjectSelectorTest.java b/processing/src/test/java/org/apache/druid/segment/virtual/ExpressionVectorObjectSelectorTest.java index 3fde3338861..8773bbd6cd2 100644 --- a/processing/src/test/java/org/apache/druid/segment/virtual/ExpressionVectorObjectSelectorTest.java +++ b/processing/src/test/java/org/apache/druid/segment/virtual/ExpressionVectorObjectSelectorTest.java @@ -20,7 +20,7 @@ package org.apache.druid.segment.virtual; import org.apache.druid.math.expr.Expr; -import org.apache.druid.math.expr.vector.ExprEvalStringVector; +import org.apache.druid.math.expr.vector.ExprEvalObjectVector; import org.apache.druid.math.expr.vector.ExprEvalVector; import org.apache.druid.math.expr.vector.ExprVectorProcessor; import org.easymock.EasyMock; @@ -57,7 +57,7 @@ public class ExpressionVectorObjectSelectorTest public void testSelectObject() { final String[] vector = new String[]{"1", "2", null, "3"}; - ExprEvalVector vectorEval = new ExprEvalStringVector(vector); + ExprEvalVector vectorEval = new ExprEvalObjectVector(vector); EasyMock.expect(binding.getCurrentVectorId()).andReturn(1).anyTimes(); EasyMock.expect(vectorProcessor.evalVector(binding)).andReturn(vectorEval).once(); EasyMock.replay(binding, vectorProcessor); diff --git a/processing/src/test/java/org/apache/druid/segment/virtual/ExpressionVectorSelectorsTest.java b/processing/src/test/java/org/apache/druid/segment/virtual/ExpressionVectorSelectorsTest.java index c3e6a928508..8e0055d44cf 100644 --- a/processing/src/test/java/org/apache/druid/segment/virtual/ExpressionVectorSelectorsTest.java +++ b/processing/src/test/java/org/apache/druid/segment/virtual/ExpressionVectorSelectorsTest.java @@ -43,6 +43,7 @@ import org.apache.druid.segment.vector.SingleValueDimensionVectorSelector; import org.apache.druid.segment.vector.VectorCursor; import org.apache.druid.segment.vector.VectorObjectSelector; import org.apache.druid.segment.vector.VectorValueSelector; +import org.apache.druid.testing.InitializedNullHandlingTest; import org.apache.druid.timeline.DataSegment; import org.apache.druid.timeline.partition.LinearShardSpec; import org.junit.AfterClass; @@ -60,7 +61,7 @@ import java.util.List; import java.util.stream.Collectors; @RunWith(Parameterized.class) -public class ExpressionVectorSelectorsTest +public class ExpressionVectorSelectorsTest extends InitializedNullHandlingTest { private static List EXPRESSIONS = ImmutableList.of( "long1 * long2",